From 6bb654a4b7c3fa31644640f0491510f74e69d80c Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Mon, 21 Sep 2026 06:00:53 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=8C=81=E4=B9=85=E5=8C=96=E6=B8=85?= =?UTF-8?q?=E7=90=86=20VM=20=E6=89=A7=E8=A1=8C=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/gitea-dynamic-runner/controller.go | 20 +++++- docs/opensandbox-runner.md | 9 +-- internal/opensandboxbackend/backend.go | 73 +++++++++++++++++++++ internal/opensandboxbackend/backend_test.go | 35 ++++++++++ 4 files changed, 131 insertions(+), 6 deletions(-) diff --git a/cmd/gitea-dynamic-runner/controller.go b/cmd/gitea-dynamic-runner/controller.go index e416a90..51f372d 100644 --- a/cmd/gitea-dynamic-runner/controller.go +++ b/cmd/gitea-dynamic-runner/controller.go @@ -88,17 +88,26 @@ func runController(ctx context.Context) error { registry := runnerfacade.NewRegistry() gate := taskscheduler.NewSingleFlightGate() var podExecutorBackend *podbackend.Backend + var vmExecutorBackend *opensandboxbackend.Backend giteaClient := giteaactions.NewClient(giteaactions.DefaultHTTPClient(), config.GiteaURL, config.GiteaUUID, config.GiteaToken) facade := &runnerfacade.Facade{ Registry: registry, Capabilities: capabilities, Upstream: giteaClient, OnTerminal: func(ctx context.Context, assignment taskassignment.Assignment) error { - if assignment.Backend == taskassignment.BackendPod { + switch assignment.Backend { + case taskassignment.BackendPod: if podExecutorBackend == nil { return errors.New("Pod lifecycle backend is not configured") } if err := podExecutorBackend.MarkTerminal(ctx, assignment.ID); err != nil { return err } + case taskassignment.BackendVM: + if vmExecutorBackend == nil { + return errors.New("VM lifecycle backend is not configured") + } + if err := vmExecutorBackend.MarkTerminal(ctx, assignment.ID); err != nil { + return err + } } gate.Release() return nil @@ -168,11 +177,18 @@ func runController(ctx context.Context) error { Entrypoint: []string{"/usr/local/bin/gitea-dynamic-runner", "executor"}, Env: map[string]string{"SPIFFE_ENDPOINT_SOCKET": config.WorkloadAPIAddr}, }} + vmExecutorBackend = &backend component, err := workerComponent(ctx, workerJS, config, taskassignment.BackendVM, config.VMCapacity, taskworker.Worker{Backend: backend, Bootstrap: bootstrap}, registry) if err != nil { return err } - components[controller.VMWorker] = component + lifecycleReconciler := opensandboxbackend.LifecycleReconciler{Backend: backend, OnError: func(err error) { log.Printf("VM lifecycle: %v", err) }} + components[controller.VMWorker] = runComponent(func(ctx context.Context) error { + group, groupContext := errgroup.WithContext(ctx) + group.Go(func() error { return component.Run(groupContext) }) + group.Go(func() error { return lifecycleReconciler.Run(groupContext) }) + return group.Wait() + }) } return controller.Run(ctx, config.Components, components) } diff --git a/docs/opensandbox-runner.md b/docs/opensandbox-runner.md index 908fb3d..7919a19 100644 --- a/docs/opensandbox-runner.md +++ b/docs/opensandbox-runner.md @@ -48,7 +48,8 @@ UID attestation 的临时 Agent 失去父级。 ## 清理与恢复 -controller 监控 Lifecycle 状态，在终止、失败、超时或取消时调用 DELETE。API delete、 -identity entry delete 均接受对象已不存在。controller 重启时，OpenSandbox timeout -仍是最终回收边界；后续可基于 metadata list 恢复主动监控，但不得为此重新引入消息 -队列。 +Gitea 接受 runner 终态后，facade 先通过 Lifecycle API 将 +`ci.ddupan.top/terminal=true` 持久化到 sandbox metadata，再向 runner 返回成功。VM +lifecycle reconciler 按该 metadata 查询并调用 DELETE；controller 在标记与删除之间重启 +也能恢复清理。API delete、identity entry delete 均接受对象已不存在，OpenSandbox +timeout 仍是最终兜底回收边界；生命周期状态不写入消息队列或新的数据库。 diff --git a/internal/opensandboxbackend/backend.go b/internal/opensandboxbackend/backend.go index 2cad2de..7976b6c 100644 --- a/internal/opensandboxbackend/backend.go +++ b/internal/opensandboxbackend/backend.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/http" + "time" opensandbox "github.com/alibaba/OpenSandbox/sdks/sandbox/go" @@ -15,14 +16,86 @@ import ( ) const assignmentMetadata = "ci.ddupan.top/assignment-id" +const terminalMetadata = "ci.ddupan.top/terminal" type Lifecycle interface { ListSandboxes(context.Context, opensandbox.ListOptions) (*opensandbox.ListSandboxesResponse, error) CreateSandbox(context.Context, opensandbox.CreateSandboxRequest) (*opensandbox.SandboxInfo, error) GetSandbox(context.Context, string) (*opensandbox.SandboxInfo, error) + PatchSandboxMetadata(context.Context, string, opensandbox.MetadataPatch) (*opensandbox.SandboxInfo, error) DeleteSandbox(context.Context, string) error } +// MarkTerminal persists the accepted Gitea terminal state on the sandbox. The +// lifecycle reconciler performs deletion separately so the runner receives the +// successful UpdateTask response before its VM is stopped. +func (b Backend) MarkTerminal(ctx context.Context, assignmentID string) error { + executor, err := b.Find(ctx, assignmentID) + if err != nil || executor == nil { + return err + } + value := "true" + _, err = b.Lifecycle.PatchSandboxMetadata(ctx, executor.Name, opensandbox.MetadataPatch{ + terminalMetadata: &value, + }) + if err != nil { + return fmt.Errorf("mark sandbox %s terminal: %w", executor.Name, err) + } + return nil +} + +// CleanupTerminated removes sandboxes whose terminal result was accepted by +// Gitea. The marker is stored by OpenSandbox, so cleanup survives restarts. +func (b Backend) CleanupTerminated(ctx context.Context) (int, error) { + if err := b.validate(); err != nil { + return 0, err + } + result, err := b.Lifecycle.ListSandboxes(ctx, opensandbox.ListOptions{ + Metadata: map[string]string{terminalMetadata: "true"}, + PageSize: 100, + }) + if err != nil { + return 0, fmt.Errorf("list terminal sandboxes: %w", err) + } + cleaned := 0 + for _, sandbox := range result.Items { + if sandbox.Metadata[assignmentMetadata] == "" { + continue + } + if err := b.Delete(ctx, executor(sandbox)); err != nil { + return cleaned, fmt.Errorf("delete terminal sandbox %s: %w", sandbox.ID, err) + } + cleaned++ + } + return cleaned, nil +} + +// Lifecycle periodically reconciles durable terminal markers into deletes. +type LifecycleReconciler struct { + Backend Backend + Interval time.Duration + OnError func(error) +} + +func (l LifecycleReconciler) Run(ctx context.Context) error { + interval := l.Interval + if interval <= 0 { + interval = 2 * time.Second + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + if _, err := l.Backend.CleanupTerminated(ctx); err != nil && ctx.Err() == nil && l.OnError != nil { + l.OnError(err) + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} + type Config struct { Pool string Timeout int diff --git a/internal/opensandboxbackend/backend_test.go b/internal/opensandboxbackend/backend_test.go index 07ee979..703265c 100644 --- a/internal/opensandboxbackend/backend_test.go +++ b/internal/opensandboxbackend/backend_test.go @@ -33,6 +33,23 @@ func (f *fakeLifecycle) GetSandbox(_ context.Context, id string) (*opensandbox.S } return &opensandbox.SandboxInfo{ID: id, Metadata: map[string]string{"ci.ddupan.top/spiffe-id": assignment().Identity.SPIFFEID}}, nil } +func (f *fakeLifecycle) PatchSandboxMetadata(_ context.Context, id string, patch opensandbox.MetadataPatch) (*opensandbox.SandboxInfo, error) { + for index := range f.items { + if f.items[index].ID != id { + continue + } + if f.items[index].Metadata == nil { + f.items[index].Metadata = map[string]string{} + } + for key, value := range patch { + if value != nil { + f.items[index].Metadata[key] = *value + } + } + return &f.items[index], nil + } + return &opensandbox.SandboxInfo{ID: id}, nil +} func (f *fakeLifecycle) DeleteSandbox(_ context.Context, id string) error { f.deleted = id; return nil } func backend(lifecycle Lifecycle) Backend { @@ -86,3 +103,21 @@ func TestBindIdentityVerifiesPersistedMetadata(t *testing.T) { t.Fatal(err) } } + +func TestTerminalMarkerDrivesDurableCleanup(t *testing.T) { + lifecycle := &fakeLifecycle{items: []opensandbox.SandboxInfo{{ + ID: "sandbox-42", + Metadata: map[string]string{assignmentMetadata: assignment().ID}, + }}} + backend := backend(lifecycle) + if err := backend.MarkTerminal(context.Background(), assignment().ID); err != nil { + t.Fatal(err) + } + if lifecycle.items[0].Metadata[terminalMetadata] != "true" { + t.Fatalf("metadata = %#v", lifecycle.items[0].Metadata) + } + cleaned, err := backend.CleanupTerminated(context.Background()) + if err != nil || cleaned != 1 || lifecycle.deleted != "sandbox-42" { + t.Fatalf("cleaned=%d deleted=%q err=%v", cleaned, lifecycle.deleted, err) + } +}