feat: 持久化清理 VM 执行器

cmd/gitea-dynamic-runner/controller.go

@@ -88,17 +88,26 @@ func runController(ctx context.Context) error {
 	registry := runnerfacade.NewRegistry()
 	gate := taskscheduler.NewSingleFlightGate()
 	var podExecutorBackend *podbackend.Backend
+	var vmExecutorBackend *opensandboxbackend.Backend
 	giteaClient := giteaactions.NewClient(giteaactions.DefaultHTTPClient(), config.GiteaURL, config.GiteaUUID, config.GiteaToken)
 	facade := &runnerfacade.Facade{
 		Registry: registry, Capabilities: capabilities, Upstream: giteaClient,
 		OnTerminal: func(ctx context.Context, assignment taskassignment.Assignment) error {
-			if assignment.Backend == taskassignment.BackendPod {
+			switch assignment.Backend {
+			case taskassignment.BackendPod:
 				if podExecutorBackend == nil {
 					return errors.New("Pod lifecycle backend is not configured")
 				}
 				if err := podExecutorBackend.MarkTerminal(ctx, assignment.ID); err != nil {
 					return err
 				}
+			case taskassignment.BackendVM:
+				if vmExecutorBackend == nil {
+					return errors.New("VM lifecycle backend is not configured")
+				}
+				if err := vmExecutorBackend.MarkTerminal(ctx, assignment.ID); err != nil {
+					return err
+				}
 			}
 			gate.Release()
 			return nil
@@ -168,11 +177,18 @@ func runController(ctx context.Context) error {
 			Entrypoint: []string{"/usr/local/bin/gitea-dynamic-runner", "executor"},
 			Env:        map[string]string{"SPIFFE_ENDPOINT_SOCKET": config.WorkloadAPIAddr},
 		}}
+		vmExecutorBackend = &backend
 		component, err := workerComponent(ctx, workerJS, config, taskassignment.BackendVM, config.VMCapacity, taskworker.Worker{Backend: backend, Bootstrap: bootstrap}, registry)
 		if err != nil {
 			return err
 		}
-		components[controller.VMWorker] = component
+		lifecycleReconciler := opensandboxbackend.LifecycleReconciler{Backend: backend, OnError: func(err error) { log.Printf("VM lifecycle: %v", err) }}
+		components[controller.VMWorker] = runComponent(func(ctx context.Context) error {
+			group, groupContext := errgroup.WithContext(ctx)
+			group.Go(func() error { return component.Run(groupContext) })
+			group.Go(func() error { return lifecycleReconciler.Run(groupContext) })
+			return group.Wait()
+		})
 	}
 	return controller.Run(ctx, config.Components, components)
 }

docs/opensandbox-runner.md

@@ -48,7 +48,8 @@ UID attestation 的临时 Agent 失去父级。
 
 ## 清理与恢复
 
-controller 监控 Lifecycle 状态，在终止、失败、超时或取消时调用 DELETE。API delete、
+Gitea 接受 runner 终态后，facade 先通过 Lifecycle API 将
-identity entry delete 均接受对象已不存在。controller 重启时，OpenSandbox timeout
+`ci.ddupan.top/terminal=true` 持久化到 sandbox metadata，再向 runner 返回成功。VM
-仍是最终回收边界；后续可基于 metadata list 恢复主动监控，但不得为此重新引入消息
+lifecycle reconciler 按该 metadata 查询并调用 DELETE；controller 在标记与删除之间重启
-队列。
+也能恢复清理。API delete、identity entry delete 均接受对象已不存在，OpenSandbox
+timeout 仍是最终兜底回收边界；生命周期状态不写入消息队列或新的数据库。

internal/opensandboxbackend/backend.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"time"
 
 	opensandbox "github.com/alibaba/OpenSandbox/sdks/sandbox/go"
 
@@ -15,14 +16,86 @@ import (
 )
 
 const assignmentMetadata = "ci.ddupan.top/assignment-id"
+const terminalMetadata = "ci.ddupan.top/terminal"
 
 type Lifecycle interface {
 	ListSandboxes(context.Context, opensandbox.ListOptions) (*opensandbox.ListSandboxesResponse, error)
 	CreateSandbox(context.Context, opensandbox.CreateSandboxRequest) (*opensandbox.SandboxInfo, error)
 	GetSandbox(context.Context, string) (*opensandbox.SandboxInfo, error)
+	PatchSandboxMetadata(context.Context, string, opensandbox.MetadataPatch) (*opensandbox.SandboxInfo, error)
 	DeleteSandbox(context.Context, string) error
 }
 
+// MarkTerminal persists the accepted Gitea terminal state on the sandbox. The
+// lifecycle reconciler performs deletion separately so the runner receives the
+// successful UpdateTask response before its VM is stopped.
+func (b Backend) MarkTerminal(ctx context.Context, assignmentID string) error {
+	executor, err := b.Find(ctx, assignmentID)
+	if err != nil || executor == nil {
+		return err
+	}
+	value := "true"
+	_, err = b.Lifecycle.PatchSandboxMetadata(ctx, executor.Name, opensandbox.MetadataPatch{
+		terminalMetadata: &value,
+	})
+	if err != nil {
+		return fmt.Errorf("mark sandbox %s terminal: %w", executor.Name, err)
+	}
+	return nil
+}
+
+// CleanupTerminated removes sandboxes whose terminal result was accepted by
+// Gitea. The marker is stored by OpenSandbox, so cleanup survives restarts.
+func (b Backend) CleanupTerminated(ctx context.Context) (int, error) {
+	if err := b.validate(); err != nil {
+		return 0, err
+	}
+	result, err := b.Lifecycle.ListSandboxes(ctx, opensandbox.ListOptions{
+		Metadata: map[string]string{terminalMetadata: "true"},
+		PageSize: 100,
+	})
+	if err != nil {
+		return 0, fmt.Errorf("list terminal sandboxes: %w", err)
+	}
+	cleaned := 0
+	for _, sandbox := range result.Items {
+		if sandbox.Metadata[assignmentMetadata] == "" {
+			continue
+		}
+		if err := b.Delete(ctx, executor(sandbox)); err != nil {
+			return cleaned, fmt.Errorf("delete terminal sandbox %s: %w", sandbox.ID, err)
+		}
+		cleaned++
+	}
+	return cleaned, nil
+}
+
+// Lifecycle periodically reconciles durable terminal markers into deletes.
+type LifecycleReconciler struct {
+	Backend  Backend
+	Interval time.Duration
+	OnError  func(error)
+}
+
+func (l LifecycleReconciler) Run(ctx context.Context) error {
+	interval := l.Interval
+	if interval <= 0 {
+		interval = 2 * time.Second
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		if _, err := l.Backend.CleanupTerminated(ctx); err != nil && ctx.Err() == nil && l.OnError != nil {
+			l.OnError(err)
+		}
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+		}
+	}
+}
+
 type Config struct {
 	Pool       string
 	Timeout    int

internal/opensandboxbackend/backend_test.go

@@ -33,6 +33,23 @@ func (f *fakeLifecycle) GetSandbox(_ context.Context, id string) (*opensandbox.S
 	}
 	return &opensandbox.SandboxInfo{ID: id, Metadata: map[string]string{"ci.ddupan.top/spiffe-id": assignment().Identity.SPIFFEID}}, nil
 }
+func (f *fakeLifecycle) PatchSandboxMetadata(_ context.Context, id string, patch opensandbox.MetadataPatch) (*opensandbox.SandboxInfo, error) {
+	for index := range f.items {
+		if f.items[index].ID != id {
+			continue
+		}
+		if f.items[index].Metadata == nil {
+			f.items[index].Metadata = map[string]string{}
+		}
+		for key, value := range patch {
+			if value != nil {
+				f.items[index].Metadata[key] = *value
+			}
+		}
+		return &f.items[index], nil
+	}
+	return &opensandbox.SandboxInfo{ID: id}, nil
+}
 func (f *fakeLifecycle) DeleteSandbox(_ context.Context, id string) error { f.deleted = id; return nil }
 
 func backend(lifecycle Lifecycle) Backend {
@@ -86,3 +103,21 @@ func TestBindIdentityVerifiesPersistedMetadata(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestTerminalMarkerDrivesDurableCleanup(t *testing.T) {
+	lifecycle := &fakeLifecycle{items: []opensandbox.SandboxInfo{{
+		ID:       "sandbox-42",
+		Metadata: map[string]string{assignmentMetadata: assignment().ID},
+	}}}
+	backend := backend(lifecycle)
+	if err := backend.MarkTerminal(context.Background(), assignment().ID); err != nil {
+		t.Fatal(err)
+	}
+	if lifecycle.items[0].Metadata[terminalMetadata] != "true" {
+		t.Fatalf("metadata = %#v", lifecycle.items[0].Metadata)
+	}
+	cleaned, err := backend.CleanupTerminated(context.Background())
+	if err != nil || cleaned != 1 || lifecycle.deleted != "sandbox-42" {
+		t.Fatalf("cleaned=%d deleted=%q err=%v", cleaned, lifecycle.deleted, err)
+	}
+}