feat: 持久化清理 VM 执行器
This commit is contained in:
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
opensandbox "github.com/alibaba/OpenSandbox/sdks/sandbox/go"
|
||||
|
||||
@@ -15,14 +16,86 @@ import (
|
||||
)
|
||||
|
||||
const assignmentMetadata = "ci.ddupan.top/assignment-id"
|
||||
const terminalMetadata = "ci.ddupan.top/terminal"
|
||||
|
||||
type Lifecycle interface {
|
||||
ListSandboxes(context.Context, opensandbox.ListOptions) (*opensandbox.ListSandboxesResponse, error)
|
||||
CreateSandbox(context.Context, opensandbox.CreateSandboxRequest) (*opensandbox.SandboxInfo, error)
|
||||
GetSandbox(context.Context, string) (*opensandbox.SandboxInfo, error)
|
||||
PatchSandboxMetadata(context.Context, string, opensandbox.MetadataPatch) (*opensandbox.SandboxInfo, error)
|
||||
DeleteSandbox(context.Context, string) error
|
||||
}
|
||||
|
||||
// MarkTerminal persists the accepted Gitea terminal state on the sandbox. The
|
||||
// lifecycle reconciler performs deletion separately so the runner receives the
|
||||
// successful UpdateTask response before its VM is stopped.
|
||||
func (b Backend) MarkTerminal(ctx context.Context, assignmentID string) error {
|
||||
executor, err := b.Find(ctx, assignmentID)
|
||||
if err != nil || executor == nil {
|
||||
return err
|
||||
}
|
||||
value := "true"
|
||||
_, err = b.Lifecycle.PatchSandboxMetadata(ctx, executor.Name, opensandbox.MetadataPatch{
|
||||
terminalMetadata: &value,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("mark sandbox %s terminal: %w", executor.Name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanupTerminated removes sandboxes whose terminal result was accepted by
|
||||
// Gitea. The marker is stored by OpenSandbox, so cleanup survives restarts.
|
||||
func (b Backend) CleanupTerminated(ctx context.Context) (int, error) {
|
||||
if err := b.validate(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
result, err := b.Lifecycle.ListSandboxes(ctx, opensandbox.ListOptions{
|
||||
Metadata: map[string]string{terminalMetadata: "true"},
|
||||
PageSize: 100,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("list terminal sandboxes: %w", err)
|
||||
}
|
||||
cleaned := 0
|
||||
for _, sandbox := range result.Items {
|
||||
if sandbox.Metadata[assignmentMetadata] == "" {
|
||||
continue
|
||||
}
|
||||
if err := b.Delete(ctx, executor(sandbox)); err != nil {
|
||||
return cleaned, fmt.Errorf("delete terminal sandbox %s: %w", sandbox.ID, err)
|
||||
}
|
||||
cleaned++
|
||||
}
|
||||
return cleaned, nil
|
||||
}
|
||||
|
||||
// Lifecycle periodically reconciles durable terminal markers into deletes.
|
||||
type LifecycleReconciler struct {
|
||||
Backend Backend
|
||||
Interval time.Duration
|
||||
OnError func(error)
|
||||
}
|
||||
|
||||
func (l LifecycleReconciler) Run(ctx context.Context) error {
|
||||
interval := l.Interval
|
||||
if interval <= 0 {
|
||||
interval = 2 * time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if _, err := l.Backend.CleanupTerminated(ctx); err != nil && ctx.Err() == nil && l.OnError != nil {
|
||||
l.OnError(err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Pool string
|
||||
Timeout int
|
||||
|
||||
@@ -33,6 +33,23 @@ func (f *fakeLifecycle) GetSandbox(_ context.Context, id string) (*opensandbox.S
|
||||
}
|
||||
return &opensandbox.SandboxInfo{ID: id, Metadata: map[string]string{"ci.ddupan.top/spiffe-id": assignment().Identity.SPIFFEID}}, nil
|
||||
}
|
||||
func (f *fakeLifecycle) PatchSandboxMetadata(_ context.Context, id string, patch opensandbox.MetadataPatch) (*opensandbox.SandboxInfo, error) {
|
||||
for index := range f.items {
|
||||
if f.items[index].ID != id {
|
||||
continue
|
||||
}
|
||||
if f.items[index].Metadata == nil {
|
||||
f.items[index].Metadata = map[string]string{}
|
||||
}
|
||||
for key, value := range patch {
|
||||
if value != nil {
|
||||
f.items[index].Metadata[key] = *value
|
||||
}
|
||||
}
|
||||
return &f.items[index], nil
|
||||
}
|
||||
return &opensandbox.SandboxInfo{ID: id}, nil
|
||||
}
|
||||
func (f *fakeLifecycle) DeleteSandbox(_ context.Context, id string) error { f.deleted = id; return nil }
|
||||
|
||||
func backend(lifecycle Lifecycle) Backend {
|
||||
@@ -86,3 +103,21 @@ func TestBindIdentityVerifiesPersistedMetadata(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTerminalMarkerDrivesDurableCleanup(t *testing.T) {
|
||||
lifecycle := &fakeLifecycle{items: []opensandbox.SandboxInfo{{
|
||||
ID: "sandbox-42",
|
||||
Metadata: map[string]string{assignmentMetadata: assignment().ID},
|
||||
}}}
|
||||
backend := backend(lifecycle)
|
||||
if err := backend.MarkTerminal(context.Background(), assignment().ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if lifecycle.items[0].Metadata[terminalMetadata] != "true" {
|
||||
t.Fatalf("metadata = %#v", lifecycle.items[0].Metadata)
|
||||
}
|
||||
cleaned, err := backend.CleanupTerminated(context.Background())
|
||||
if err != nil || cleaned != 1 || lifecycle.deleted != "sandbox-42" {
|
||||
t.Fatalf("cleaned=%d deleted=%q err=%v", cleaned, lifecycle.deleted, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user