feat: 自动清理终态 Pod 执行器

internal/podbackend/backend.go

@@ -13,7 +13,10 @@ import (
 	"git.ddupan.top/panxiao81/gitea-dynamic-runner/internal/taskworker"
 )
 
-const assignmentLabel = "ci.ddupan.top/assignment-id"
+const (
+	assignmentLabel = "ci.ddupan.top/assignment-id"
+	terminalLabel   = "ci.ddupan.top/terminal"
+)
 
 // Pod is the backend state required by the reconciler, not an in-memory lifecycle record.
 type Pod struct {
@@ -53,10 +56,49 @@ type API interface {
 	ListPods(context.Context, string, string) ([]Pod, error)
 	CreatePod(context.Context, PodManifest) (Pod, error)
 	DeletePod(context.Context, string, string) error
+	LabelPod(context.Context, string, string, map[string]string) error
 	EnsureIdentityEntry(context.Context, IdentityEntry) error
 	DeleteIdentityEntry(context.Context, string) error
 }
 
+// MarkTerminal persists Gitea's accepted terminal state on the backend
+// resource. Cleanup can therefore resume after a controller restart.
+func (b Backend) MarkTerminal(ctx context.Context, assignmentID string) error {
+	if err := b.validate(); err != nil {
+		return err
+	}
+	if assignmentID == "" {
+		return errors.New("assignment ID is required")
+	}
+	if err := b.API.LabelPod(ctx, b.Config.Namespace, assignmentID, map[string]string{terminalLabel: "true"}); err != nil {
+		return fmt.Errorf("mark assignment Pod terminal: %w", err)
+	}
+	return nil
+}
+
+// CleanupTerminated removes only executors whose terminal update was accepted
+// by Gitea and whose process has exited.
+func (b Backend) CleanupTerminated(ctx context.Context) (int, error) {
+	if err := b.validate(); err != nil {
+		return 0, err
+	}
+	pods, err := b.API.ListPods(ctx, b.Config.Namespace, terminalLabel+"=true")
+	if err != nil {
+		return 0, fmt.Errorf("list terminal assignment Pods: %w", err)
+	}
+	cleaned := 0
+	for _, pod := range pods {
+		if pod.Phase != "Succeeded" && pod.Phase != "Failed" {
+			continue
+		}
+		if err := b.Delete(ctx, executor(pod)); err != nil {
+			return cleaned, err
+		}
+		cleaned++
+	}
+	return cleaned, nil
+}
+
 type Config struct {
 	Namespace      string
 	Image          string

internal/podbackend/backend_test.go

@@ -18,6 +18,7 @@ type fakeAPI struct {
 	entry     IdentityEntry
 	entryGone string
 	deleted   string
+	marked    map[string]string
 }
 
 func (a *fakeAPI) ListPods(_ context.Context, _ string, selector string) ([]Pod, error) {
@@ -32,6 +33,10 @@ func (a *fakeAPI) DeletePod(_ context.Context, _, name string) error {
 	a.deleted = name
 	return nil
 }
+func (a *fakeAPI) LabelPod(_ context.Context, _, _ string, labels map[string]string) error {
+	a.marked = labels
+	return nil
+}
 func (a *fakeAPI) EnsureIdentityEntry(_ context.Context, entry IdentityEntry) error {
 	a.entry = entry
 	return nil
@@ -119,6 +124,27 @@ func TestDeleteRemovesIdentityBeforePod(t *testing.T) {
 	}
 }
 
+func TestTerminalMarkerAndCleanupUseBackendState(t *testing.T) {
+	api := &fakeAPI{pods: []Pod{
+		{Name: "running", UID: "running-uid", Phase: "Running"},
+		{Name: "finished", UID: "finished-uid", Phase: "Succeeded"},
+	}}
+	backend := backend(api)
+	if err := backend.MarkTerminal(context.Background(), "gitea-task-42"); err != nil {
+		t.Fatal(err)
+	}
+	if api.marked[terminalLabel] != "true" {
+		t.Fatalf("labels = %#v", api.marked)
+	}
+	cleaned, err := backend.CleanupTerminated(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if api.selector != terminalLabel+"=true" || cleaned != 1 || api.deleted != "finished" || api.entryGone != "finished" {
+		t.Fatalf("selector=%q cleaned=%d deleted=%q entry=%q", api.selector, cleaned, api.deleted, api.entryGone)
+	}
+}
+
 func TestFindRejectsDuplicatePods(t *testing.T) {
 	api := &fakeAPI{pods: []Pod{{Name: "one"}, {Name: "two"}}}
 	if _, err := backend(api).Find(context.Background(), "gitea-task-42"); err == nil {

internal/podbackend/client.go

@@ -2,6 +2,7 @@ package podbackend
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"reflect"
 
@@ -10,6 +11,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -102,6 +104,15 @@ func (c *Client) DeletePod(ctx context.Context, namespace, name string) error {
 	return err
 }
 
+func (c *Client) LabelPod(ctx context.Context, namespace, name string, labels map[string]string) error {
+	patch, err := json.Marshal(map[string]any{"metadata": map[string]any{"labels": labels}})
+	if err != nil {
+		return err
+	}
+	_, err = c.Kubernetes.CoreV1().Pods(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{})
+	return err
+}
+
 func (c *Client) EnsureIdentityEntry(ctx context.Context, entry IdentityEntry) error {
 	resource := c.Dynamic.Resource(identityEntryResource)
 	existing, err := resource.Get(ctx, entry.Name, metav1.GetOptions{})

internal/podbackend/client_test.go

@@ -37,6 +37,13 @@ func TestClientPodLifecycleUsesTypedClient(t *testing.T) {
 	if got := pod.Spec.Containers[0].Env; len(got) != 1 || got[0].Name != "CI_RUNNER_CAPABILITY" || got[0].Value != "capability" {
 		t.Fatalf("environment = %#v", got)
 	}
+	if err := client.LabelPod(context.Background(), "gitea-actions", created.Name, map[string]string{terminalLabel: "true"}); err != nil {
+		t.Fatal(err)
+	}
+	pod, err = client.Kubernetes.CoreV1().Pods("gitea-actions").Get(context.Background(), created.Name, metav1.GetOptions{})
+	if err != nil || pod.Labels[terminalLabel] != "true" {
+		t.Fatalf("terminal label pod=%#v err=%v", pod, err)
+	}
 	pod.UID = types.UID("pod-uid")
 	pod.Status.Phase = corev1.PodRunning
 	if _, err := client.Kubernetes.CoreV1().Pods("gitea-actions").Update(context.Background(), pod, metav1.UpdateOptions{}); err != nil {

internal/podbackend/lifecycle.go

@@ -0,0 +1,32 @@
+package podbackend
+
+import (
+	"context"
+	"time"
+)
+
+// Lifecycle reconciles durable terminal markers into backend cleanup.
+type Lifecycle struct {
+	Backend  Backend
+	Interval time.Duration
+	OnError  func(error)
+}
+
+func (l Lifecycle) Run(ctx context.Context) error {
+	interval := l.Interval
+	if interval <= 0 {
+		interval = 2 * time.Second
+	}
+	for {
+		if _, err := l.Backend.CleanupTerminated(ctx); err != nil && ctx.Err() == nil && l.OnError != nil {
+			l.OnError(err)
+		}
+		timer := time.NewTimer(interval)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return nil
+		case <-timer.C:
+		}
+	}
+}

internal/runnerfacade/facade.go

@@ -57,7 +57,7 @@ type Facade struct {
 	Registry     *Registry
 	Capabilities Capabilities
 	Upstream     Upstream
-	OnTerminal   func(taskassignment.Assignment)
+	OnTerminal   func(context.Context, taskassignment.Assignment) error
 }
 
 func (f *Facade) Handler() (string, http.Handler) {
@@ -101,7 +101,9 @@ func (f *Facade) UpdateTask(ctx context.Context, request *connect.Request[runner
 	}
 	response, err := f.Upstream.UpdateTask(ctx, connect.NewRequest(request.Msg))
 	if err == nil && request.Msg.GetState().GetResult() != runnerv1.Result_RESULT_UNSPECIFIED && f.OnTerminal != nil {
-		f.OnTerminal(assignment)
+		if terminalErr := f.OnTerminal(ctx, assignment); terminalErr != nil {
+			return nil, connect.NewError(connect.CodeUnavailable, terminalErr)
+		}
 	}
 	return response, err
 }

internal/runnerfacade/facade_test.go

@@ -170,11 +170,12 @@ func TestFacadeSignalsTerminalTaskAfterUpstreamAcceptsIt(t *testing.T) {
 		t.Fatal(err)
 	}
 	completed := 0
-	facade.OnTerminal = func(got taskassignment.Assignment) {
+	facade.OnTerminal = func(_ context.Context, got taskassignment.Assignment) error {
 		if got.ID != assignment.ID {
 			t.Fatalf("terminal assignment = %s", got.ID)
 		}
 		completed++
+		return nil
 	}
 	if _, err := facade.UpdateTask(ctx, authenticatedRequest(&runnerv1.UpdateTaskRequest{
 		State: &runnerv1.TaskState{Id: 42, Result: runnerv1.Result_RESULT_SUCCESS},