fix: 从后端恢复 Runner claim
This commit is contained in:
@@ -158,6 +158,15 @@ func runController(ctx context.Context) error {
|
||||
SPIREAgentID: config.SPIREAgentID, ExecutorUID: config.PodExecutorUID,
|
||||
}}
|
||||
podExecutorBackend = &backend
|
||||
assignments, err := backend.RecoverAssignments(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, assignment := range assignments {
|
||||
if err := registry.RecoverClaimed(assignment); err != nil {
|
||||
return fmt.Errorf("recover Pod facade claim %s: %w", assignment.ID, err)
|
||||
}
|
||||
}
|
||||
component, err := workerComponent(ctx, workerJS, config, taskassignment.BackendPod, config.PodCapacity, taskworker.Worker{Backend: backend, Bootstrap: bootstrap}, registry)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -178,6 +187,15 @@ func runController(ctx context.Context) error {
|
||||
Env: map[string]string{"SPIFFE_ENDPOINT_SOCKET": config.WorkloadAPIAddr},
|
||||
}}
|
||||
vmExecutorBackend = &backend
|
||||
assignments, err := backend.RecoverAssignments(ctx, config.TrustDomain)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, assignment := range assignments {
|
||||
if err := registry.RecoverClaimed(assignment); err != nil {
|
||||
return fmt.Errorf("recover VM facade claim %s: %w", assignment.ID, err)
|
||||
}
|
||||
}
|
||||
component, err := workerComponent(ctx, workerJS, config, taskassignment.BackendVM, config.VMCapacity, taskworker.Worker{Backend: backend, Bootstrap: bootstrap}, registry)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -108,6 +108,30 @@ type Backend struct {
|
||||
Config Config
|
||||
}
|
||||
|
||||
func (b Backend) RecoverAssignments(ctx context.Context, trustDomain string) ([]taskassignment.Assignment, error) {
|
||||
if err := b.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, err := b.Lifecycle.ListSandboxes(ctx, opensandbox.ListOptions{
|
||||
Metadata: map[string]string{"ci.ddupan.top/backend": "vm"}, PageSize: 100,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list recoverable sandboxes: %w", err)
|
||||
}
|
||||
assignments := make([]taskassignment.Assignment, 0, len(result.Items))
|
||||
for _, sandbox := range result.Items {
|
||||
if sandbox.Metadata[terminalMetadata] == "true" {
|
||||
continue
|
||||
}
|
||||
assignment, err := taskassignment.FromMetadata(sandbox.Metadata, sandbox.Metadata, trustDomain)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("recover sandbox %s: %w", sandbox.ID, err)
|
||||
}
|
||||
assignments = append(assignments, assignment)
|
||||
}
|
||||
return assignments, nil
|
||||
}
|
||||
|
||||
func NewLifecycleClient(baseURL, apiKey string, client *http.Client) *opensandbox.LifecycleClient {
|
||||
if client != nil {
|
||||
return opensandbox.NewLifecycleClient(baseURL, apiKey, opensandbox.WithHTTPClient(client))
|
||||
|
||||
@@ -116,6 +116,28 @@ type Backend struct {
|
||||
Config Config
|
||||
}
|
||||
|
||||
func (b Backend) RecoverAssignments(ctx context.Context) ([]taskassignment.Assignment, error) {
|
||||
if err := b.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pods, err := b.API.ListPods(ctx, b.Config.Namespace, "ci.ddupan.top/backend=pod")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list recoverable assignment Pods: %w", err)
|
||||
}
|
||||
assignments := make([]taskassignment.Assignment, 0, len(pods))
|
||||
for _, pod := range pods {
|
||||
if pod.Labels[terminalLabel] == "true" {
|
||||
continue
|
||||
}
|
||||
assignment, err := taskassignment.FromMetadata(pod.Labels, pod.Annotations, b.Config.TrustDomain)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("recover Pod %s: %w", pod.Name, err)
|
||||
}
|
||||
assignments = append(assignments, assignment)
|
||||
}
|
||||
return assignments, nil
|
||||
}
|
||||
|
||||
func (b Backend) Find(ctx context.Context, assignmentID string) (*taskworker.Executor, error) {
|
||||
if err := b.validate(); err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -225,3 +225,15 @@ func TestCapabilitiesAreDeterministicAndAssignmentScoped(t *testing.T) {
|
||||
t.Fatal("capability scope is invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryRecoversClaimedAssignment(t *testing.T) {
|
||||
registry := NewRegistry()
|
||||
assignment := facadeAssignment(t)
|
||||
if err := registry.RecoverClaimed(assignment); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resolved, err := registry.Resolve(assignment.ID, assignment.Identity.SPIFFEID)
|
||||
if err != nil || resolved.Task.GetId() != assignment.Task.GetId() {
|
||||
t.Fatalf("resolved=%#v err=%v", resolved, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,3 +90,21 @@ func (r *Registry) Remove(assignmentID string) {
|
||||
defer r.mu.Unlock()
|
||||
delete(r.claims, assignmentID)
|
||||
}
|
||||
|
||||
// RecoverClaimed restores authorization for an executor that already claimed
|
||||
// its task before the controller restarted.
|
||||
func (r *Registry) RecoverClaimed(assignment taskassignment.Assignment) error {
|
||||
ready, err := r.Offer(assignment)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.mu.Lock()
|
||||
entry := r.claims[assignment.ID]
|
||||
if !entry.claimed {
|
||||
entry.claimed = true
|
||||
close(entry.ready)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
<-ready
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
runnerv1 "gitea.dev/actionslib/runner/v1"
|
||||
@@ -32,6 +33,32 @@ type Assignment struct {
|
||||
Identity taskidentity.Identity
|
||||
}
|
||||
|
||||
// FromMetadata reconstructs the minimal assignment needed to authorize an
|
||||
// already-running executor after a controller restart. Backend metadata was
|
||||
// originally derived from the trusted Gitea task and is validated again here.
|
||||
func FromMetadata(labels, annotations map[string]string, trustDomain string) (Assignment, error) {
|
||||
taskID, err := strconv.ParseInt(labels["ci.ddupan.top/task-id"], 10, 64)
|
||||
if err != nil || taskID < 1 {
|
||||
return Assignment{}, errors.New("backend metadata has invalid task ID")
|
||||
}
|
||||
backend := Backend(labels["ci.ddupan.top/backend"])
|
||||
if backend != BackendPod && backend != BackendVM {
|
||||
return Assignment{}, errors.New("backend metadata has invalid backend")
|
||||
}
|
||||
id := labels["ci.ddupan.top/assignment-id"]
|
||||
if id != fmt.Sprintf("gitea-task-%d", taskID) {
|
||||
return Assignment{}, errors.New("backend metadata assignment ID does not match task ID")
|
||||
}
|
||||
identity, err := taskidentity.FromMetadata(
|
||||
annotations["ci.ddupan.top/repository"], annotations["ci.ddupan.top/job-key"],
|
||||
annotations["ci.ddupan.top/spiffe-id"], trustDomain,
|
||||
)
|
||||
if err != nil {
|
||||
return Assignment{}, err
|
||||
}
|
||||
return Assignment{ID: id, Backend: backend, Task: &runnerv1.Task{Id: taskID}, Identity: identity}, nil
|
||||
}
|
||||
|
||||
type envelope struct {
|
||||
Version int `json:"version"`
|
||||
ID string `json:"id"`
|
||||
|
||||
@@ -73,3 +73,18 @@ func TestAssignmentWireRoundTripAndValidation(t *testing.T) {
|
||||
t.Fatal("expected tampered backend to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromMetadataRecoversMinimalAssignment(t *testing.T) {
|
||||
assignment, err := FromMetadata(map[string]string{
|
||||
"ci.ddupan.top/assignment-id": "gitea-task-42",
|
||||
"ci.ddupan.top/task-id": "42",
|
||||
"ci.ddupan.top/backend": "vm",
|
||||
}, map[string]string{
|
||||
"ci.ddupan.top/repository": "owner/repo",
|
||||
"ci.ddupan.top/job-key": "publish",
|
||||
"ci.ddupan.top/spiffe-id": "spiffe://ddupan.top/ci/owner/repo/publish",
|
||||
}, "ddupan.top")
|
||||
if err != nil || assignment.Task.GetId() != 42 || assignment.Backend != BackendVM {
|
||||
t.Fatalf("assignment=%#v err=%v", assignment, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,21 @@ type Identity struct {
|
||||
SPIFFEID string
|
||||
}
|
||||
|
||||
// FromMetadata validates identity fields recovered from backend-owned state.
|
||||
func FromMetadata(repository, task, spiffeID, trustDomain string) (Identity, error) {
|
||||
parts := strings.Split(repository, "/")
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" || !safeTaskKey.MatchString(task) {
|
||||
return Identity{}, errors.New("invalid recovered repository or task identity")
|
||||
}
|
||||
expected := "spiffe://" + trustDomain + "/ci/" + strings.Join([]string{
|
||||
sanitize(parts[0]), sanitize(parts[1]), task,
|
||||
}, "/")
|
||||
if spiffeID != expected {
|
||||
return Identity{}, fmt.Errorf("recovered SPIFFE ID %q does not match %q", spiffeID, expected)
|
||||
}
|
||||
return Identity{Repository: repository, Task: task, SPIFFEID: spiffeID}, nil
|
||||
}
|
||||
|
||||
// FromTask derives the repository/task SPIFFE ID from Gitea's trusted task
|
||||
// context. Workflow input never supplies or overrides the resulting ID.
|
||||
func FromTask(task *runnerv1.Task, trustDomain string) (Identity, error) {
|
||||
|
||||
Reference in New Issue
Block a user