fix: 从后端恢复 Runner claim
This commit is contained in:
@@ -158,6 +158,15 @@ func runController(ctx context.Context) error {
|
|||||||
SPIREAgentID: config.SPIREAgentID, ExecutorUID: config.PodExecutorUID,
|
SPIREAgentID: config.SPIREAgentID, ExecutorUID: config.PodExecutorUID,
|
||||||
}}
|
}}
|
||||||
podExecutorBackend = &backend
|
podExecutorBackend = &backend
|
||||||
|
assignments, err := backend.RecoverAssignments(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, assignment := range assignments {
|
||||||
|
if err := registry.RecoverClaimed(assignment); err != nil {
|
||||||
|
return fmt.Errorf("recover Pod facade claim %s: %w", assignment.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
component, err := workerComponent(ctx, workerJS, config, taskassignment.BackendPod, config.PodCapacity, taskworker.Worker{Backend: backend, Bootstrap: bootstrap}, registry)
|
component, err := workerComponent(ctx, workerJS, config, taskassignment.BackendPod, config.PodCapacity, taskworker.Worker{Backend: backend, Bootstrap: bootstrap}, registry)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -178,6 +187,15 @@ func runController(ctx context.Context) error {
|
|||||||
Env: map[string]string{"SPIFFE_ENDPOINT_SOCKET": config.WorkloadAPIAddr},
|
Env: map[string]string{"SPIFFE_ENDPOINT_SOCKET": config.WorkloadAPIAddr},
|
||||||
}}
|
}}
|
||||||
vmExecutorBackend = &backend
|
vmExecutorBackend = &backend
|
||||||
|
assignments, err := backend.RecoverAssignments(ctx, config.TrustDomain)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, assignment := range assignments {
|
||||||
|
if err := registry.RecoverClaimed(assignment); err != nil {
|
||||||
|
return fmt.Errorf("recover VM facade claim %s: %w", assignment.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
component, err := workerComponent(ctx, workerJS, config, taskassignment.BackendVM, config.VMCapacity, taskworker.Worker{Backend: backend, Bootstrap: bootstrap}, registry)
|
component, err := workerComponent(ctx, workerJS, config, taskassignment.BackendVM, config.VMCapacity, taskworker.Worker{Backend: backend, Bootstrap: bootstrap}, registry)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -108,6 +108,30 @@ type Backend struct {
|
|||||||
Config Config
|
Config Config
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b Backend) RecoverAssignments(ctx context.Context, trustDomain string) ([]taskassignment.Assignment, error) {
|
||||||
|
if err := b.validate(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
result, err := b.Lifecycle.ListSandboxes(ctx, opensandbox.ListOptions{
|
||||||
|
Metadata: map[string]string{"ci.ddupan.top/backend": "vm"}, PageSize: 100,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("list recoverable sandboxes: %w", err)
|
||||||
|
}
|
||||||
|
assignments := make([]taskassignment.Assignment, 0, len(result.Items))
|
||||||
|
for _, sandbox := range result.Items {
|
||||||
|
if sandbox.Metadata[terminalMetadata] == "true" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
assignment, err := taskassignment.FromMetadata(sandbox.Metadata, sandbox.Metadata, trustDomain)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("recover sandbox %s: %w", sandbox.ID, err)
|
||||||
|
}
|
||||||
|
assignments = append(assignments, assignment)
|
||||||
|
}
|
||||||
|
return assignments, nil
|
||||||
|
}
|
||||||
|
|
||||||
func NewLifecycleClient(baseURL, apiKey string, client *http.Client) *opensandbox.LifecycleClient {
|
func NewLifecycleClient(baseURL, apiKey string, client *http.Client) *opensandbox.LifecycleClient {
|
||||||
if client != nil {
|
if client != nil {
|
||||||
return opensandbox.NewLifecycleClient(baseURL, apiKey, opensandbox.WithHTTPClient(client))
|
return opensandbox.NewLifecycleClient(baseURL, apiKey, opensandbox.WithHTTPClient(client))
|
||||||
|
|||||||
@@ -116,6 +116,28 @@ type Backend struct {
|
|||||||
Config Config
|
Config Config
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b Backend) RecoverAssignments(ctx context.Context) ([]taskassignment.Assignment, error) {
|
||||||
|
if err := b.validate(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
pods, err := b.API.ListPods(ctx, b.Config.Namespace, "ci.ddupan.top/backend=pod")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("list recoverable assignment Pods: %w", err)
|
||||||
|
}
|
||||||
|
assignments := make([]taskassignment.Assignment, 0, len(pods))
|
||||||
|
for _, pod := range pods {
|
||||||
|
if pod.Labels[terminalLabel] == "true" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
assignment, err := taskassignment.FromMetadata(pod.Labels, pod.Annotations, b.Config.TrustDomain)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("recover Pod %s: %w", pod.Name, err)
|
||||||
|
}
|
||||||
|
assignments = append(assignments, assignment)
|
||||||
|
}
|
||||||
|
return assignments, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (b Backend) Find(ctx context.Context, assignmentID string) (*taskworker.Executor, error) {
|
func (b Backend) Find(ctx context.Context, assignmentID string) (*taskworker.Executor, error) {
|
||||||
if err := b.validate(); err != nil {
|
if err := b.validate(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -225,3 +225,15 @@ func TestCapabilitiesAreDeterministicAndAssignmentScoped(t *testing.T) {
|
|||||||
t.Fatal("capability scope is invalid")
|
t.Fatal("capability scope is invalid")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRegistryRecoversClaimedAssignment(t *testing.T) {
|
||||||
|
registry := NewRegistry()
|
||||||
|
assignment := facadeAssignment(t)
|
||||||
|
if err := registry.RecoverClaimed(assignment); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resolved, err := registry.Resolve(assignment.ID, assignment.Identity.SPIFFEID)
|
||||||
|
if err != nil || resolved.Task.GetId() != assignment.Task.GetId() {
|
||||||
|
t.Fatalf("resolved=%#v err=%v", resolved, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -90,3 +90,21 @@ func (r *Registry) Remove(assignmentID string) {
|
|||||||
defer r.mu.Unlock()
|
defer r.mu.Unlock()
|
||||||
delete(r.claims, assignmentID)
|
delete(r.claims, assignmentID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RecoverClaimed restores authorization for an executor that already claimed
|
||||||
|
// its task before the controller restarted.
|
||||||
|
func (r *Registry) RecoverClaimed(assignment taskassignment.Assignment) error {
|
||||||
|
ready, err := r.Offer(assignment)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
r.mu.Lock()
|
||||||
|
entry := r.claims[assignment.ID]
|
||||||
|
if !entry.claimed {
|
||||||
|
entry.claimed = true
|
||||||
|
close(entry.ready)
|
||||||
|
}
|
||||||
|
r.mu.Unlock()
|
||||||
|
<-ready
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
"gitea.dev/actionslib/pkg/model"
|
"gitea.dev/actionslib/pkg/model"
|
||||||
runnerv1 "gitea.dev/actionslib/runner/v1"
|
runnerv1 "gitea.dev/actionslib/runner/v1"
|
||||||
@@ -32,6 +33,32 @@ type Assignment struct {
|
|||||||
Identity taskidentity.Identity
|
Identity taskidentity.Identity
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FromMetadata reconstructs the minimal assignment needed to authorize an
|
||||||
|
// already-running executor after a controller restart. Backend metadata was
|
||||||
|
// originally derived from the trusted Gitea task and is validated again here.
|
||||||
|
func FromMetadata(labels, annotations map[string]string, trustDomain string) (Assignment, error) {
|
||||||
|
taskID, err := strconv.ParseInt(labels["ci.ddupan.top/task-id"], 10, 64)
|
||||||
|
if err != nil || taskID < 1 {
|
||||||
|
return Assignment{}, errors.New("backend metadata has invalid task ID")
|
||||||
|
}
|
||||||
|
backend := Backend(labels["ci.ddupan.top/backend"])
|
||||||
|
if backend != BackendPod && backend != BackendVM {
|
||||||
|
return Assignment{}, errors.New("backend metadata has invalid backend")
|
||||||
|
}
|
||||||
|
id := labels["ci.ddupan.top/assignment-id"]
|
||||||
|
if id != fmt.Sprintf("gitea-task-%d", taskID) {
|
||||||
|
return Assignment{}, errors.New("backend metadata assignment ID does not match task ID")
|
||||||
|
}
|
||||||
|
identity, err := taskidentity.FromMetadata(
|
||||||
|
annotations["ci.ddupan.top/repository"], annotations["ci.ddupan.top/job-key"],
|
||||||
|
annotations["ci.ddupan.top/spiffe-id"], trustDomain,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return Assignment{}, err
|
||||||
|
}
|
||||||
|
return Assignment{ID: id, Backend: backend, Task: &runnerv1.Task{Id: taskID}, Identity: identity}, nil
|
||||||
|
}
|
||||||
|
|
||||||
type envelope struct {
|
type envelope struct {
|
||||||
Version int `json:"version"`
|
Version int `json:"version"`
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
|
|||||||
@@ -73,3 +73,18 @@ func TestAssignmentWireRoundTripAndValidation(t *testing.T) {
|
|||||||
t.Fatal("expected tampered backend to fail")
|
t.Fatal("expected tampered backend to fail")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFromMetadataRecoversMinimalAssignment(t *testing.T) {
|
||||||
|
assignment, err := FromMetadata(map[string]string{
|
||||||
|
"ci.ddupan.top/assignment-id": "gitea-task-42",
|
||||||
|
"ci.ddupan.top/task-id": "42",
|
||||||
|
"ci.ddupan.top/backend": "vm",
|
||||||
|
}, map[string]string{
|
||||||
|
"ci.ddupan.top/repository": "owner/repo",
|
||||||
|
"ci.ddupan.top/job-key": "publish",
|
||||||
|
"ci.ddupan.top/spiffe-id": "spiffe://ddupan.top/ci/owner/repo/publish",
|
||||||
|
}, "ddupan.top")
|
||||||
|
if err != nil || assignment.Task.GetId() != 42 || assignment.Backend != BackendVM {
|
||||||
|
t.Fatalf("assignment=%#v err=%v", assignment, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -23,6 +23,21 @@ type Identity struct {
|
|||||||
SPIFFEID string
|
SPIFFEID string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FromMetadata validates identity fields recovered from backend-owned state.
|
||||||
|
func FromMetadata(repository, task, spiffeID, trustDomain string) (Identity, error) {
|
||||||
|
parts := strings.Split(repository, "/")
|
||||||
|
if len(parts) != 2 || parts[0] == "" || parts[1] == "" || !safeTaskKey.MatchString(task) {
|
||||||
|
return Identity{}, errors.New("invalid recovered repository or task identity")
|
||||||
|
}
|
||||||
|
expected := "spiffe://" + trustDomain + "/ci/" + strings.Join([]string{
|
||||||
|
sanitize(parts[0]), sanitize(parts[1]), task,
|
||||||
|
}, "/")
|
||||||
|
if spiffeID != expected {
|
||||||
|
return Identity{}, fmt.Errorf("recovered SPIFFE ID %q does not match %q", spiffeID, expected)
|
||||||
|
}
|
||||||
|
return Identity{Repository: repository, Task: task, SPIFFEID: spiffeID}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// FromTask derives the repository/task SPIFFE ID from Gitea's trusted task
|
// FromTask derives the repository/task SPIFFE ID from Gitea's trusted task
|
||||||
// context. Workflow input never supplies or overrides the resulting ID.
|
// context. Workflow input never supplies or overrides the resulting ID.
|
||||||
func FromTask(task *runnerv1.Task, trustDomain string) (Identity, error) {
|
func FromTask(task *runnerv1.Task, trustDomain string) (Identity, error) {
|
||||||
|
|||||||
Reference in New Issue
Block a user