refactor: separate workload class from placement driver
test / shell (pull_request) Successful in 28s
test / python (pull_request) Successful in 59s
test / go (pull_request) Successful in 3m35s

This commit is contained in:
2026-09-25 16:48:32 +00:00
parent 162f742880
commit 897d327a26
21 changed files with 305 additions and 205 deletions
+12 -11
View File
@@ -19,7 +19,7 @@ type publishAPI interface {
PublishMsg(context.Context, *nats.Msg, ...jetstream.PublishOpt) (*jetstream.PubAck, error)
}
// Publisher implements the scheduler dispatcher with one subject per backend.
// Publisher implements the scheduler dispatcher with one subject per placement.
type Publisher struct {
JetStream publishAPI
SubjectBase string
@@ -38,7 +38,7 @@ func (p Publisher) Dispatch(ctx context.Context, assignment taskassignment.Assig
return errors.New("assignment subject base is required")
}
message := &nats.Msg{
Subject: base + "." + string(assignment.Backend),
Subject: base + "." + assignment.Placement.Key(),
Header: nats.Header{jetstream.MsgIDHeader: []string{assignment.ID}},
Data: body,
}
@@ -86,13 +86,13 @@ type Processor struct {
type Event struct {
Name string
AssignmentID string
Backend taskassignment.Backend
Placement taskassignment.Placement
RetryDelay time.Duration
}
func (p Processor) event(name string, assignment taskassignment.Assignment, retryDelay time.Duration) {
if p.OnEvent != nil {
p.OnEvent(Event{Name: name, AssignmentID: assignment.ID, Backend: assignment.Backend, RetryDelay: retryDelay})
p.OnEvent(Event{Name: name, AssignmentID: assignment.ID, Placement: assignment.Placement, RetryDelay: retryDelay})
}
}
@@ -178,24 +178,25 @@ type consumerManager interface {
// OpenConsumer creates the durable backend cursor. Capacity is enforced both
// server-side and by ConsumerComponent's local semaphore.
func OpenConsumer(ctx context.Context, manager consumerManager, stream, subjectBase string, backend taskassignment.Backend, capacity int) (jetstream.Consumer, error) {
func OpenConsumer(ctx context.Context, manager consumerManager, stream, subjectBase string, placement taskassignment.Placement, capacity int) (jetstream.Consumer, error) {
if manager == nil || stream == "" || strings.TrimSuffix(subjectBase, ".") == "" || capacity < 1 {
return nil, errors.New("JetStream manager, stream, subject base, and positive capacity are required")
}
if backend != taskassignment.BackendPod && backend != taskassignment.BackendVM {
return nil, fmt.Errorf("unsupported assignment backend %q", backend)
if err := placement.Validate(); err != nil {
return nil, err
}
key := placement.Key()
consumer, err := manager.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{
Name: string(backend),
Durable: string(backend),
FilterSubject: strings.TrimSuffix(subjectBase, ".") + "." + string(backend),
Name: key,
Durable: key,
FilterSubject: strings.TrimSuffix(subjectBase, ".") + "." + key,
AckPolicy: jetstream.AckExplicitPolicy,
AckWait: 5 * time.Minute,
MaxAckPending: capacity,
MaxDeliver: 1000,
})
if err != nil {
return nil, fmt.Errorf("open %s assignment consumer: %w", backend, err)
return nil, fmt.Errorf("open %s assignment consumer: %w", key, err)
}
return consumer, nil
}
+6 -6
View File
@@ -38,13 +38,13 @@ func (p *fakePublisher) PublishMsg(_ context.Context, message *nats.Msg, _ ...je
return &jetstream.PubAck{}, nil
}
func TestPublisherUsesBackendSubjectAndAssignmentDeduplication(t *testing.T) {
func TestPublisherUsesPlacementSubjectAndAssignmentDeduplication(t *testing.T) {
api := &fakePublisher{}
publisher := Publisher{JetStream: api, SubjectBase: "ci.assignment"}
if err := publisher.Dispatch(context.Background(), testAssignment(t)); err != nil {
t.Fatal(err)
}
if api.message.Subject != "ci.assignment.pod" {
if api.message.Subject != "ci.assignment.container.kubernetes" {
t.Fatalf("subject = %q", api.message.Subject)
}
if api.message.Header.Get(jetstream.MsgIDHeader) != "gitea-task-42" {
@@ -139,7 +139,7 @@ func TestProcessorAcknowledgesPersistedHandoff(t *testing.T) {
t.Fatalf("events = %#v", events)
}
for index := range want {
if events[index].Name != want[index] || events[index].AssignmentID != "gitea-task-42" || events[index].Backend != taskassignment.BackendPod {
if events[index].Name != want[index] || events[index].AssignmentID != "gitea-task-42" || events[index].Placement != taskassignment.KubernetesContainer {
t.Fatalf("event[%d] = %#v", index, events[index])
}
}
@@ -210,12 +210,12 @@ func (m *fakeConsumerManager) CreateOrUpdateConsumer(_ context.Context, _ string
return nil, nil
}
func TestOpenConsumerUsesIndependentDurablePerBackend(t *testing.T) {
func TestOpenConsumerUsesIndependentDurablePerPlacement(t *testing.T) {
manager := &fakeConsumerManager{}
if _, err := OpenConsumer(context.Background(), manager, "CI_RUNNER", "ci.assignment", taskassignment.BackendPod, 4); err != nil {
if _, err := OpenConsumer(context.Background(), manager, "CI_RUNNER", "ci.assignment", taskassignment.KubernetesContainer, 4); err != nil {
t.Fatal(err)
}
if manager.config.Durable != "pod" || manager.config.FilterSubject != "ci.assignment.pod" || manager.config.AckPolicy != jetstream.AckExplicitPolicy || manager.config.MaxAckPending != 4 || manager.config.MaxDeliver != 1000 {
if manager.config.Durable != "container.kubernetes" || manager.config.FilterSubject != "ci.assignment.container.kubernetes" || manager.config.AckPolicy != jetstream.AckExplicitPolicy || manager.config.MaxAckPending != 4 || manager.config.MaxDeliver != 1000 {
t.Fatalf("config = %#v", manager.config)
}
}
+10 -4
View File
@@ -14,12 +14,12 @@ import (
type ComponentName string
const (
Scheduler ComponentName = "scheduler"
PodWorker ComponentName = "pod-worker"
VMWorker ComponentName = "vm-worker"
Scheduler ComponentName = "scheduler"
KubernetesWorker ComponentName = "kubernetes-worker"
OpenSandboxWorker ComponentName = "opensandbox-worker"
)
var defaultComponents = []ComponentName{Scheduler, PodWorker, VMWorker}
var defaultComponents = []ComponentName{Scheduler, KubernetesWorker, OpenSandboxWorker}
// Selection parses --components. An empty value enables all components.
type Selection []ComponentName
@@ -31,6 +31,12 @@ func ParseSelection(value string) (Selection, error) {
var selected Selection
for _, raw := range strings.Split(value, ",") {
name := ComponentName(strings.TrimSpace(raw))
switch name {
case "pod-worker":
name = KubernetesWorker
case "vm-worker":
name = OpenSandboxWorker
}
if !slices.Contains(defaultComponents, name) {
return nil, fmt.Errorf("unknown controller component %q", name)
}
+16 -6
View File
@@ -13,18 +13,18 @@ func TestParseSelectionDefaultsToAll(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(selection) != 3 || selection[0] != Scheduler || selection[1] != PodWorker || selection[2] != VMWorker {
if len(selection) != 3 || selection[0] != Scheduler || selection[1] != KubernetesWorker || selection[2] != OpenSandboxWorker {
t.Fatalf("selection = %v", selection)
}
}
}
func TestParseSelectionAllowsOneOrMoreComponents(t *testing.T) {
selection, err := ParseSelection("vm-worker,scheduler,vm-worker")
selection, err := ParseSelection("opensandbox-worker,scheduler,opensandbox-worker")
if err != nil {
t.Fatal(err)
}
if len(selection) != 2 || selection[0] != VMWorker || selection[1] != Scheduler {
if len(selection) != 2 || selection[0] != OpenSandboxWorker || selection[1] != Scheduler {
t.Fatalf("selection = %v", selection)
}
if _, err := ParseSelection("webhook"); err == nil {
@@ -32,6 +32,16 @@ func TestParseSelectionAllowsOneOrMoreComponents(t *testing.T) {
}
}
func TestParseSelectionNormalizesLegacyWorkerNames(t *testing.T) {
selection, err := ParseSelection("pod-worker,vm-worker")
if err != nil {
t.Fatal(err)
}
if len(selection) != 2 || selection[0] != KubernetesWorker || selection[1] != OpenSandboxWorker {
t.Fatalf("selection = %v", selection)
}
}
type componentFunc func(context.Context) error
func (f componentFunc) Run(ctx context.Context) error { return f(ctx) }
@@ -45,14 +55,14 @@ func TestRunStartsSelectedComponentsAndCancelsPeers(t *testing.T) {
started <- Scheduler
return errors.New("poll failed")
}),
PodWorker: componentFunc(func(ctx context.Context) error {
started <- PodWorker
KubernetesWorker: componentFunc(func(ctx context.Context) error {
started <- KubernetesWorker
<-ctx.Done()
once.Do(func() { close(peerStopped) })
return ctx.Err()
}),
}
err := Run(context.Background(), Selection{Scheduler, PodWorker}, registry)
err := Run(context.Background(), Selection{Scheduler, KubernetesWorker}, registry)
if err == nil || !errors.Is(err, context.Canceled) && err.Error() != "component scheduler: poll failed" {
t.Fatalf("Run() error = %v", err)
}
+6 -3
View File
@@ -119,7 +119,10 @@ func (b Backend) RecoverAssignments(ctx context.Context, trustDomain string) ([]
return nil, err
}
result, err := b.Lifecycle.ListSandboxes(ctx, opensandbox.ListOptions{
Metadata: map[string]string{"ci.ddupan.top/backend": "vm"}, PageSize: 100,
Metadata: map[string]string{
"ci.ddupan.top/workload-class": "vm",
"ci.ddupan.top/driver": "opensandbox",
}, PageSize: 100,
})
if err != nil {
return nil, fmt.Errorf("list recoverable sandboxes: %w", err)
@@ -172,8 +175,8 @@ func (b Backend) Create(ctx context.Context, assignment taskassignment.Assignmen
if err := b.validate(); err != nil {
return nil, err
}
if assignment.Backend != taskassignment.BackendVM {
return nil, fmt.Errorf("OpenSandbox backend cannot create %q assignment", assignment.Backend)
if assignment.Placement != taskassignment.OpenSandboxVM {
return nil, fmt.Errorf("OpenSandbox backend cannot create %q", assignment.Placement.Key())
}
environment := clone(b.Config.Env)
for key, value := range launch.Environment {
+1 -1
View File
@@ -63,7 +63,7 @@ func backend(lifecycle Lifecycle) Backend {
func assignment() taskassignment.Assignment {
return taskassignment.Assignment{
ID: "gitea-task-42", Backend: taskassignment.BackendVM,
ID: "gitea-task-42", Placement: taskassignment.OpenSandboxVM,
Task: &runnerv1.Task{Id: 42},
Identity: taskidentity.Identity{Repository: "owner/repo", Task: "publish", SPIFFEID: "spiffe://ddupan.top/ci/owner/repo/publish"},
}
+3 -3
View File
@@ -120,7 +120,7 @@ func (b Backend) RecoverAssignments(ctx context.Context) ([]taskassignment.Assig
if err := b.validate(); err != nil {
return nil, err
}
pods, err := b.API.ListPods(ctx, b.Config.Namespace, "ci.ddupan.top/backend=pod")
pods, err := b.API.ListPods(ctx, b.Config.Namespace, "ci.ddupan.top/workload-class=container,ci.ddupan.top/driver=kubernetes")
if err != nil {
return nil, fmt.Errorf("list recoverable assignment Pods: %w", err)
}
@@ -159,8 +159,8 @@ func (b Backend) Create(ctx context.Context, assignment taskassignment.Assignmen
if err := b.validate(); err != nil {
return nil, err
}
if assignment.Backend != taskassignment.BackendPod {
return nil, fmt.Errorf("Pod backend cannot create %q assignment", assignment.Backend)
if assignment.Placement != taskassignment.KubernetesContainer {
return nil, fmt.Errorf("Kubernetes backend cannot create %q", assignment.Placement.Key())
}
labels := clone(launch.Metadata.Labels)
labels["app.kubernetes.io/name"] = "gitea-dynamic-runner"
+1 -1
View File
@@ -59,7 +59,7 @@ func backend(api API) Backend {
func assignment() taskassignment.Assignment {
return taskassignment.Assignment{
ID: "gitea-task-42", Backend: taskassignment.BackendPod,
ID: "gitea-task-42", Placement: taskassignment.KubernetesContainer,
Task: &runnerv1.Task{Id: 42},
Identity: taskidentity.Identity{
Repository: "owner/repo", Task: "publish",
+13 -11
View File
@@ -13,12 +13,13 @@ import (
)
const (
EnvAssignmentID = "CI_ASSIGNMENT_ID"
EnvCapability = "CI_RUNNER_CAPABILITY"
EnvFacadeURL = "CI_RUNNER_FACADE_URL"
EnvFacadeID = "CI_RUNNER_FACADE_SPIFFE_ID"
EnvSPIFFEID = "CI_SPIFFE_ID"
EnvBackend = "CI_RUNNER_BACKEND"
EnvAssignmentID = "CI_ASSIGNMENT_ID"
EnvCapability = "CI_RUNNER_CAPABILITY"
EnvFacadeURL = "CI_RUNNER_FACADE_URL"
EnvFacadeID = "CI_RUNNER_FACADE_SPIFFE_ID"
EnvSPIFFEID = "CI_SPIFFE_ID"
EnvWorkloadClass = "CI_WORKLOAD_CLASS"
EnvDriver = "CI_WORKLOAD_DRIVER"
)
// Bootstrap emits assignment-scoped launch configuration. FacadeURL is the
@@ -48,7 +49,8 @@ func (b Bootstrap) Environment(assignment taskassignment.Assignment) (map[string
EnvFacadeURL: b.FacadeURL,
EnvFacadeID: b.FacadeSPIFFEID,
EnvSPIFFEID: assignment.Identity.SPIFFEID,
EnvBackend: string(assignment.Backend),
EnvWorkloadClass: string(assignment.Placement.Class),
EnvDriver: string(assignment.Placement.Driver),
"SPIFFE_ENDPOINT_SOCKET": b.WorkloadAPIAddr,
}, nil
}
@@ -67,7 +69,7 @@ type Registration struct {
Ephemeral bool `json:"ephemeral"`
}
func RegistrationJSON(assignmentID, capability, localProxyURL string, backend taskassignment.Backend) ([]byte, error) {
func RegistrationJSON(assignmentID, capability, localProxyURL string, placement taskassignment.Placement) ([]byte, error) {
if assignmentID == "" || capability == "" {
return nil, errors.New("assignment ID and runner capability are required")
}
@@ -75,13 +77,13 @@ func RegistrationJSON(assignmentID, capability, localProxyURL string, backend ta
if err != nil || parsed.Scheme != "http" || parsed.Host == "" {
return nil, errors.New("local runner proxy URL must be an absolute http URL")
}
if backend != taskassignment.BackendPod && backend != taskassignment.BackendVM {
return nil, fmt.Errorf("unsupported runner backend %q", backend)
if err := placement.Validate(); err != nil {
return nil, err
}
registration := Registration{
Warning: "Generated for one preassigned task by gitea-dynamic-runner.",
UUID: assignmentID, Name: assignmentID, Token: capability,
Address: localProxyURL, Labels: []string{"self-hosted", string(backend)}, Ephemeral: true,
Address: localProxyURL, Labels: []string{"self-hosted", string(placement.Class), string(placement.Driver)}, Ephemeral: true,
}
data, err := json.MarshalIndent(registration, "", " ")
if err != nil {
+5 -5
View File
@@ -25,7 +25,7 @@ func testBootstrap(t *testing.T) Bootstrap {
func testAssignment() taskassignment.Assignment {
return taskassignment.Assignment{
ID: "gitea-task-42", Backend: taskassignment.BackendPod,
ID: "gitea-task-42", Placement: taskassignment.KubernetesContainer,
Identity: taskidentity.Identity{SPIFFEID: "spiffe://ddupan.top/ci/owner/repo/publish"},
}
}
@@ -46,7 +46,7 @@ func TestEnvironmentIsDeterministicAndAssignmentScoped(t *testing.T) {
if first[EnvAssignmentID] != "gitea-task-42" || first[EnvSPIFFEID] != testAssignment().Identity.SPIFFEID {
t.Fatalf("environment = %#v", first)
}
if first[EnvBackend] != "pod" || first[EnvFacadeID] == "" {
if first[EnvWorkloadClass] != "container" || first[EnvDriver] != "kubernetes" || first[EnvFacadeID] == "" {
t.Fatalf("environment = %#v", first)
}
if first["SPIFFE_ENDPOINT_SOCKET"] != "unix:///run/spire/agent-sockets/spire-agent.sock" {
@@ -55,7 +55,7 @@ func TestEnvironmentIsDeterministicAndAssignmentScoped(t *testing.T) {
}
func TestRegistrationMatchesOfficialRunnerSchema(t *testing.T) {
data, err := RegistrationJSON("gitea-task-42", "capability", "http://127.0.0.1:8080", taskassignment.BackendVM)
data, err := RegistrationJSON("gitea-task-42", "capability", "http://127.0.0.1:8080", taskassignment.OpenSandboxVM)
if err != nil {
t.Fatal(err)
}
@@ -66,13 +66,13 @@ func TestRegistrationMatchesOfficialRunnerSchema(t *testing.T) {
if registration.UUID != "gitea-task-42" || registration.Token != "capability" || registration.Address != "http://127.0.0.1:8080" || !registration.Ephemeral {
t.Fatalf("registration = %#v", registration)
}
if len(registration.Labels) != 2 || registration.Labels[0] != "self-hosted" || registration.Labels[1] != "vm" {
if len(registration.Labels) != 3 || registration.Labels[0] != "self-hosted" || registration.Labels[1] != "vm" || registration.Labels[2] != "opensandbox" {
t.Fatalf("labels = %#v", registration.Labels)
}
}
func TestRegistrationRejectsNonLocalTLSAddress(t *testing.T) {
if _, err := RegistrationJSON("id", "capability", "https://facade.example", taskassignment.BackendPod); err == nil {
if _, err := RegistrationJSON("id", "capability", "https://facade.example", taskassignment.KubernetesContainer); err == nil {
t.Fatal("expected local proxy URL validation error")
}
}
+6 -6
View File
@@ -17,7 +17,7 @@ import (
type ExecutorConfig struct {
AssignmentID string
Capability string
Backend taskassignment.Backend
Placement taskassignment.Placement
FacadeURL string
FacadeSPIFFEID string
WorkloadAPIAddr string
@@ -71,7 +71,7 @@ func RunExecutor(ctx context.Context, config ExecutorConfig) error {
defer os.RemoveAll(workDir)
}
registration, err := RegistrationJSON(
config.AssignmentID, config.Capability, "http://"+listener.Addr().String(), config.Backend,
config.AssignmentID, config.Capability, "http://"+listener.Addr().String(), config.Placement,
)
if err != nil {
return err
@@ -144,13 +144,13 @@ func waitForFacade(ctx context.Context, endpoint string) error {
// the assignment-scoped values injected by the backend. The Workload API
// address follows SPIFFE_ENDPOINT_SOCKET through go-spiffe when not set here.
func ExecutorConfigFromEnvironment() (ExecutorConfig, error) {
backend := taskassignment.Backend(os.Getenv(EnvBackend))
if backend != taskassignment.BackendPod && backend != taskassignment.BackendVM {
return ExecutorConfig{}, fmt.Errorf("invalid %s %q", EnvBackend, backend)
placement := taskassignment.Placement{Class: taskassignment.WorkloadClass(os.Getenv(EnvWorkloadClass)), Driver: taskassignment.Driver(os.Getenv(EnvDriver))}
if err := placement.Validate(); err != nil {
return ExecutorConfig{}, err
}
config := ExecutorConfig{
AssignmentID: os.Getenv(EnvAssignmentID), Capability: os.Getenv(EnvCapability),
Backend: backend, FacadeURL: os.Getenv(EnvFacadeURL), FacadeSPIFFEID: os.Getenv(EnvFacadeID),
Placement: placement, FacadeURL: os.Getenv(EnvFacadeURL), FacadeSPIFFEID: os.Getenv(EnvFacadeID),
RunnerBinary: os.Getenv("GITEA_RUNNER_BINARY"), RunnerConfig: os.Getenv("GITEA_RUNNER_CONFIG_FILE"),
ListenAddress: "127.0.0.1:0",
Stdout: os.Stdout, Stderr: os.Stderr,
+29 -46
View File
@@ -6,7 +6,6 @@ import (
"encoding/json"
"errors"
"fmt"
"slices"
"strconv"
"gitea.dev/actionslib/pkg/model"
@@ -16,34 +15,27 @@ import (
"git.ddupan.top/panxiao81/gitea-dynamic-runner/internal/taskidentity"
)
const wireVersion = 1
type Backend string
const (
BackendPod Backend = "pod"
BackendVM Backend = "vm"
)
const wireVersion = 2
// Assignment is the only document persisted in the handoff queue.
type Assignment struct {
ID string
Backend Backend
Task *runnerv1.Task
Identity taskidentity.Identity
ID string
Placement Placement
Task *runnerv1.Task
Identity taskidentity.Identity
}
// FromMetadata reconstructs the minimal assignment needed to authorize an
// already-running executor after a controller restart. Backend metadata was
// already-running executor after a controller restart. Placement metadata was
// originally derived from the trusted Gitea task and is validated again here.
func FromMetadata(labels, annotations map[string]string, trustDomain string) (Assignment, error) {
taskID, err := strconv.ParseInt(labels["ci.ddupan.top/task-id"], 10, 64)
if err != nil || taskID < 1 {
return Assignment{}, errors.New("backend metadata has invalid task ID")
}
backend := Backend(labels["ci.ddupan.top/backend"])
if backend != BackendPod && backend != BackendVM {
return Assignment{}, errors.New("backend metadata has invalid backend")
placement := Placement{Class: WorkloadClass(labels["ci.ddupan.top/workload-class"]), Driver: Driver(labels["ci.ddupan.top/driver"])}
if err := placement.Validate(); err != nil {
return Assignment{}, err
}
id := labels["ci.ddupan.top/assignment-id"]
if id != fmt.Sprintf("gitea-task-%d", taskID) {
@@ -56,15 +48,15 @@ func FromMetadata(labels, annotations map[string]string, trustDomain string) (As
if err != nil {
return Assignment{}, err
}
return Assignment{ID: id, Backend: backend, Task: &runnerv1.Task{Id: taskID}, Identity: identity}, nil
return Assignment{ID: id, Placement: placement, Task: &runnerv1.Task{Id: taskID}, Identity: identity}, nil
}
type envelope struct {
Version int `json:"version"`
ID string `json:"id"`
Backend Backend `json:"backend"`
Task []byte `json:"task"`
Identity taskidentity.Identity `json:"identity"`
Version int `json:"version"`
ID string `json:"id"`
Placement Placement `json:"placement"`
Task []byte `json:"task"`
Identity taskidentity.Identity `json:"identity"`
}
// New derives all trusted assignment fields from the task fetched from Gitea.
@@ -76,40 +68,28 @@ func New(task *runnerv1.Task, trustDomain string) (Assignment, error) {
if err != nil {
return Assignment{}, err
}
backend, err := backendFromTask(task)
placement, err := placementFromTask(task)
if err != nil {
return Assignment{}, err
}
return Assignment{
ID: fmt.Sprintf("gitea-task-%d", task.GetId()),
Backend: backend,
Task: task,
Identity: identity,
ID: fmt.Sprintf("gitea-task-%d", task.GetId()),
Placement: placement,
Task: task,
Identity: identity,
}, nil
}
func backendFromTask(task *runnerv1.Task) (Backend, error) {
func placementFromTask(task *runnerv1.Task) (Placement, error) {
workflow, err := model.ReadWorkflow(bytes.NewReader(task.GetWorkflowPayload()))
if err != nil {
return "", fmt.Errorf("parse task workflow for backend: %w", err)
return Placement{}, fmt.Errorf("parse task workflow for placement: %w", err)
}
jobIDs := workflow.GetJobIDs()
if len(jobIDs) != 1 || workflow.GetJob(jobIDs[0]) == nil {
return "", fmt.Errorf("task workflow must contain exactly one non-empty job")
return Placement{}, fmt.Errorf("task workflow must contain exactly one non-empty job")
}
labels := workflow.GetJob(jobIDs[0]).RunsOnLabels()
if !slices.Contains(labels, "self-hosted") {
return "", fmt.Errorf("task runs-on labels must include self-hosted: %v", labels)
}
hasPod := slices.Contains(labels, string(BackendPod))
hasVM := slices.Contains(labels, string(BackendVM)) || slices.Contains(labels, "vm-dev")
if hasPod == hasVM {
return "", fmt.Errorf("task runs-on labels must select exactly one of pod or vm: %v", labels)
}
if hasPod {
return BackendPod, nil
}
return BackendVM, nil
return PlacementFromLabels(workflow.GetJob(jobIDs[0]).RunsOnLabels())
}
// Marshal encodes a versioned assignment. Protobuf preserves the exact Gitea task.
@@ -117,13 +97,16 @@ func Marshal(assignment Assignment) ([]byte, error) {
if assignment.Task == nil {
return nil, errors.New("assignment task is required")
}
if err := assignment.Placement.Validate(); err != nil {
return nil, err
}
task, err := proto.Marshal(assignment.Task)
if err != nil {
return nil, fmt.Errorf("marshal Gitea task: %w", err)
}
return json.Marshal(envelope{
Version: wireVersion,
ID: assignment.ID, Backend: assignment.Backend,
ID: assignment.ID, Placement: assignment.Placement,
Task: task, Identity: assignment.Identity,
})
}
@@ -145,7 +128,7 @@ func Unmarshal(data []byte, trustDomain string) (Assignment, error) {
if err != nil {
return Assignment{}, err
}
if wire.ID != canonical.ID || wire.Backend != canonical.Backend || wire.Identity != canonical.Identity {
if wire.ID != canonical.ID || wire.Placement != canonical.Placement || wire.Identity != canonical.Identity {
return Assignment{}, errors.New("assignment metadata does not match its Gitea task")
}
return canonical, nil
+24 -15
View File
@@ -21,29 +21,37 @@ func task(t *testing.T, labels string) *runnerv1.Task {
}
}
func TestNewSelectsBackendFromRunsOn(t *testing.T) {
func TestNewSelectsPlacementFromRunsOn(t *testing.T) {
for _, test := range []struct {
labels string
backend Backend
labels string
placement Placement
}{
{"[self-hosted, pod]", BackendPod},
{"[self-hosted, vm]", BackendVM},
{"[self-hosted, vm-dev]", BackendVM},
{"[self-hosted, pod]", KubernetesContainer},
{"[self-hosted, container]", KubernetesContainer},
{"[self-hosted, container, kubernetes]", KubernetesContainer},
{"[self-hosted, vm]", OpenSandboxVM},
{"[self-hosted, vm-dev]", OpenSandboxVM},
{"[self-hosted, vm, opensandbox]", OpenSandboxVM},
} {
assignment, err := New(task(t, test.labels), "ddupan.top")
if err != nil {
t.Fatal(err)
}
if assignment.Backend != test.backend || assignment.ID != "gitea-task-42" {
if assignment.Placement != test.placement || assignment.ID != "gitea-task-42" {
t.Fatalf("assignment = %#v", assignment)
}
}
}
func TestNewRejectsAmbiguousBackend(t *testing.T) {
func TestNewRejectsInvalidPlacement(t *testing.T) {
for _, labels := range []string{
"[self-hosted]",
"[self-hosted, pod, vm]",
"[self-hosted, pod, container]",
"[self-hosted, pod, kubernetes]",
"[self-hosted, container, opensandbox]",
"[self-hosted, vm, kubernetes]",
"[self-hosted, container, kubernetes, opensandbox]",
"[pod]",
} {
if _, err := New(task(t, labels), "ddupan.top"); err == nil {
@@ -65,27 +73,28 @@ func TestAssignmentWireRoundTripAndValidation(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if got.ID != want.ID || got.Backend != want.Backend || got.Identity != want.Identity || !bytes.Equal(got.Task.WorkflowPayload, want.Task.WorkflowPayload) {
if got.ID != want.ID || got.Placement != want.Placement || got.Identity != want.Identity || !bytes.Equal(got.Task.WorkflowPayload, want.Task.WorkflowPayload) {
t.Fatalf("round trip = %#v, want %#v", got, want)
}
tampered := bytes.Replace(data, []byte(`"backend":"pod"`), []byte(`"backend":"vm"`), 1)
tampered := bytes.Replace(data, []byte(`"driver":"kubernetes"`), []byte(`"driver":"opensandbox"`), 1)
if _, err := Unmarshal(tampered, "ddupan.top"); err == nil {
t.Fatal("expected tampered backend to fail")
t.Fatal("expected tampered placement to fail")
}
}
func TestFromMetadataRecoversMinimalAssignment(t *testing.T) {
assignment, err := FromMetadata(map[string]string{
"ci.ddupan.top/assignment-id": "gitea-task-42",
"ci.ddupan.top/task-id": "42",
"ci.ddupan.top/backend": "vm",
"ci.ddupan.top/assignment-id": "gitea-task-42",
"ci.ddupan.top/task-id": "42",
"ci.ddupan.top/workload-class": "vm",
"ci.ddupan.top/driver": "opensandbox",
}, map[string]string{
"ci.ddupan.top/repository": "owner/repo",
"ci.ddupan.top/job-key": "publish",
"ci.ddupan.top/spiffe-id": "spiffe://ddupan.top/ci/owner/repo/publish",
}, "ddupan.top")
if err != nil || assignment.Task.GetId() != 42 || assignment.Backend != BackendVM {
if err != nil || assignment.Task.GetId() != 42 || assignment.Placement != OpenSandboxVM {
t.Fatalf("assignment=%#v err=%v", assignment, err)
}
}
+76
View File
@@ -0,0 +1,76 @@
package taskassignment
import (
"errors"
"fmt"
"slices"
)
type WorkloadClass string
const (
WorkloadContainer WorkloadClass = "container"
WorkloadVM WorkloadClass = "vm"
)
type Driver string
const (
DriverKubernetes Driver = "kubernetes"
DriverOpenSandbox Driver = "opensandbox"
)
type Placement struct {
Class WorkloadClass `json:"workload_class"`
Driver Driver `json:"driver"`
}
var (
KubernetesContainer = Placement{Class: WorkloadContainer, Driver: DriverKubernetes}
OpenSandboxVM = Placement{Class: WorkloadVM, Driver: DriverOpenSandbox}
)
func (p Placement) Validate() error {
switch p {
case KubernetesContainer, OpenSandboxVM:
return nil
default:
return fmt.Errorf("unsupported workload placement %s/%s", p.Class, p.Driver)
}
}
func (p Placement) Key() string { return string(p.Class) + "." + string(p.Driver) }
func PlacementFromLabels(labels []string) (Placement, error) {
if !slices.Contains(labels, "self-hosted") {
return Placement{}, fmt.Errorf("task runs-on labels must include self-hosted: %v", labels)
}
legacyPod := slices.Contains(labels, "pod")
container := slices.Contains(labels, string(WorkloadContainer))
vm := slices.Contains(labels, string(WorkloadVM)) || slices.Contains(labels, "vm-dev")
kubernetes := slices.Contains(labels, string(DriverKubernetes))
opensandbox := slices.Contains(labels, string(DriverOpenSandbox))
if legacyPod {
if container || vm || kubernetes || opensandbox {
return Placement{}, errors.New("legacy pod label cannot be combined with workload or VM driver labels")
}
return KubernetesContainer, nil
}
if container == vm {
return Placement{}, fmt.Errorf("task runs-on labels must select exactly one workload class: %v", labels)
}
if kubernetes && opensandbox {
return Placement{}, fmt.Errorf("task runs-on labels select multiple drivers: %v", labels)
}
if container {
if opensandbox {
return Placement{}, fmt.Errorf("opensandbox does not support container workloads")
}
return KubernetesContainer, nil
}
if kubernetes {
return Placement{}, fmt.Errorf("kubernetes does not support VM workloads")
}
return OpenSandboxVM, nil
}
+5 -4
View File
@@ -206,10 +206,11 @@ func (w Worker) launchSpec(assignment taskassignment.Assignment) (LaunchSpec, er
func BackendMetadata(assignment taskassignment.Assignment) Metadata {
return Metadata{
Labels: map[string]string{
"ci.ddupan.top/runner": "true",
"ci.ddupan.top/assignment-id": assignment.ID,
"ci.ddupan.top/task-id": strconv.FormatInt(assignment.Task.GetId(), 10),
"ci.ddupan.top/backend": string(assignment.Backend),
"ci.ddupan.top/runner": "true",
"ci.ddupan.top/assignment-id": assignment.ID,
"ci.ddupan.top/task-id": strconv.FormatInt(assignment.Task.GetId(), 10),
"ci.ddupan.top/workload-class": string(assignment.Placement.Class),
"ci.ddupan.top/driver": string(assignment.Placement.Driver),
},
Annotations: map[string]string{
"ci.ddupan.top/repository": assignment.Identity.Repository,
+3 -3
View File
@@ -53,9 +53,9 @@ func (t *fakeTasks) Report(_ context.Context, _ int64, phase Phase) error {
func assignment() taskassignment.Assignment {
return taskassignment.Assignment{
ID: "gitea-task-42",
Backend: taskassignment.BackendPod,
Task: &runnerv1.Task{Id: 42},
ID: "gitea-task-42",
Placement: taskassignment.KubernetesContainer,
Task: &runnerv1.Task{Id: 42},
Identity: taskidentity.Identity{
Repository: "owner/repo",
Task: "publish",