panxiao81/gitea-dynamic-runner
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: separate workload class from placement driver
test / shell (pull_request) Successful in 28s
Details
test / python (pull_request) Successful in 59s
Details
test / go (pull_request) Successful in 3m35s
Details

Browse Source
This commit is contained in:
panxiao81
2026-09-25 16:48:32 +00:00
parent 162f742880
commit 897d327a26
21 changed files with 305 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/gitea-dynamic-runner/controller.go
+50 -50
View File
@@ -38,6 +38,15 @@ type runComponent func(context.Context) error
func (function runComponent) Run(ctx context.Context) error { return function(ctx) }
type terminalBackend interface {
MarkTerminal(context.Context, string) error
}
type placementRuntime struct {
backend terminalBackend
pool *backendpool.Pool
}
type controllerConfig struct {
Components controller.Selection
TrustDomain, WorkloadAPIAddr string
@@ -61,7 +70,7 @@ func runController(ctx context.Context) error {
if !slices.Contains(config.Components, controller.Scheduler) {
return errors.New("split worker deployment is not yet safe: scheduler/facade must be enabled with workers")
}
if !slices.Contains(config.Components, controller.PodWorker) && !slices.Contains(config.Components, controller.VMWorker) {
if !slices.Contains(config.Components, controller.KubernetesWorker) && !slices.Contains(config.Components, controller.OpenSandboxWorker) {
return errors.New("scheduler requires at least one local backend worker")
}
@@ -92,32 +101,19 @@ func runController(ctx context.Context) error {
return err
}
registry := runnerfacade.NewRegistry()
podPool := backendpool.New(config.PodCapacity)
vmPool := backendpool.New(config.VMCapacity)
var podExecutorBackend *podbackend.Backend
var vmExecutorBackend *opensandboxbackend.Backend
runtimes := make(map[taskassignment.Placement]placementRuntime)
giteaClient := giteaactions.NewClient(giteaactions.DefaultHTTPClient(), config.GiteaURL, config.GiteaUUID, config.GiteaToken)
facade := &runnerfacade.Facade{
Registry: registry, Capabilities: capabilities, Upstream: giteaClient,
OnTerminal: func(ctx context.Context, assignment taskassignment.Assignment) error {
switch assignment.Backend {
case taskassignment.BackendPod:
if podExecutorBackend == nil {
return errors.New("Pod lifecycle backend is not configured")
}
if err := podExecutorBackend.MarkTerminal(ctx, assignment.ID); err != nil {
return err
}
podPool.Release(assignment.ID)
case taskassignment.BackendVM:
if vmExecutorBackend == nil {
return errors.New("VM lifecycle backend is not configured")
}
if err := vmExecutorBackend.MarkTerminal(ctx, assignment.ID); err != nil {
return err
}
vmPool.Release(assignment.ID)
runtime, ok := runtimes[assignment.Placement]
if !ok {
return fmt.Errorf("placement runtime %s is not configured", assignment.Placement.Key())
}
if err := runtime.backend.MarkTerminal(ctx, assignment.ID); err != nil {
return err
}
runtime.pool.Release(assignment.ID)
return nil
},
}
@@ -127,11 +123,11 @@ func runController(ctx context.Context) error {
}
labels := []string{"self-hosted"}
if slices.Contains(config.Components, controller.PodWorker) {
labels = append(labels, string(taskassignment.BackendPod))
if slices.Contains(config.Components, controller.KubernetesWorker) {
labels = append(labels, "pod", string(taskassignment.WorkloadContainer), string(taskassignment.DriverKubernetes))
}
if slices.Contains(config.Components, controller.VMWorker) {
labels = append(labels, config.VMRunnerLabel)
if slices.Contains(config.Components, controller.OpenSandboxWorker) {
labels = append(labels, config.VMRunnerLabel, string(taskassignment.DriverOpenSandbox))
}
poller := taskscheduler.Poller{
Client: giteaClient,
@@ -168,7 +164,9 @@ func runController(ctx context.Context) error {
}),
}
if slices.Contains(config.Components, controller.PodWorker) {
if slices.Contains(config.Components, controller.KubernetesWorker) {
placement := taskassignment.KubernetesContainer
pool := backendpool.New(config.PodCapacity)
client, err := podbackend.NewInClusterClient()
if err != nil {
return err
@@ -179,57 +177,59 @@ func runController(ctx context.Context) error {
SPIRECluster: config.SPIRECluster, SPIREClass: config.SPIREClass,
SPIREAgentID: config.SPIREAgentID, ExecutorUID: config.PodExecutorUID,
}}
podExecutorBackend = &backend
runtimes[placement] = placementRuntime{backend: backend, pool: pool}
assignments, err := backend.RecoverAssignments(ctx)
if err != nil {
return err
}
for _, assignment := range assignments {
podPool.Restore(assignment.ID)
pool.Restore(assignment.ID)
if err := registry.RecoverClaimed(assignment); err != nil {
return fmt.Errorf("recover Pod facade claim %s: %w", assignment.ID, err)
}
}
component, err := workerComponent(ctx, workerJS, config, taskassignment.BackendPod, config.PodCapacity, taskworker.Worker{Backend: backend, Bootstrap: bootstrap, OnEvent: workerEventLogger(taskassignment.BackendPod)}, registry, podPool)
component, err := workerComponent(ctx, workerJS, config, placement, config.PodCapacity, taskworker.Worker{Backend: backend, Bootstrap: bootstrap, OnEvent: workerEventLogger(placement)}, registry, pool)
if err != nil {
return err
}
lifecycle := podbackend.Lifecycle{Backend: backend, OnError: func(err error) {
slog.Error("backend lifecycle error", "component", "lifecycle", "backend", taskassignment.BackendPod, "error", err)
slog.Error("backend lifecycle error", "component", "lifecycle", "placement", placement.Key(), "error", err)
}}
components[controller.PodWorker] = runComponent(func(ctx context.Context) error {
components[controller.KubernetesWorker] = runComponent(func(ctx context.Context) error {
group, groupContext := errgroup.WithContext(ctx)
group.Go(func() error { return component.Run(groupContext) })
group.Go(func() error { return lifecycle.Run(groupContext) })
return group.Wait()
})
}
if slices.Contains(config.Components, controller.VMWorker) {
if slices.Contains(config.Components, controller.OpenSandboxWorker) {
placement := taskassignment.OpenSandboxVM
pool := backendpool.New(config.VMCapacity)
lifecycle := opensandboxbackend.NewLifecycleClient(config.OpenSandboxURL, config.OpenSandboxAPIKey, &http.Client{Timeout: 60 * time.Second})
backend := opensandboxbackend.Backend{Lifecycle: lifecycle, Config: opensandboxbackend.Config{
Pool: config.OpenSandboxPool, Timeout: config.VMTimeout,
Entrypoint: []string{"/usr/local/bin/gitea-dynamic-runner", "executor"},
Env: map[string]string{"SPIFFE_ENDPOINT_SOCKET": config.WorkloadAPIAddr},
}}
vmExecutorBackend = &backend
runtimes[placement] = placementRuntime{backend: backend, pool: pool}
assignments, err := backend.RecoverAssignments(ctx, config.TrustDomain)
if err != nil {
return err
}
for _, assignment := range assignments {
vmPool.Restore(assignment.ID)
pool.Restore(assignment.ID)
if err := registry.RecoverClaimed(assignment); err != nil {
return fmt.Errorf("recover VM facade claim %s: %w", assignment.ID, err)
}
}
component, err := workerComponent(ctx, workerJS, config, taskassignment.BackendVM, config.VMCapacity, taskworker.Worker{Backend: backend, Bootstrap: bootstrap, OnEvent: workerEventLogger(taskassignment.BackendVM)}, registry, vmPool)
component, err := workerComponent(ctx, workerJS, config, placement, config.VMCapacity, taskworker.Worker{Backend: backend, Bootstrap: bootstrap, OnEvent: workerEventLogger(placement)}, registry, pool)
if err != nil {
return err
}
lifecycleReconciler := opensandboxbackend.LifecycleReconciler{Backend: backend, OnError: func(err error) {
slog.Error("backend lifecycle error", "component", "lifecycle", "backend", taskassignment.BackendVM, "error", err)
slog.Error("backend lifecycle error", "component", "lifecycle", "placement", placement.Key(), "error", err)
}}
components[controller.VMWorker] = runComponent(func(ctx context.Context) error {
components[controller.OpenSandboxWorker] = runComponent(func(ctx context.Context) error {
group, groupContext := errgroup.WithContext(ctx)
group.Go(func() error { return component.Run(groupContext) })
group.Go(func() error { return lifecycleReconciler.Run(groupContext) })
@@ -291,25 +291,25 @@ func connectNATS(server, user, password, caFile, clientName string) (*nats.Conn,
return nats.Connect(server, options...)
}
func workerComponent(ctx context.Context, js jetstream.JetStream, config controllerConfig, backend taskassignment.Backend, capacity int, accepter assignmentqueue.Accepter, claims assignmentqueue.Claims, admission assignmentqueue.Admission) (controller.Component, error) {
consumer, err := assignmentqueue.OpenConsumer(ctx, js, config.Stream, config.SubjectBase, backend, capacity)
func workerComponent(ctx context.Context, js jetstream.JetStream, config controllerConfig, placement taskassignment.Placement, capacity int, accepter assignmentqueue.Accepter, claims assignmentqueue.Claims, admission assignmentqueue.Admission) (controller.Component, error) {
consumer, err := assignmentqueue.OpenConsumer(ctx, js, config.Stream, config.SubjectBase, placement, capacity)
if err != nil {
return nil, err
}
return assignmentqueue.ConsumerComponent{
Consumer: consumer, Capacity: capacity,
Processor: assignmentqueue.Processor{TrustDomain: config.TrustDomain, Accepter: accepter, Claims: claims, Admission: admission, OnEvent: func(event assignmentqueue.Event) {
slog.Info("assignment transition", "component", "worker", "event", event.Name, "backend", event.Backend, "assignment", event.AssignmentID, "retry_delay", event.RetryDelay)
slog.Info("assignment transition", "component", "worker", "event", event.Name, "placement", event.Placement.Key(), "assignment", event.AssignmentID, "retry_delay", event.RetryDelay)
}},
OnError: func(err error) {
slog.Error("assignment processing error", "component", "worker", "backend", backend, "error", err)
slog.Error("assignment processing error", "component", "worker", "placement", placement.Key(), "error", err)
},
}, nil
}
func workerEventLogger(backend taskassignment.Backend) func(taskworker.Event) {
func workerEventLogger(placement taskassignment.Placement) func(taskworker.Event) {
return func(event taskworker.Event) {
slog.Info("executor transition", "component", "worker", "event", event.Name, "backend", backend, "assignment", event.AssignmentID, "executor", event.Executor, "phase", event.Phase)
slog.Info("executor transition", "component", "worker", "event", event.Name, "placement", placement.Key(), "assignment", event.AssignmentID, "executor", event.Executor, "phase", event.Phase)
}
}
@@ -364,18 +364,18 @@ func loadControllerConfig() (controllerConfig, error) {
if config.WorkloadAPIAddr == "" || config.FacadeURL == "" || config.FacadeSPIFFEID == "" {
return controllerConfig{}, errors.New("SPIFFE_ENDPOINT_SOCKET, RUNNER_FACADE_URL, and RUNNER_FACADE_SPIFFE_ID are required")
}
if slices.Contains(selection, controller.PodWorker) && config.PodImage == "" {
return controllerConfig{}, errors.New("POD_EXECUTOR_IMAGE is required for pod-worker")
if slices.Contains(selection, controller.KubernetesWorker) && config.PodImage == "" {
return controllerConfig{}, errors.New("POD_EXECUTOR_IMAGE is required for kubernetes-worker")
}
if slices.Contains(selection, controller.PodWorker) && config.SPIREAgentID == "" {
return controllerConfig{}, errors.New("SPIRE_AGENT_ID is required for pod-worker")
if slices.Contains(selection, controller.KubernetesWorker) && config.SPIREAgentID == "" {
return controllerConfig{}, errors.New("SPIRE_AGENT_ID is required for kubernetes-worker")
}
if slices.Contains(selection, controller.VMWorker) {
if slices.Contains(selection, controller.OpenSandboxWorker) {
if config.VMRunnerLabel != "vm" && config.VMRunnerLabel != "vm-dev" {
return controllerConfig{}, errors.New("VM_RUNNER_LABEL must be vm or vm-dev")
}
if config.OpenSandboxURL == "" {
return controllerConfig{}, errors.New("OPENSANDBOX_API is required for vm-worker")
return controllerConfig{}, errors.New("OPENSANDBOX_API is required for opensandbox-worker")
}
config.OpenSandboxAPIKey, err = read("OPENSANDBOX_API_KEY_FILE")
if err != nil {
cmd/gitea-dynamic-runner/controller_test.go
+1 -1
View File
@@ -34,7 +34,7 @@ func TestLoadControllerConfigUsesFileSecrets(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(config.Components) != 2 || config.Components[0] != controller.Scheduler || config.Components[1] != controller.PodWorker {
if len(config.Components) != 2 || config.Components[0] != controller.Scheduler || config.Components[1] != controller.KubernetesWorker {
t.Fatalf("components = %#v", config.Components)
}
if config.GiteaUUID != "scheduler-uuid" || config.GiteaToken != "scheduler-token" || config.NATSProducerPassword != "producer-password" || config.NATSWorkerPassword != "worker-password" {