fix: gate scheduler task concurrency
This commit is contained in:
@@ -85,8 +85,12 @@ func runController(ctx context.Context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
registry := runnerfacade.NewRegistry()
|
registry := runnerfacade.NewRegistry()
|
||||||
|
gate := taskscheduler.NewSingleFlightGate()
|
||||||
giteaClient := giteaactions.NewClient(giteaactions.DefaultHTTPClient(), config.GiteaURL, config.GiteaUUID, config.GiteaToken)
|
giteaClient := giteaactions.NewClient(giteaactions.DefaultHTTPClient(), config.GiteaURL, config.GiteaUUID, config.GiteaToken)
|
||||||
facade := &runnerfacade.Facade{Registry: registry, Capabilities: capabilities, Upstream: giteaClient}
|
facade := &runnerfacade.Facade{
|
||||||
|
Registry: registry, Capabilities: capabilities, Upstream: giteaClient,
|
||||||
|
OnTerminal: func(taskassignment.Assignment) { gate.Release() },
|
||||||
|
}
|
||||||
bootstrap := runnerbootstrap.Bootstrap{
|
bootstrap := runnerbootstrap.Bootstrap{
|
||||||
Capabilities: capabilities, FacadeURL: config.FacadeURL, FacadeSPIFFEID: config.FacadeSPIFFEID,
|
Capabilities: capabilities, FacadeURL: config.FacadeURL, FacadeSPIFFEID: config.FacadeSPIFFEID,
|
||||||
}
|
}
|
||||||
@@ -99,7 +103,7 @@ func runController(ctx context.Context) error {
|
|||||||
labels = append(labels, string(taskassignment.BackendVM))
|
labels = append(labels, string(taskassignment.BackendVM))
|
||||||
}
|
}
|
||||||
poller := taskscheduler.Poller{
|
poller := taskscheduler.Poller{
|
||||||
Client: giteaClient,
|
Client: giteaClient, Gate: gate,
|
||||||
Scheduler: &taskscheduler.Scheduler{TrustDomain: config.TrustDomain, Dispatcher: assignmentqueue.Publisher{
|
Scheduler: &taskscheduler.Scheduler{TrustDomain: config.TrustDomain, Dispatcher: assignmentqueue.Publisher{
|
||||||
JetStream: producerJS, SubjectBase: config.SubjectBase,
|
JetStream: producerJS, SubjectBase: config.SubjectBase,
|
||||||
}},
|
}},
|
||||||
|
|||||||
@@ -79,6 +79,9 @@ facade,并严格校验 facade 的 SPIFFE ID。这样无需修改 runner 或把
|
|||||||
- consumer 在 executor 使用上述 facade 成功 claim task 后确认 assignment;无需把完整
|
- consumer 在 executor 使用上述 facade 成功 claim task 后确认 assignment;无需把完整
|
||||||
task 写入 Pod annotation、OpenSandbox metadata 或环境变量。
|
task 写入 Pod annotation、OpenSandbox metadata 或环境变量。
|
||||||
- Pod 与 VM 共享 task/executor 协议,只有环境创建和销毁实现不同。
|
- Pod 与 VM 共享 task/executor 协议,只有环境创建和销毁实现不同。
|
||||||
|
- 首次生产 canary 使用 controller 进程内 single-flight gate:只有官方 runner 的终态
|
||||||
|
`UpdateTask` 已被 Gitea 接受后才允许 Fetch 下一条任务。它把未知故障收敛为停止领取,
|
||||||
|
而不是在 backlog 下连续创建 executor;后续容量调度必须以 backend 实际运行资源为准。
|
||||||
- 两种 backend 都注入同一份 runner bootstrap 环境;Pod 仍由 homelab Kubernetes 原生
|
- 两种 backend 都注入同一份 runner bootstrap 环境;Pod 仍由 homelab Kubernetes 原生
|
||||||
创建,只有 VM 经 OpenSandbox 创建,bootstrap 机制不改变 backend 边界。
|
创建,只有 VM 经 OpenSandbox 创建,bootstrap 机制不改变 backend 边界。
|
||||||
|
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ type Facade struct {
|
|||||||
Registry *Registry
|
Registry *Registry
|
||||||
Capabilities Capabilities
|
Capabilities Capabilities
|
||||||
Upstream Upstream
|
Upstream Upstream
|
||||||
|
OnTerminal func(taskassignment.Assignment)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Facade) Handler() (string, http.Handler) {
|
func (f *Facade) Handler() (string, http.Handler) {
|
||||||
@@ -98,7 +99,11 @@ func (f *Facade) UpdateTask(ctx context.Context, request *connect.Request[runner
|
|||||||
if request.Msg.GetState().GetId() != assignment.Task.GetId() {
|
if request.Msg.GetState().GetId() != assignment.Task.GetId() {
|
||||||
return nil, connect.NewError(connect.CodePermissionDenied, errors.New("task update does not match assignment"))
|
return nil, connect.NewError(connect.CodePermissionDenied, errors.New("task update does not match assignment"))
|
||||||
}
|
}
|
||||||
return f.Upstream.UpdateTask(ctx, connect.NewRequest(request.Msg))
|
response, err := f.Upstream.UpdateTask(ctx, connect.NewRequest(request.Msg))
|
||||||
|
if err == nil && request.Msg.GetState().GetResult() != runnerv1.Result_RESULT_UNSPECIFIED && f.OnTerminal != nil {
|
||||||
|
f.OnTerminal(assignment)
|
||||||
|
}
|
||||||
|
return response, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Facade) UpdateLog(ctx context.Context, request *connect.Request[runnerv1.UpdateLogRequest]) (*connect.Response[runnerv1.UpdateLogResponse], error) {
|
func (f *Facade) UpdateLog(ctx context.Context, request *connect.Request[runnerv1.UpdateLogRequest]) (*connect.Response[runnerv1.UpdateLogResponse], error) {
|
||||||
|
|||||||
@@ -163,6 +163,29 @@ func TestFacadeForwardsOnlyMatchingTaskAndLogUpdates(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFacadeSignalsTerminalTaskAfterUpstreamAcceptsIt(t *testing.T) {
|
||||||
|
facade, assignment, token := testFacade(t)
|
||||||
|
ctx := WithSPIFFEID(context.Background(), assignment.Identity.SPIFFEID)
|
||||||
|
if _, err := facade.FetchTask(ctx, authenticatedRequest(&runnerv1.FetchTaskRequest{}, assignment.ID, token)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
completed := 0
|
||||||
|
facade.OnTerminal = func(got taskassignment.Assignment) {
|
||||||
|
if got.ID != assignment.ID {
|
||||||
|
t.Fatalf("terminal assignment = %s", got.ID)
|
||||||
|
}
|
||||||
|
completed++
|
||||||
|
}
|
||||||
|
if _, err := facade.UpdateTask(ctx, authenticatedRequest(&runnerv1.UpdateTaskRequest{
|
||||||
|
State: &runnerv1.TaskState{Id: 42, Result: runnerv1.Result_RESULT_SUCCESS},
|
||||||
|
}, assignment.ID, token)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if completed != 1 {
|
||||||
|
t.Fatalf("terminal notifications = %d", completed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCapabilitiesAreDeterministicAndAssignmentScoped(t *testing.T) {
|
func TestCapabilitiesAreDeterministicAndAssignmentScoped(t *testing.T) {
|
||||||
capabilities, err := NewCapabilities([]byte("0123456789abcdef0123456789abcdef"))
|
capabilities, err := NewCapabilities([]byte("0123456789abcdef0123456789abcdef"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
package taskscheduler
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
// SingleFlightGate keeps at most one fetched task in flight. Release is
|
||||||
|
// idempotent so repeated terminal updates cannot increase capacity.
|
||||||
|
type SingleFlightGate struct {
|
||||||
|
token chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewSingleFlightGate() *SingleFlightGate {
|
||||||
|
gate := &SingleFlightGate{token: make(chan struct{}, 1)}
|
||||||
|
gate.token <- struct{}{}
|
||||||
|
return gate
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *SingleFlightGate) Acquire(ctx context.Context) error {
|
||||||
|
select {
|
||||||
|
case <-g.token:
|
||||||
|
return nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *SingleFlightGate) Release() {
|
||||||
|
select {
|
||||||
|
case g.token <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package taskscheduler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSingleFlightGateBlocksUntilTerminalRelease(t *testing.T) {
|
||||||
|
gate := NewSingleFlightGate()
|
||||||
|
if err := gate.Acquire(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
blocked, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
if err := gate.Acquire(blocked); err == nil {
|
||||||
|
t.Fatal("second task acquired capacity before release")
|
||||||
|
}
|
||||||
|
gate.Release()
|
||||||
|
gate.Release()
|
||||||
|
if err := gate.Acquire(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
blockedAgain, cancelAgain := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||||
|
defer cancelAgain()
|
||||||
|
if err := gate.Acquire(blockedAgain); err == nil {
|
||||||
|
t.Fatal("duplicate terminal release increased capacity")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -26,6 +26,7 @@ type PollerConfig struct {
|
|||||||
type Poller struct {
|
type Poller struct {
|
||||||
Client PollClient
|
Client PollClient
|
||||||
Scheduler *Scheduler
|
Scheduler *Scheduler
|
||||||
|
Gate *SingleFlightGate
|
||||||
Config PollerConfig
|
Config PollerConfig
|
||||||
OnError func(error)
|
OnError func(error)
|
||||||
}
|
}
|
||||||
@@ -50,7 +51,14 @@ func (p Poller) Run(ctx context.Context) error {
|
|||||||
errorBackoff = 5 * time.Second
|
errorBackoff = 5 * time.Second
|
||||||
}
|
}
|
||||||
var tasksVersion int64
|
var tasksVersion int64
|
||||||
|
haveLease := false
|
||||||
for {
|
for {
|
||||||
|
if p.Gate != nil && !haveLease {
|
||||||
|
if err := p.Gate.Acquire(ctx); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
haveLease = true
|
||||||
|
}
|
||||||
response, err := p.Client.FetchTask(ctx, tasksVersion)
|
response, err := p.Client.FetchTask(ctx, tasksVersion)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
@@ -72,6 +80,10 @@ func (p Poller) Run(ctx context.Context) error {
|
|||||||
tasksVersion = response.GetTasksVersion()
|
tasksVersion = response.GetTasksVersion()
|
||||||
task := response.GetTask()
|
task := response.GetTask()
|
||||||
if task == nil {
|
if task == nil {
|
||||||
|
if p.Gate != nil {
|
||||||
|
p.Gate.Release()
|
||||||
|
haveLease = false
|
||||||
|
}
|
||||||
if !wait(ctx, emptyBackoff) {
|
if !wait(ctx, emptyBackoff) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -79,6 +91,7 @@ func (p Poller) Run(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
for {
|
for {
|
||||||
if err := p.Scheduler.Run(ctx, task); err == nil {
|
if err := p.Scheduler.Run(ctx, task); err == nil {
|
||||||
|
haveLease = false
|
||||||
break
|
break
|
||||||
} else {
|
} else {
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
|
|||||||
Reference in New Issue
Block a user