feat: 接通 Database 凭据准备闭环
This commit is contained in:
@@ -19,6 +19,7 @@ import (
|
||||
const (
|
||||
TenantFinalizer = "database.ayatori.ddupan.top/tenant-protection"
|
||||
DatabaseFinalizer = "database.ayatori.ddupan.top/database-protection"
|
||||
readyCondition = "Ready"
|
||||
)
|
||||
|
||||
// BindingResources 读取领域所需事实,并把用例结果呈现为 CR、finalizer 与 Conditions。
|
||||
@@ -148,7 +149,7 @@ func (r *BindingResources) presentStatus(ctx context.Context, object *databasev1
|
||||
}
|
||||
}
|
||||
meta.SetStatusCondition(&object.Status.Conditions, metav1.Condition{
|
||||
Type: "Ready", Status: metav1.ConditionFalse, Reason: status.Reason, Message: status.Message,
|
||||
Type: readyCondition, Status: metav1.ConditionFalse, Reason: status.Reason, Message: status.Message,
|
||||
ObservedGeneration: object.Generation,
|
||||
})
|
||||
if equality.Semantic.DeepEqual(*previous, object.Status) {
|
||||
@@ -186,6 +187,6 @@ func bindingVersionConflict(resource, name string) error {
|
||||
}
|
||||
|
||||
func currentReady(generation int64, conditions []metav1.Condition) bool {
|
||||
condition := meta.FindStatusCondition(conditions, "Ready")
|
||||
condition := meta.FindStatusCondition(conditions, readyCondition)
|
||||
return condition != nil && condition.Status == metav1.ConditionTrue && condition.ObservedGeneration == generation
|
||||
}
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package kubernetes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
databasev1alpha1 "git.ddupan.top/panxiao81/ayatori/api/database/v1alpha1"
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/application"
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/domain/binding"
|
||||
"k8s.io/apimachinery/pkg/api/equality"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
)
|
||||
|
||||
// CredentialResources 只映射资源事实与状态写入,供应资格和执行顺序由用例决定。
|
||||
type CredentialResources struct {
|
||||
Client client.Client
|
||||
Reader client.Reader
|
||||
}
|
||||
|
||||
var _ application.CredentialResources = (*CredentialResources)(nil)
|
||||
|
||||
func (r *CredentialResources) Load(ctx context.Context, name string) (*application.CredentialRecord, error) {
|
||||
database := &databasev1alpha1.PostgreSQLDatabase{}
|
||||
if err := r.Reader.Get(ctx, types.NamespacedName{Name: name}, database); err != nil {
|
||||
return nil, client.IgnoreNotFound(err)
|
||||
}
|
||||
record := &application.CredentialRecord{
|
||||
Database: *bindingDatabase(database), DatabaseProtected: controllerutil.ContainsFinalizer(database, DatabaseFinalizer),
|
||||
Status: credentialStatus(database),
|
||||
}
|
||||
if database.Spec.Source != "Provision" {
|
||||
return record, nil
|
||||
}
|
||||
if ref := database.Spec.TenantRef; ref != nil {
|
||||
tenant := &databasev1alpha1.PostgreSQLTenant{}
|
||||
err := r.Reader.Get(ctx, types.NamespacedName{Namespace: ref.Namespace, Name: string(ref.Name)}, tenant)
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
if err == nil {
|
||||
record.Tenant = bindingTenant(tenant)
|
||||
record.TenantProtected = controllerutil.ContainsFinalizer(tenant, TenantFinalizer)
|
||||
}
|
||||
}
|
||||
instance := &databasev1alpha1.PostgreSQLInstance{}
|
||||
err := r.Reader.Get(ctx, types.NamespacedName{Name: string(database.Spec.InstanceRef.Name)}, instance)
|
||||
if err != nil {
|
||||
return record, client.IgnoreNotFound(err)
|
||||
}
|
||||
observed, err := instanceRecord(instance)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
record.Instance = &application.CredentialInstance{
|
||||
Identity: binding.Identity{Name: instance.Name, UID: string(instance.UID)},
|
||||
Deleting: !instance.DeletionTimestamp.IsZero(), Ready: currentReady(instance.Generation, instance.Status.Conditions),
|
||||
Generation: instance.Generation, Endpoint: observed.Target.Definition().Endpoint(),
|
||||
}
|
||||
return record, nil
|
||||
}
|
||||
|
||||
func credentialStatus(database *databasev1alpha1.PostgreSQLDatabase) application.CredentialStatus {
|
||||
status := application.CredentialStatus{Version: database.Status.CredentialVersion}
|
||||
if ref := database.Status.CredentialRef; ref != nil {
|
||||
status.Location = &application.CredentialLocation{Mount: ref.Mount, Path: ref.Path}
|
||||
}
|
||||
if condition := meta.FindStatusCondition(database.Status.Conditions, application.CredentialsReady); condition != nil {
|
||||
status.Ready = condition.Status == metav1.ConditionTrue && condition.ObservedGeneration == database.Generation
|
||||
status.Reason, status.Message = condition.Reason, condition.Message
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
func (r *CredentialResources) Save(ctx context.Context, record *application.CredentialRecord, status application.CredentialStatus) (*application.CredentialRecord, error) {
|
||||
bindingResources := &BindingResources{Client: r.Client, Reader: r.Reader}
|
||||
database, err := bindingResources.databaseAtVersion(ctx, &record.Database)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
previous := database.Status.DeepCopy()
|
||||
if status.Location != nil {
|
||||
database.Status.CredentialRef = &databasev1alpha1.CredentialReference{Mount: status.Location.Mount, Path: status.Location.Path}
|
||||
}
|
||||
database.Status.CredentialVersion = status.Version
|
||||
conditionStatus := metav1.ConditionFalse
|
||||
if status.Ready {
|
||||
conditionStatus = metav1.ConditionTrue
|
||||
}
|
||||
meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{
|
||||
Type: application.CredentialsReady, Status: conditionStatus, Reason: status.Reason, Message: status.Message,
|
||||
ObservedGeneration: database.Generation,
|
||||
})
|
||||
meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{
|
||||
Type: readyCondition, Status: metav1.ConditionFalse, Reason: "ProvisioningIncomplete",
|
||||
Message: "凭据准备不代表 PostgreSQL 供应、应用登录或 Tenant 交付已经完成", ObservedGeneration: database.Generation,
|
||||
})
|
||||
database.Status.ObservedGeneration = database.Generation
|
||||
if !equality.Semantic.DeepEqual(*previous, database.Status) {
|
||||
if err := r.Client.Status().Update(ctx, database); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
updated := *record
|
||||
updated.Database = *bindingDatabase(database)
|
||||
updated.Status = credentialStatus(database)
|
||||
// 旧 CRD 会裁剪未知 status 字段。不能把 HTTP 成功当作位置/版本已保存后继续写后端。
|
||||
if !equality.Semantic.DeepEqual(updated.Status.Location, status.Location) || updated.Status.Version != status.Version {
|
||||
return nil, fmt.Errorf("凭据位置或版本未被 API 保留;请先升级 Database CRD,未继续外部操作")
|
||||
}
|
||||
return &updated, nil
|
||||
}
|
||||
|
||||
func (r *CredentialResources) CheckCurrent(ctx context.Context, record *application.CredentialRecord) error {
|
||||
current, err := r.Load(ctx, record.Database.Identity.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current == nil || current.Database.Identity != record.Database.Identity || current.Database.Revision != record.Database.Revision ||
|
||||
!sameCredentialDependencies(current, record) {
|
||||
return apierrors.NewConflict(databasev1alpha1.GroupVersion.WithResource("postgresqldatabases").GroupResource(), record.Database.Identity.Name,
|
||||
fmt.Errorf("凭据准备的资源快照已变化;停止本轮操作并重新观察"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sameCredentialDependencies(current, previous *application.CredentialRecord) bool {
|
||||
if current.Tenant == nil || previous.Tenant == nil || current.Instance == nil || previous.Instance == nil {
|
||||
return false
|
||||
}
|
||||
// Tenant Ready 的诊断变化、Instance 对同一 generation 的观测刷新不改变写入目标。
|
||||
// 仍检查申请 spec generation、完整绑定身份、删除状态、保护和当前 Instance Ready。
|
||||
return current.Tenant.Generation == previous.Tenant.Generation &&
|
||||
equality.Semantic.DeepEqual(current.Tenant.Tenant, previous.Tenant.Tenant) &&
|
||||
current.TenantProtected == previous.TenantProtected && current.DatabaseProtected == previous.DatabaseProtected &&
|
||||
current.Instance.Instance == previous.Instance.Instance && current.Instance.Generation == previous.Instance.Generation &&
|
||||
current.Instance.Endpoint == previous.Instance.Endpoint
|
||||
}
|
||||
@@ -84,7 +84,7 @@ func (r *InstanceResources) PresentInstance(ctx context.Context, result applicat
|
||||
ready = metav1.ConditionTrue
|
||||
}
|
||||
meta.SetStatusCondition(&object.Status.Conditions, metav1.Condition{
|
||||
Type: "Ready", Status: ready, ObservedGeneration: object.Generation,
|
||||
Type: readyCondition, Status: ready, ObservedGeneration: object.Generation,
|
||||
Reason: result.Reason, Message: result.Message,
|
||||
})
|
||||
if !equality.Semantic.DeepEqual(*previous, object.Status) {
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package openbao
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/application"
|
||||
)
|
||||
|
||||
var _ application.CredentialStore = (*Credentials)(nil)
|
||||
|
||||
func (c *Credentials) ProvisionLocation(uid string) (application.CredentialLocation, error) {
|
||||
path, err := c.ProvisionPath(uid)
|
||||
if err != nil {
|
||||
return application.CredentialLocation{}, err
|
||||
}
|
||||
return application.CredentialLocation{Mount: c.mount, Path: path}, nil
|
||||
}
|
||||
|
||||
func (c *Credentials) ReadCredential(ctx context.Context, location application.CredentialLocation, version int64) (application.ApplicationCredential, error) {
|
||||
if location.Mount != c.mount {
|
||||
return application.ApplicationCredential{}, ErrInvalidLocation
|
||||
}
|
||||
if version == 0 {
|
||||
return c.Read(ctx, location.Path)
|
||||
}
|
||||
return c.ReadConfirmed(ctx, location.Path, version)
|
||||
}
|
||||
|
||||
func (c *Credentials) CreateCredential(ctx context.Context, location application.CredentialLocation, credential application.ApplicationCredential) (int64, error) {
|
||||
if location.Mount != c.mount {
|
||||
return 0, ErrInvalidLocation
|
||||
}
|
||||
if err := c.Create(ctx, location.Path, credential); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Create 的合同是 CAS=0 且回读版本 1 和七键完全一致。
|
||||
return 1, nil
|
||||
}
|
||||
@@ -32,11 +32,11 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidLocation = errors.New("credential location is outside the configured scope")
|
||||
ErrUnavailable = errors.New("credential backend unavailable")
|
||||
ErrNotFound = errors.New("application credential not found")
|
||||
ErrConflict = errors.New("credential creation requires manual conflict resolution")
|
||||
ErrUncertain = errors.New("credential creation outcome is uncertain; manual resolution required")
|
||||
ErrInvalidLocation = application.ErrCredentialLocation
|
||||
ErrUnavailable = application.ErrCredentialUnavailable
|
||||
ErrNotFound = application.ErrCredentialNotFound
|
||||
ErrConflict = application.ErrCredentialConflict
|
||||
ErrUncertain = application.ErrCredentialUncertain
|
||||
)
|
||||
|
||||
var pathSegment = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
|
||||
@@ -45,6 +45,7 @@ var pathSegment = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
|
||||
// 本适配器既不自动认领已有值,也不提供覆盖、轮换或删除操作。
|
||||
type Credentials struct {
|
||||
kv *bao.KVv2
|
||||
mount string
|
||||
basePath string
|
||||
}
|
||||
|
||||
@@ -54,7 +55,7 @@ func NewCredentials(client *bao.Client, mount, basePath string) (*Credentials, e
|
||||
if client == nil || !validPath(mount) || !validPath(basePath) {
|
||||
return nil, ErrInvalidLocation
|
||||
}
|
||||
return &Credentials{kv: client.KVv2(mount), basePath: basePath}, nil
|
||||
return &Credentials{kv: client.KVv2(mount), mount: mount, basePath: basePath}, nil
|
||||
}
|
||||
|
||||
func validPath(value string) bool {
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
//go:build integration
|
||||
|
||||
package openbao_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/application"
|
||||
)
|
||||
|
||||
// 两个用例都先拿到同一真实 resourceVersion,再竞争固定位置;后续写入仍经过 API server。
|
||||
type concurrentCredentialResources struct {
|
||||
application.CredentialResources
|
||||
readers atomic.Int32
|
||||
loaded sync.WaitGroup
|
||||
}
|
||||
|
||||
func (r *concurrentCredentialResources) Load(ctx context.Context, name string) (*application.CredentialRecord, error) {
|
||||
record, err := r.CredentialResources.Load(ctx, name)
|
||||
if r.readers.Add(1) <= 2 {
|
||||
r.loaded.Done()
|
||||
r.loaded.Wait()
|
||||
}
|
||||
return record, err
|
||||
}
|
||||
|
||||
func testPreparationConcurrency(t *testing.T, f *preparationFixture) {
|
||||
database, _ := f.bound(t, "concurrent")
|
||||
service := f.service(t)
|
||||
resources := &concurrentCredentialResources{CredentialResources: service.Resources}
|
||||
resources.loaded.Add(2)
|
||||
service.Resources = resources
|
||||
results := make(chan error, 2)
|
||||
for range 2 {
|
||||
go func() { results <- service.Reconcile(t.Context(), database.Name) }()
|
||||
}
|
||||
succeeded, conflicted := 0, 0
|
||||
for range 2 {
|
||||
err := <-results
|
||||
switch {
|
||||
case err == nil:
|
||||
succeeded++
|
||||
case apierrors.IsConflict(err):
|
||||
conflicted++
|
||||
default:
|
||||
t.Fatalf("并发用例返回意外错误: %v", err)
|
||||
}
|
||||
}
|
||||
if succeeded != 1 || conflicted != 1 {
|
||||
t.Fatal("同一快照只能有一个用例成功固定位置并继续创建")
|
||||
}
|
||||
f.status(t, database, 1, application.CredentialPrepared)
|
||||
stored, err := f.bao.KVv2("secret").Get(t.Context(), database.Status.CredentialRef.Path)
|
||||
if err != nil || stored.VersionMetadata.Version != 1 {
|
||||
t.Fatal("并发准备用例只能产生一个凭据版本")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
//go:build integration
|
||||
|
||||
package openbao_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
bao "github.com/openbao/openbao/api/v2"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/client-go/rest"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/envtest"
|
||||
|
||||
databasev1alpha1 "git.ddupan.top/panxiao81/ayatori/api/database/v1alpha1"
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/adapter/kubernetes"
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/application"
|
||||
databasecontroller "git.ddupan.top/panxiao81/ayatori/internal/database/controller"
|
||||
)
|
||||
|
||||
const preparationNamespace = "credential-preparation"
|
||||
|
||||
type preparationFixture struct {
|
||||
api client.Client
|
||||
config *rest.Config
|
||||
scheme *runtime.Scheme
|
||||
bao *bao.Client
|
||||
}
|
||||
|
||||
func newPreparationFixture(t *testing.T) *preparationFixture {
|
||||
t.Helper()
|
||||
environment := &envtest.Environment{CRDDirectoryPaths: []string{"../../../../config/crd/bases"}, ErrorIfCRDPathMissing: true}
|
||||
config, err := environment.Start()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := environment.Stop(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
scheme := runtime.NewScheme()
|
||||
if err := databasev1alpha1.AddToScheme(scheme); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := corev1.AddToScheme(scheme); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
api, err := client.New(config, client.Options{Scheme: scheme})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := api.Create(t.Context(), &corev1.Namespace{Name: preparationNamespace}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &preparationFixture{api: api, config: config, scheme: scheme, bao: baoFixture(t)}
|
||||
}
|
||||
|
||||
func (f *preparationFixture) bound(t *testing.T, name string) (*databasev1alpha1.PostgreSQLDatabase, *databasev1alpha1.PostgreSQLTenant) {
|
||||
t.Helper()
|
||||
instance := &databasev1alpha1.PostgreSQLInstance{Name: name, Spec: databasev1alpha1.PostgreSQLInstanceSpec{
|
||||
Endpoint: databasev1alpha1.PostgreSQLEndpoint{Host: "postgres.example", HostAddr: "192.0.2.1"},
|
||||
AdminCredentialRef: databasev1alpha1.AdminCredentialReference{Name: "management"},
|
||||
}}
|
||||
if err := f.api.Create(t.Context(), instance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 本切片不连接 PG;这里只声明供应前置观察,真实管理能力由既有 PG 集成测试覆盖。
|
||||
meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{
|
||||
Type: "Ready", Status: metav1.ConditionTrue, Reason: "FixtureReady", Message: "隔离测试前置观察", ObservedGeneration: instance.Generation,
|
||||
})
|
||||
if err := f.api.Status().Update(t.Context(), instance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tenant := &databasev1alpha1.PostgreSQLTenant{Name: name, Namespace: preparationNamespace}
|
||||
tenant.Spec.Provision = &databasev1alpha1.DatabaseProvisionRequest{InstanceRef: databasev1alpha1.InstanceReference{Name: databasev1alpha1.ObjectName(name)}}
|
||||
if err := f.api.Create(t.Context(), tenant); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binder := &databasecontroller.BindingReconciler{Client: f.api, Reader: f.api}
|
||||
if _, err := binder.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(tenant)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.api.Get(t.Context(), client.ObjectKeyFromObject(tenant), tenant); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
database := &databasev1alpha1.PostgreSQLDatabase{}
|
||||
if err := f.api.Get(t.Context(), client.ObjectKey{Name: string(tenant.Status.DatabaseRef.Name)}, database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return database, tenant
|
||||
}
|
||||
|
||||
func (f *preparationFixture) service(t *testing.T) application.CredentialPreparation {
|
||||
t.Helper()
|
||||
return application.CredentialPreparation{
|
||||
Resources: &kubernetes.CredentialResources{Client: f.api, Reader: f.api}, Store: fixtureStore(t, f.bao),
|
||||
}
|
||||
}
|
||||
|
||||
func (f *preparationFixture) status(t *testing.T, database *databasev1alpha1.PostgreSQLDatabase, version int64, reason string) {
|
||||
t.Helper()
|
||||
if err := f.api.Get(t.Context(), client.ObjectKeyFromObject(database), database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
condition := meta.FindStatusCondition(database.Status.Conditions, application.CredentialsReady)
|
||||
if database.Status.CredentialVersion != version || condition == nil || condition.Reason != reason {
|
||||
t.Fatalf("凭据版本或条件不符:version=%d,期望 reason=%s", database.Status.CredentialVersion, reason)
|
||||
}
|
||||
if meta.IsStatusConditionTrue(database.Status.Conditions, "Ready") {
|
||||
t.Fatal("凭据准备不得宣告 Database Ready")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
//go:build integration
|
||||
|
||||
package openbao_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
"testing"
|
||||
|
||||
bao "github.com/openbao/openbao/api/v2"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
databasev1alpha1 "git.ddupan.top/panxiao81/ayatori/api/database/v1alpha1"
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/adapter/kubernetes"
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/adapter/openbao"
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/application"
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/domain/binding"
|
||||
)
|
||||
|
||||
func TestCredentialPreparationWithRealBackends(t *testing.T) {
|
||||
fixture := newPreparationFixture(t)
|
||||
t.Run("创建重启和幂等", func(t *testing.T) { testPreparationRestart(t, fixture) })
|
||||
t.Run("已有值不认领", func(t *testing.T) { testPreparationExisting(t, fixture) })
|
||||
t.Run("确认写入失败", func(t *testing.T) { testPreparationLostConfirmation(t, fixture) })
|
||||
t.Run("后端结果不确定", func(t *testing.T) { testPreparationUncertain(t, fixture) })
|
||||
t.Run("创建中绑定变化", func(t *testing.T) { testPreparationChangedBinding(t, fixture) })
|
||||
t.Run("依赖恢复与固定位置", func(t *testing.T) { testPreparationDependencies(t, fixture) })
|
||||
t.Run("并发用例只有一个写入", func(t *testing.T) { testPreparationConcurrency(t, fixture) })
|
||||
t.Run("位置被API裁剪时拒绝外部写入", func(t *testing.T) { testPreparationPruning(t, fixture) })
|
||||
t.Run("实际manager的watch与重启", func(t *testing.T) { testPreparationWatch(t, fixture) })
|
||||
}
|
||||
|
||||
func testPreparationRestart(t *testing.T, f *preparationFixture) {
|
||||
database, tenant := f.bound(t, "restart")
|
||||
if err := f.service(t).Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 1, application.CredentialPrepared)
|
||||
path := database.Status.CredentialRef.Path
|
||||
before, err := f.bao.KVv2("secret").Get(t.Context(), path)
|
||||
if err != nil {
|
||||
t.Fatal("无法读取隔离测试凭据")
|
||||
}
|
||||
if before.Data["username"] != "restart" || before.Data["database"] != "restart" || len(before.Data) != 7 {
|
||||
t.Fatal("用例未使用绑定目标生成七键凭据")
|
||||
}
|
||||
// 两次新用例实例模拟重启,第二轮必须没有无意义状态更新。
|
||||
for range 2 {
|
||||
if err := f.service(t).Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
f.status(t, database, 1, application.CredentialPrepared)
|
||||
revision := database.ResourceVersion
|
||||
if err := f.service(t).Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 1, application.CredentialPrepared)
|
||||
if database.ResourceVersion != revision {
|
||||
t.Fatal("幂等重试不应改写 status")
|
||||
}
|
||||
after, err := f.bao.KVv2("secret").Get(t.Context(), path)
|
||||
if err != nil || after.VersionMetadata.Version != 1 || !maps.Equal(before.Data, after.Data) {
|
||||
t.Fatal("重启后不应生成或覆盖凭据")
|
||||
}
|
||||
if err := f.api.Delete(t.Context(), tenant); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.service(t).Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 1, "PreparationStopped")
|
||||
if err := f.api.Delete(t.Context(), database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.service(t).Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 1, "PreparationStopped")
|
||||
if len(database.Finalizers) == 0 {
|
||||
t.Fatal("本切片不得解除删除保护")
|
||||
}
|
||||
if _, err := f.bao.KVv2("secret").Get(t.Context(), path); err != nil {
|
||||
t.Fatal("删除流程不得在本切片清理凭据")
|
||||
}
|
||||
}
|
||||
|
||||
func testPreparationExisting(t *testing.T, f *preparationFixture) {
|
||||
database, _ := f.bound(t, "existing")
|
||||
store := fixtureStore(t, f.bao)
|
||||
path, err := store.ProvisionPath(string(database.UID))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Create(t.Context(), path, fixtureCredential(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.service(t).Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 0, binding.Conflict)
|
||||
// 即使管理员删除了后端值,未解除的不确定状态也不能自动创建。
|
||||
if err := f.bao.KVv2("secret").DeleteMetadata(t.Context(), path); err != nil {
|
||||
t.Fatal("无法清理隔离测试 key")
|
||||
}
|
||||
if err := f.service(t).Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.bao.KVv2("secret").Get(t.Context(), path); !errors.Is(err, bao.ErrSecretNotFound) {
|
||||
t.Fatal("Conflict 重入不应生成替代凭据")
|
||||
}
|
||||
}
|
||||
|
||||
// 故障只注入确认保存边界;此前写入使用真实 API server 和 Bao。
|
||||
type failedConfirmationClient struct{ client.Client }
|
||||
|
||||
func (c failedConfirmationClient) Status() client.SubResourceWriter {
|
||||
return failedConfirmationWriter{SubResourceWriter: c.Client.Status()}
|
||||
}
|
||||
|
||||
type failedConfirmationWriter struct{ client.SubResourceWriter }
|
||||
|
||||
func (w failedConfirmationWriter) Update(ctx context.Context, object client.Object, options ...client.SubResourceUpdateOption) error {
|
||||
if database, ok := object.(*databasev1alpha1.PostgreSQLDatabase); ok && database.Status.CredentialVersion > 0 {
|
||||
return errors.New("fixture refuses confirmation update")
|
||||
}
|
||||
return w.SubResourceWriter.Update(ctx, object, options...)
|
||||
}
|
||||
|
||||
func testPreparationLostConfirmation(t *testing.T, f *preparationFixture) {
|
||||
database, _ := f.bound(t, "confirmation")
|
||||
service := f.service(t)
|
||||
service.Resources = &kubernetes.CredentialResources{Client: failedConfirmationClient{f.api}, Reader: f.api}
|
||||
if err := service.Reconcile(t.Context(), database.Name); err == nil {
|
||||
t.Fatal("确认写入失败应返回 API 错误")
|
||||
}
|
||||
f.status(t, database, 0, application.CredentialCreationStarted)
|
||||
if err := f.service(t).Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 0, binding.Conflict)
|
||||
stored, err := f.bao.KVv2("secret").Get(t.Context(), database.Status.CredentialRef.Path)
|
||||
if err != nil || stored.VersionMetadata.Version != 1 {
|
||||
t.Fatal("写入结果必须保留且不能被重启认领")
|
||||
}
|
||||
}
|
||||
|
||||
type afterCreateStore struct {
|
||||
application.CredentialStore
|
||||
after func() error
|
||||
}
|
||||
|
||||
func (s afterCreateStore) CreateCredential(ctx context.Context, location application.CredentialLocation, credential application.ApplicationCredential) (int64, error) {
|
||||
version, err := s.CredentialStore.CreateCredential(ctx, location, credential)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := s.after(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return version, nil
|
||||
}
|
||||
|
||||
func testPreparationUncertain(t *testing.T, f *preparationFixture) {
|
||||
database, _ := f.bound(t, "uncertain")
|
||||
service := f.service(t)
|
||||
service.Store = afterCreateStore{CredentialStore: service.Store, after: func() error { return application.ErrCredentialUncertain }}
|
||||
if err := service.Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 0, binding.Conflict)
|
||||
if err := f.service(t).Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 0, binding.Conflict)
|
||||
}
|
||||
|
||||
func testPreparationChangedBinding(t *testing.T, f *preparationFixture) {
|
||||
database, tenant := f.bound(t, "changed")
|
||||
service := f.service(t)
|
||||
service.Store = afterCreateStore{CredentialStore: service.Store, after: func() error {
|
||||
return f.api.Delete(t.Context(), tenant)
|
||||
}}
|
||||
if err := service.Reconcile(t.Context(), database.Name); err == nil {
|
||||
t.Fatal("中途删除 Tenant 后不得确认凭据")
|
||||
}
|
||||
f.status(t, database, 0, application.CredentialCreationStarted)
|
||||
if err := f.service(t).Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 0, binding.Conflict)
|
||||
}
|
||||
|
||||
func testPreparationDependencies(t *testing.T, f *preparationFixture) {
|
||||
database, _ := f.bound(t, "dependency")
|
||||
denied := fixtureClient(t, f.bao.Address())
|
||||
denied.ClearToken()
|
||||
service := f.service(t)
|
||||
service.Store = fixtureStore(t, denied)
|
||||
if err := service.Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 0, binding.DependencyUnavailable)
|
||||
if err := f.service(t).Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 1, application.CredentialPrepared)
|
||||
if err := service.Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 1, binding.DependencyUnavailable)
|
||||
if err := f.service(t).Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 1, application.CredentialPrepared)
|
||||
moved, err := openbao.NewCredentials(f.bao, "other", "elsewhere")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service.Store = moved
|
||||
if err := service.Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 1, binding.DependencyUnavailable)
|
||||
if database.Status.CredentialRef.Mount != "secret" {
|
||||
t.Fatal("配置变化不得迁移固定位置")
|
||||
}
|
||||
if err := f.bao.KVv2("secret").Delete(t.Context(), database.Status.CredentialRef.Path); err != nil {
|
||||
t.Fatal("无法准备测试软删除")
|
||||
}
|
||||
if err := f.service(t).Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 1, binding.Conflict)
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
//go:build integration
|
||||
|
||||
package openbao_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
bao "github.com/openbao/openbao/api/v2"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
databasev1alpha1 "git.ddupan.top/panxiao81/ayatori/api/database/v1alpha1"
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/adapter/kubernetes"
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/application"
|
||||
)
|
||||
|
||||
// 在 API 边界模拟旧 schema 裁剪位置字段;剩余写入仍由真实 API server 处理。
|
||||
type prunedLocationClient struct{ client.Client }
|
||||
|
||||
func (c prunedLocationClient) Status() client.SubResourceWriter {
|
||||
return prunedLocationWriter{SubResourceWriter: c.Client.Status()}
|
||||
}
|
||||
|
||||
type prunedLocationWriter struct{ client.SubResourceWriter }
|
||||
|
||||
func (w prunedLocationWriter) Update(ctx context.Context, object client.Object, options ...client.SubResourceUpdateOption) error {
|
||||
if database, ok := object.(*databasev1alpha1.PostgreSQLDatabase); ok {
|
||||
database.Status.CredentialRef = nil
|
||||
}
|
||||
return w.SubResourceWriter.Update(ctx, object, options...)
|
||||
}
|
||||
|
||||
func testPreparationPruning(t *testing.T, f *preparationFixture) {
|
||||
database, _ := f.bound(t, "pruning")
|
||||
service := f.service(t)
|
||||
service.Resources = &kubernetes.CredentialResources{Client: prunedLocationClient{f.api}, Reader: f.api}
|
||||
if err := service.Reconcile(t.Context(), database.Name); err == nil {
|
||||
t.Fatal("API 未保留位置时不得继续供应")
|
||||
}
|
||||
location, err := service.Store.ProvisionLocation(string(database.UID))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.bao.KVv2(location.Mount).Get(t.Context(), location.Path); !errors.Is(err, bao.ErrSecretNotFound) {
|
||||
t.Fatal("没有持久化位置时产生了外部凭据")
|
||||
}
|
||||
if err := f.service(t).Reconcile(t.Context(), database.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.status(t, database, 1, application.CredentialPrepared)
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
//go:build integration
|
||||
|
||||
package openbao_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
controllerconfig "sigs.k8s.io/controller-runtime/pkg/config"
|
||||
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
|
||||
|
||||
databasev1alpha1 "git.ddupan.top/panxiao81/ayatori/api/database/v1alpha1"
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/application"
|
||||
databasecontroller "git.ddupan.top/panxiao81/ayatori/internal/database/controller"
|
||||
)
|
||||
|
||||
func testPreparationWatch(t *testing.T, f *preparationFixture) {
|
||||
database, _ := f.bound(t, "watch")
|
||||
t.Cleanup(func() {
|
||||
if !t.Failed() {
|
||||
return
|
||||
}
|
||||
if err := f.api.Get(context.Background(), client.ObjectKeyFromObject(database), database); err == nil {
|
||||
if condition := meta.FindStatusCondition(database.Status.Conditions, application.CredentialsReady); condition != nil {
|
||||
t.Logf("失败时凭据条件: %s: %s", condition.Reason, condition.Message)
|
||||
}
|
||||
}
|
||||
})
|
||||
instance := &databasev1alpha1.PostgreSQLInstance{}
|
||||
if err := f.api.Get(t.Context(), client.ObjectKey{Name: "watch"}, instance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
setReady := func(ready metav1.ConditionStatus) {
|
||||
t.Helper()
|
||||
meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{
|
||||
Type: "Ready", Status: ready, Reason: "FixtureObservation", Message: "隔离测试前置观察", ObservedGeneration: instance.Generation,
|
||||
})
|
||||
if err := f.api.Status().Update(t.Context(), instance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
setReady(metav1.ConditionFalse)
|
||||
stop := startPreparationManager(t, f)
|
||||
waitForPreparation(t, func() bool {
|
||||
if err := f.api.Get(t.Context(), client.ObjectKeyFromObject(database), database); err != nil {
|
||||
return false
|
||||
}
|
||||
condition := meta.FindStatusCondition(database.Status.Conditions, application.CredentialsReady)
|
||||
return condition != nil && condition.Reason == "DependencyUnavailable"
|
||||
})
|
||||
if database.Status.CredentialRef != nil {
|
||||
t.Fatal("Instance 未 Ready 时不应固定或写入凭据")
|
||||
}
|
||||
setReady(metav1.ConditionTrue)
|
||||
waitForPreparation(t, func() bool {
|
||||
return f.api.Get(t.Context(), client.ObjectKeyFromObject(database), database) == nil && database.Status.CredentialVersion == 1
|
||||
})
|
||||
stop()
|
||||
// 重启真实 manager 后继续观察;不得只依赖旧进程中的记忆。
|
||||
setReady(metav1.ConditionFalse)
|
||||
stop = startPreparationManager(t, f)
|
||||
defer stop()
|
||||
waitForPreparation(t, func() bool {
|
||||
if err := f.api.Get(t.Context(), client.ObjectKeyFromObject(database), database); err != nil {
|
||||
return false
|
||||
}
|
||||
condition := meta.FindStatusCondition(database.Status.Conditions, application.CredentialsReady)
|
||||
return condition != nil && condition.Status == metav1.ConditionFalse && condition.Reason == "DependencyUnavailable"
|
||||
})
|
||||
setReady(metav1.ConditionTrue)
|
||||
waitForPreparation(t, func() bool {
|
||||
if err := f.api.Get(t.Context(), client.ObjectKeyFromObject(database), database); err != nil {
|
||||
return false
|
||||
}
|
||||
condition := meta.FindStatusCondition(database.Status.Conditions, application.CredentialsReady)
|
||||
return condition != nil && condition.Status == metav1.ConditionTrue && database.Status.CredentialVersion == 1
|
||||
})
|
||||
stored, err := f.bao.KVv2("secret").Get(t.Context(), database.Status.CredentialRef.Path)
|
||||
if err != nil || stored.VersionMetadata.Version != 1 {
|
||||
t.Fatal("manager 重启不得重建凭据")
|
||||
}
|
||||
}
|
||||
|
||||
func startPreparationManager(t *testing.T, f *preparationFixture) func() {
|
||||
t.Helper()
|
||||
manager, err := ctrl.NewManager(f.config, ctrl.Options{
|
||||
Scheme: f.scheme, Metrics: metricsserver.Options{BindAddress: "0"}, HealthProbeBindAddress: "0",
|
||||
// 测试在同一进程顺序重启 manager;旧 worker 已停止,但名称注册表仍是进程级。
|
||||
Controller: controllerconfig.Controller{SkipNameValidation: new(true)},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := (&databasecontroller.BindingReconciler{}).SetupWithManager(t.Context(), manager); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := (&databasecontroller.CredentialReconciler{Store: fixtureStore(t, f.bao)}).SetupWithManager(manager); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- manager.Start(ctx) }()
|
||||
stopped := false
|
||||
stop := func() {
|
||||
if stopped {
|
||||
return
|
||||
}
|
||||
stopped = true
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Error("凭据 manager 未停止")
|
||||
}
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
return stop
|
||||
}
|
||||
|
||||
func waitForPreparation(t *testing.T, check func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if check() {
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("凭据 watch 没有在低频重查之前驱动协调")
|
||||
}
|
||||
Reference in New Issue
Block a user