feat: 接入 Instance 原生管理观测与删除保护
This commit is contained in:
@@ -0,0 +1,149 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/domain/instance"
|
||||
)
|
||||
|
||||
const (
|
||||
instanceDependencyUnavailable = "DependencyUnavailable"
|
||||
instanceAuthenticationFailed = "AuthenticationFailed"
|
||||
)
|
||||
|
||||
// InstanceRecord 是 API 快照;Revision 仅用于持久化并发保护,不是领域版本。
|
||||
type InstanceRecord struct {
|
||||
Target instance.ObservationTarget
|
||||
Revision string
|
||||
Deleting bool
|
||||
}
|
||||
|
||||
type InstanceResources interface {
|
||||
LoadInstance(context.Context, string) (*InstanceRecord, error)
|
||||
ProtectInstance(context.Context, *InstanceRecord) (*InstanceRecord, error)
|
||||
// InstanceReferences 返回一个可定位的阻塞引用;空字符串表示没有引用。
|
||||
InstanceReferences(context.Context, string) (string, error)
|
||||
}
|
||||
|
||||
type InstanceObserver interface {
|
||||
ObserveManagement(context.Context, instance.ObservationTarget) (InstanceObservation, error)
|
||||
Forget(string)
|
||||
}
|
||||
|
||||
type InstanceResult struct {
|
||||
Record *InstanceRecord
|
||||
Snapshot instance.Snapshot
|
||||
Reason string
|
||||
Message string
|
||||
RemoveProtection bool
|
||||
}
|
||||
|
||||
// InstanceReconciliation 协调 API 保护、实时观察和领域判断,不拼装 Kubernetes status。
|
||||
type InstanceReconciliation struct {
|
||||
Resources InstanceResources
|
||||
Observer InstanceObserver
|
||||
}
|
||||
|
||||
func (s *InstanceReconciliation) Reconcile(ctx context.Context, name string) (InstanceResult, error) {
|
||||
record, err := s.Resources.LoadInstance(ctx, name)
|
||||
if err != nil {
|
||||
return InstanceResult{}, err
|
||||
}
|
||||
if record == nil {
|
||||
s.Observer.Forget(name)
|
||||
return InstanceResult{}, nil
|
||||
}
|
||||
if record.Deleting {
|
||||
return s.deleting(ctx, record)
|
||||
}
|
||||
record, err = s.Resources.ProtectInstance(ctx, record)
|
||||
if err != nil {
|
||||
return InstanceResult{}, err
|
||||
}
|
||||
// 每轮从无证据的领域对象开始;持久化 Ready 和连接存活不能替代本轮检查。
|
||||
aggregate, err := instance.Reconstitute(record.Target, instance.Snapshot{}, false)
|
||||
if err != nil {
|
||||
return InstanceResult{}, err
|
||||
}
|
||||
if err := aggregate.BeginValidation(); err != nil {
|
||||
return InstanceResult{}, err
|
||||
}
|
||||
observation, observationErr := s.Observer.ObserveManagement(ctx, record.Target)
|
||||
result := InstanceResult{Record: record}
|
||||
if observationErr != nil {
|
||||
result.Snapshot = aggregate.Snapshot()
|
||||
result.Snapshot.Readiness = instance.NotReady
|
||||
result.Snapshot.ObservedRevision = record.Target.Revision().Value()
|
||||
result.Reason, result.Message = observationFailure(observationErr)
|
||||
return result, nil
|
||||
}
|
||||
capabilities, err := observation.Capabilities()
|
||||
if err != nil {
|
||||
return InstanceResult{}, err
|
||||
}
|
||||
if err := aggregate.AssessManagement(capabilities); err != nil {
|
||||
return InstanceResult{}, err
|
||||
}
|
||||
result.Snapshot = aggregate.Snapshot()
|
||||
result.Snapshot.ReportedVersion = observation.Version()
|
||||
result.Reason, result.Message = managementResult(result.Snapshot.Failure)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *InstanceReconciliation) deleting(ctx context.Context, record *InstanceRecord) (InstanceResult, error) {
|
||||
name := record.Target.Identity().Name()
|
||||
s.Observer.Forget(name)
|
||||
aggregate, err := instance.Reconstitute(record.Target, instance.Snapshot{}, true)
|
||||
if err != nil {
|
||||
return InstanceResult{}, err
|
||||
}
|
||||
if err := aggregate.BeginDeletion(); err != nil {
|
||||
return InstanceResult{}, err
|
||||
}
|
||||
result := InstanceResult{Record: record, Snapshot: aggregate.Snapshot(), Reason: "Deleting"}
|
||||
reference, err := s.Resources.InstanceReferences(ctx, name)
|
||||
if err != nil {
|
||||
result.Reason = instanceDependencyUnavailable
|
||||
result.Message = "无法确认 Database/Tenant 引用已解除;保留 Instance 删除保护并重试"
|
||||
return result, nil
|
||||
}
|
||||
if reference != "" {
|
||||
result.Reason = "InstanceInUse"
|
||||
result.Message = "仍被 " + reference + " 引用;先处理该资源,不会级联删除外部数据库"
|
||||
return result, nil
|
||||
}
|
||||
result.Message = "引用已解除,仅移除登记保护;不删除 PostgreSQL 或凭据"
|
||||
result.RemoveProtection = true
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func observationFailure(err error) (string, string) {
|
||||
switch {
|
||||
case errors.Is(err, ErrAuthentication):
|
||||
return instanceAuthenticationFailed, "管理连接认证或 TLS 校验失败;检查管理 Secret 和 CA/证书配置"
|
||||
case errors.Is(err, ErrCredentialsInvalid):
|
||||
return "InvalidCredentials", "管理 Secret 的用户名或密码字段缺失;检查引用字段映射"
|
||||
case errors.Is(err, ErrCredentialsChanged):
|
||||
return "CredentialsChanged", "观察期间管理凭据变化,已丢弃结果并关闭旧连接;等待重新验证"
|
||||
case errors.Is(err, ErrCredentialsUnavailable):
|
||||
return instanceDependencyUnavailable, "无法读取管理 Secret;检查其是否存在及 controller namespace 内的读取权限"
|
||||
default:
|
||||
return instanceDependencyUnavailable, "管理连接或能力查询失败;检查 PostgreSQL 可达性、catalog 读取权限和超时"
|
||||
}
|
||||
}
|
||||
|
||||
func managementResult(failure instance.Failure) (string, string) {
|
||||
switch failure {
|
||||
case instance.NoFailure:
|
||||
return "ManagementReady", "当前管理能力检查通过;具体资源授权和扩展安装仍需执行时验证"
|
||||
case instance.InsufficientPrivileges:
|
||||
return "InsufficientPrivileges", "原生管理要求非 superuser 且具备 CREATEDB/CREATEROLE;不会自动修改账号权限"
|
||||
case instance.DependencyUnavailable:
|
||||
return instanceDependencyUnavailable, "当前 PostgreSQL 不可写或所需管理能力暂不可用"
|
||||
case instance.AuthenticationFailed:
|
||||
return instanceAuthenticationFailed, "当前管理能力检查未通过认证"
|
||||
default:
|
||||
return "ObservationIncomplete", "管理能力检查尚有缺项,不能仅凭 metadata 查询成功标记 Ready"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"git.ddupan.top/panxiao81/ayatori/internal/database/domain/instance"
|
||||
)
|
||||
|
||||
func TestInstanceFailurePresentation(t *testing.T) {
|
||||
cases := []struct {
|
||||
err error
|
||||
reason string
|
||||
}{
|
||||
{ErrAuthentication, "AuthenticationFailed"},
|
||||
{ErrCredentialsInvalid, "InvalidCredentials"},
|
||||
{ErrCredentialsChanged, "CredentialsChanged"},
|
||||
{ErrCredentialsUnavailable, instanceDependencyUnavailable},
|
||||
{context.DeadlineExceeded, instanceDependencyUnavailable},
|
||||
{errors.New("private backend detail"), instanceDependencyUnavailable},
|
||||
}
|
||||
for _, test := range cases {
|
||||
reason, message := observationFailure(test.err)
|
||||
if reason != test.reason || message == "" || message == test.err.Error() {
|
||||
t.Fatal("观察失败没有安全且可诊断的状态")
|
||||
}
|
||||
}
|
||||
for _, failure := range []instance.Failure{
|
||||
instance.NoFailure, instance.ObservationIncomplete, instance.DependencyUnavailable,
|
||||
instance.AuthenticationFailed, instance.InsufficientPrivileges,
|
||||
} {
|
||||
reason, message := managementResult(failure)
|
||||
if reason == "" || message == "" || (reason == "ManagementReady") != (failure == instance.NoFailure) {
|
||||
t.Fatal("领域能力判定与状态不一致")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataCannotEstablishManagementReadiness(t *testing.T) {
|
||||
source := &sourceStub{}
|
||||
source.credentials, _ = NewCredentials("test", serviceTestPassword)
|
||||
connector := &connectorStub{}
|
||||
service, err := NewInstanceService(source, connector)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer service.Close()
|
||||
target := serviceTarget(t, "uid", "postgres.test", "management", 1)
|
||||
if _, err := service.ObserveMetadata(t.Context(), target); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
connector.databases[0].metadata.Management = instance.ManagementChecks{
|
||||
Connection: instance.CheckPassed, Metadata: instance.CheckPassed,
|
||||
Roles: instance.CheckPassed, Databases: instance.CheckPassed,
|
||||
Grants: instance.CheckPassed, Extensions: instance.CheckPassed,
|
||||
}
|
||||
observation, err := service.ObserveMetadata(t.Context(), target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
capabilities, err := observation.Capabilities()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
aggregate, err := instance.Reconstitute(target, instance.Snapshot{}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := aggregate.BeginValidation(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := aggregate.AssessManagement(capabilities); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if aggregate.Snapshot().Failure != instance.ObservationIncomplete {
|
||||
t.Fatal("metadata 入口不应携带完整管理检查")
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,7 @@ var (
|
||||
// Metadata 只查询版本与可用扩展,不能产生领域 Ready。
|
||||
type Database interface {
|
||||
InspectMetadata(context.Context) (DatabaseMetadata, error)
|
||||
InspectManagement(context.Context) (DatabaseMetadata, error)
|
||||
Close()
|
||||
}
|
||||
|
||||
@@ -82,17 +83,26 @@ func (s *InstanceService) ObserveVersion(ctx context.Context, target instance.Ob
|
||||
|
||||
// ObserveMetadata 返回当前目标和凭据下的版本与扩展;任何失败均丢弃全部结果。
|
||||
// 调用者仍需使用 CR resourceVersion 保存前提防止 spec 并发修改;本方法不建立跨系统事务。
|
||||
func (s *InstanceService) ObserveMetadata(ctx context.Context, target instance.ObservationTarget) (MetadataObservation, error) {
|
||||
func (s *InstanceService) ObserveMetadata(ctx context.Context, target instance.ObservationTarget) (InstanceObservation, error) {
|
||||
return s.observe(ctx, target, false)
|
||||
}
|
||||
|
||||
// ObserveManagement 复用同一凭据刷新与回读边界,但每轮重新检查原生管理能力。
|
||||
func (s *InstanceService) ObserveManagement(ctx context.Context, target instance.ObservationTarget) (InstanceObservation, error) {
|
||||
return s.observe(ctx, target, true)
|
||||
}
|
||||
|
||||
func (s *InstanceService) observe(ctx context.Context, target instance.ObservationTarget, management bool) (InstanceObservation, error) {
|
||||
if err := target.Validate(); err != nil {
|
||||
return MetadataObservation{}, err
|
||||
return InstanceObservation{}, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
return MetadataObservation{}, ErrClosed
|
||||
return InstanceObservation{}, ErrClosed
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return MetadataObservation{}, err
|
||||
return InstanceObservation{}, err
|
||||
}
|
||||
|
||||
// 先读取有效凭据。读取失败时不得继续使用缓存中的旧连接。
|
||||
@@ -100,11 +110,11 @@ func (s *InstanceService) ObserveMetadata(ctx context.Context, target instance.O
|
||||
credentials, err := s.source.Read(ctx, target.Definition().AdminCredential())
|
||||
if err != nil {
|
||||
s.release(name)
|
||||
return MetadataObservation{}, credentialError(err)
|
||||
return InstanceObservation{}, credentialError(err)
|
||||
}
|
||||
if credentials.username == "" || credentials.password == "" {
|
||||
s.release(name)
|
||||
return MetadataObservation{}, ErrCredentialsInvalid
|
||||
return InstanceObservation{}, ErrCredentialsInvalid
|
||||
}
|
||||
|
||||
// 连接身份与有效值均未变化时复用 pgxpool;generation 本身不要求换池。
|
||||
@@ -117,7 +127,7 @@ func (s *InstanceService) ObserveMetadata(ctx context.Context, target instance.O
|
||||
if current == nil {
|
||||
database, err := s.connector.Connect(ctx, target.Definition().Endpoint(), credentials)
|
||||
if err != nil {
|
||||
return MetadataObservation{}, err
|
||||
return InstanceObservation{}, err
|
||||
}
|
||||
current = &entry{
|
||||
target: target,
|
||||
@@ -127,30 +137,38 @@ func (s *InstanceService) ObserveMetadata(ctx context.Context, target instance.O
|
||||
s.entries[name] = current
|
||||
}
|
||||
|
||||
metadata, err := current.database.InspectMetadata(ctx)
|
||||
var metadata DatabaseMetadata
|
||||
if management {
|
||||
metadata, err = current.database.InspectManagement(ctx)
|
||||
} else {
|
||||
metadata, err = current.database.InspectMetadata(ctx)
|
||||
// 即使 adapter 误填权限,也不能把只读 metadata 入口升级为 Ready。
|
||||
metadata.Management = instance.ManagementChecks{}
|
||||
}
|
||||
if err != nil {
|
||||
s.release(name)
|
||||
return MetadataObservation{}, err
|
||||
return InstanceObservation{}, err
|
||||
}
|
||||
if metadata.Version == "" {
|
||||
s.release(name)
|
||||
return MetadataObservation{}, ErrObservation
|
||||
return InstanceObservation{}, ErrObservation
|
||||
}
|
||||
|
||||
// 回读后再检查凭据,避免把轮换前取得的结果交给新凭据的调用链。
|
||||
latest, err := s.source.Read(ctx, target.Definition().AdminCredential())
|
||||
if err != nil {
|
||||
s.release(name)
|
||||
return MetadataObservation{}, credentialError(err)
|
||||
return InstanceObservation{}, credentialError(err)
|
||||
}
|
||||
if latest != credentials {
|
||||
s.release(name)
|
||||
return MetadataObservation{}, ErrCredentialsChanged
|
||||
return InstanceObservation{}, ErrCredentialsChanged
|
||||
}
|
||||
return MetadataObservation{
|
||||
return InstanceObservation{
|
||||
target: target,
|
||||
version: metadata.Version,
|
||||
extensions: instance.ObserveExtensionSupport(metadata.AvailableExtensions),
|
||||
management: metadata.Management,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -42,6 +42,10 @@ type databaseStub struct {
|
||||
metadata DatabaseMetadata
|
||||
}
|
||||
|
||||
func (d *databaseStub) InspectManagement(ctx context.Context) (DatabaseMetadata, error) {
|
||||
return d.InspectMetadata(ctx)
|
||||
}
|
||||
|
||||
func (d *databaseStub) InspectMetadata(context.Context) (DatabaseMetadata, error) {
|
||||
return d.metadata, d.err
|
||||
}
|
||||
|
||||
@@ -18,23 +18,31 @@ package application
|
||||
|
||||
import "git.ddupan.top/panxiao81/ayatori/internal/database/domain/instance"
|
||||
|
||||
// DatabaseMetadata 是一次只读查询的事实,不包含管理权限或完整就绪结论。
|
||||
// DatabaseMetadata 是一次只读查询的事实,可附带原生管理检查,但不包含就绪结论。
|
||||
// AvailableExtensions 是服务器提供的可用列表,不是已安装列表或安装授权。
|
||||
type DatabaseMetadata struct {
|
||||
Version string
|
||||
AvailableExtensions []string
|
||||
// Management 仅由 InspectManagement 填充;metadata 查询必须保持未观察。
|
||||
Management instance.ManagementChecks
|
||||
}
|
||||
|
||||
// MetadataObservation 只在查询成功且有效凭据再次核对一致后产生。
|
||||
// InstanceObservation 只在查询成功且有效凭据再次核对一致后产生。
|
||||
// target 绑定本次调用,而非连接最初创建时的 generation;零值表示没有观察。
|
||||
type MetadataObservation struct {
|
||||
type InstanceObservation struct {
|
||||
target instance.ObservationTarget
|
||||
version string
|
||||
extensions instance.ExtensionSupport
|
||||
management instance.ManagementChecks
|
||||
}
|
||||
|
||||
func (o MetadataObservation) Target() instance.ObservationTarget { return o.target }
|
||||
func (o MetadataObservation) Version() string { return o.version }
|
||||
func (o MetadataObservation) Extensions() instance.ExtensionSupport {
|
||||
// Capabilities 保留缺项为未观察;不能从 metadata 的成功补齐管理检查。
|
||||
func (o InstanceObservation) Capabilities() (instance.CapabilityObservation, error) {
|
||||
return instance.NewCapabilityObservation(o.target, o.version, o.management)
|
||||
}
|
||||
|
||||
func (o InstanceObservation) Target() instance.ObservationTarget { return o.target }
|
||||
func (o InstanceObservation) Version() string { return o.version }
|
||||
func (o InstanceObservation) Extensions() instance.ExtensionSupport {
|
||||
return o.extensions
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user