diff --git a/cmd/main.go b/cmd/main.go index 4144bd6..6215b1a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -42,6 +42,8 @@ func init() { // nolint:gocyclo func main() { + var openBao openBaoOptions + openBao.bindFlags(flag.CommandLine) var databaseNamespace, databaseRootCert string flag.StringVar(&databaseNamespace, "database-secret-namespace", os.Getenv("POD_NAMESPACE"), "固定管理 Secret namespace；为空时不启用 Instance 观测") @@ -179,6 +181,10 @@ func main() { } // +kubebuilder:scaffold:builder + if err := setupOpenBaoAuthentication(mgr, openBao); err != nil { + setupLog.Error(err, "Failed to set up OpenBao authentication") + os.Exit(1) + } var instanceService *application.InstanceService if databaseNamespace != "" { instanceService, err = setupInstanceObservation(mgr, databaseNamespace, databaseRootCert) diff --git a/cmd/openbao.go b/cmd/openbao.go new file mode 100644 index 0000000..360e771 --- /dev/null +++ b/cmd/openbao.go @@ -0,0 +1,95 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "flag" + "net/http" + "net/url" + "strings" + + bao "github.com/openbao/openbao/api/v2" + ctrl "sigs.k8s.io/controller-runtime" + + "git.ddupan.top/panxiao81/ayatori/internal/database/adapter/openbao" +) + +type openBaoOptions struct { + address string + caCert string + mount string + role string + identity openbao.KubernetesIdentity +} + +func (o *openBaoOptions) bindFlags(flags *flag.FlagSet) { + flags.StringVar(&o.address, "openbao-address", "", "OpenBao HTTPS 地址；为空时不启用认证会话") + flags.StringVar(&o.caCert, "openbao-ca-cert", "", "OpenBao 公开 CA PEM 路径；默认使用系统信任根") + flags.StringVar(&o.mount, "openbao-auth-mount", "kubernetes", "OpenBao Kubernetes auth mount") + flags.StringVar(&o.role, "openbao-auth-role", "", "OpenBao 登录 role") + flags.StringVar(&o.identity.Namespace, "openbao-service-account-namespace", "", + "TokenRequest 的固定 ServiceAccount namespace") + flags.StringVar(&o.identity.ServiceAccount, "openbao-service-account-name", "", "TokenRequest 的固定 ServiceAccount 名称") + flags.StringVar(&o.identity.Audience, "openbao-token-audience", "openbao", "SA JWT audience，须匹配 OpenBao role") +} + +func (o openBaoOptions) client() (*bao.Client, error) { + address, err := url.Parse(o.address) + if err != nil || address.Scheme != "https" || address.Host == "" || address.User != nil || + address.RawQuery != "" || address.ForceQuery || address.Fragment != "" || + (address.Path != "" && address.Path != "/") { + return nil, errors.New("OpenBao address must be an absolute HTTPS URL without credentials, query, fragment or path") + } + // NewConfig 不读取 BAO_TOKEN/BAO_SKIP_VERIFY 等环境配置，不允许旁路 Kubernetes 身份或 TLS。 + config := bao.NewConfig() + config.Address = strings.TrimSuffix(o.address, "/") + if config.Error != nil || config.ConfigureTLS(&bao.TLSConfig{CACert: o.caCert}) != nil { + return nil, errors.New("cannot configure OpenBao TLS trust") + } + client, err := bao.NewClient(config) + if err != nil { + return nil, errors.New("cannot construct OpenBao client") + } + return client, nil +} + +func setupOpenBaoAuthentication(manager ctrl.Manager, options openBaoOptions) error { + if options.address == "" { + return nil + } + client, err := options.client() + if err != nil { + return err + } + // 与 reconcile 共用 GetConfig：支持 --kubeconfig/KUBECONFIG 和 in-cluster，绝不推断 Pod 文件位置。 + session, err := openbao.NewKubernetesSession( + client, manager.GetConfig(), options.mount, options.role, options.identity, + ) + if err != nil { + return err + } + if err := manager.Add(session); err != nil { + return err + } + return manager.AddReadyzCheck("openbao-auth", func(_ *http.Request) error { + if !session.Ready() { + return errors.New("OpenBao Kubernetes authentication unavailable") + } + return nil + }) +} diff --git a/cmd/openbao_test.go b/cmd/openbao_test.go new file mode 100644 index 0000000..f307d10 --- /dev/null +++ b/cmd/openbao_test.go @@ -0,0 +1,67 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "net/http" + "net/http/httptest" + "testing" +) + +func TestOpenBaoClientDoesNotUseEnvironmentIdentityOrAddress(t *testing.T) { + t.Setenv("BAO_TOKEN", "TEST-ONLY-unwanted-static-token") + t.Setenv("BAO_ADDR", "http://unwanted.invalid") + t.Setenv("BAO_SKIP_VERIFY", "true") + var options openBaoOptions + options.bindFlags(flag.NewFlagSet("test", flag.ContinueOnError)) + options.address = "https://bao.example/" + client, err := options.client() + if err != nil { + t.Fatal(err) + } + if client.Address() != "https://bao.example" || client.Token() != "" { + t.Fatal("ambient environment replaced the explicit connection or identity") + } + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + options.address = server.URL + untrusted, err := options.client() + if err != nil { + t.Fatal(err) + } + untrusted.SetMaxRetries(0) + if _, err := untrusted.Sys().HealthWithContext(t.Context()); err == nil { + t.Fatal("BAO_SKIP_VERIFY bypassed TLS validation") + } +} + +func TestOpenBaoClientRejectsUnsafeConfiguration(t *testing.T) { + for _, address := range []string{ + "", "http://bao.example", "https://user:secret@bao.example", + "https://bao.example/?token=secret", "https://bao.example/#secret", "https://bao.example/path", + } { + if _, err := (openBaoOptions{address: address}).client(); err == nil { + t.Fatal("accepted unsafe OpenBao address") + } + } + if _, err := (openBaoOptions{address: "https://bao.example", caCert: "/nonexistent/fixture-ca"}).client(); err == nil { + t.Fatal("accepted missing explicit CA") + } +} diff --git a/config/samples/database_openbao_auth_rbac.yaml b/config/samples/database_openbao_auth_rbac.yaml new file mode 100644 index 0000000..c10e13a --- /dev/null +++ b/config/samples/database_openbao_auth_rbac.yaml @@ -0,0 +1,37 @@ +# 示例：只授权 controller 为固定登录 SA 创建短期 JWT；由管理员替换 namespace/subject 后应用。 +# 不自动纳入 config/default，不包含 kubeconfig、长期 token 或 OpenBao 管理权限。 +apiVersion: v1 +kind: ServiceAccount +metadata: + name: database-openbao-login + namespace: ayatori-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: database-openbao-token + namespace: ayatori-system +rules: +- apiGroups: [""] + resources: [serviceaccounts/token] + resourceNames: [database-openbao-login] + verbs: [create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: database-openbao-token + namespace: ayatori-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: database-openbao-token +subjects: +# 集群外示例：对应管理员签发 kubeconfig 的实际用户名，不是登录目标 SA 的名字。 +- kind: User + apiGroup: rbac.authorization.k8s.io + name: ayatori-controller +# 集群内可以改为 manager 自己的 ServiceAccount；不要求两个 subject 同时授权。 +# - kind: ServiceAccount +# name: controller-manager +# namespace: ayatori-system diff --git a/docs/database/README.md b/docs/database/README.md index 5c5c9d6..b384397 100644 --- a/docs/database/README.md +++ b/docs/database/README.md @@ -168,33 +168,42 @@ Instance 删除首先释放本地连接并撤销 Ready。任何引用它的 Data 真实后端覆盖创建/回读、并发唯一创建、重建适配器读取、软删除冲突、固定前缀 token 拒绝管理路径，以及成功写入后丢失响应；HTTP 故障测试补充不重试和错误脱敏。 -凭据存储与下面的认证会话尚未接入 manager；Database 状态中的稳定位置和已确认步骤、 -供应 service/controller、PostgreSQL 创建以及 ESO 交付仍未完成。 +认证会话已按下面的显式参数接入 manager；凭据存储尚未接入供应用例。 +Database 状态中的稳定位置和已确认步骤、供应 service/controller、PostgreSQL 创建以及 ESO 交付仍未完成。 测试 token 只用于临时 fixture，不是生产静态 token 配置接口。现有绑定不会触发外部写入。 ## OpenBao Kubernetes 认证会话 `adapter/openbao.KubernetesSession` 复用官方 Kubernetes auth helper 和 `LifetimeWatcher` -（均为 v2.7.0）。调用方提供专用 SDK client、固定 auth mount/role 与绝对 SA token 文件路径； -构造时清空既有 token，不使用静态 token 作为失败后的回退身份。每次重新登录读取指定文件， -拒绝缺失或空文件，避免 SDK 的空 token 默认路径回退。登录结果必须包含有效 token 和有限 TTL。 +（均为 v2.7.0）。认证与 reconcile 共用 manager 的 Kubernetes 配置：标准 `--kubeconfig` / +`KUBECONFIG` 支持 systemd 或其他集群外运行方式，集群内使用 in-cluster 配置，不要求存在 Pod。 +Kubernetes 身份的签发和更新由部署管理及 client-go 的认证机制负责，不另建 kubeconfig 读取器。 + +每次登录前，使用该身份调用固定 namespace/name 的 ServiceAccount TokenRequest，申请 +audience 匹配 OpenBao role、期望有效期 600 秒的短期 JWT；检查返回值非空且未过期，再交给 +官方 Kubernetes auth helper。JWT 不缓存，不读取投射文件，也不回退静态 OpenBao token； +实际 JWT 有效期由 API server 决定。RBAC 拒绝或 TokenRequest 失败时不会继续 Bao 登录。 +OpenBao 登录结果必须包含有效 token 和有限 TTL。 续期、到期阈值与等待时间由 SDK 管理；可续期 token 的续期失败就撤下本地 token，不可续期 -token 由 SDK 监测剩余寿命。会话结束后最多每 5 秒重新登录一次，读取新的投射内容。 +token 由 SDK 监测剩余寿命。会话结束后最多每 5 秒重新登录一次，并重新申请 Kubernetes JWT。 `Ready()` 仅表示当前 lease 正受 SDK 管理，不授权任何 Database 写入，也不能保证下一次请求 必然成功。认证/续期响应不写日志、不返回给调用方；后端操作仍独立检查并返回脱敏错误。 `Start(ctx)` 退出时清空 client token 并等待续期 goroutine 结束。SDK Stop 不取消已经发出的 续期 HTTP 请求，因此专用 client 的请求期限固定为 15 秒；不增加新连接池或自己的续期算法。 -同一会话拒绝并发 Start。当前还未添加 manager flags、Runnable/健康检查装配或生产 auth 配置， -不能把适配器验收当成已部署的认证入口。 +同一会话拒绝并发 Start。manager 使用 Runnable 管理生命周期，并增加 `openbao-auth` readiness +检查；认证故障不影响 liveness。`--openbao-address` 为空时不启用，不自动修改生产 auth/RBAC。 +HTTPS 和显式 CA/系统信任根不可通过 BAO 环境变量降级，参数见 [部署合同](deployment.md)。 依据官方 [Kubernetes auth](https://openbao.org/docs/auth/kubernetes/) 与 [token 生命周期](https://openbao.org/docs/concepts/auth/)。真实测试使用 envtest 签发 SA token， -OpenBao 通过专用 reviewer 调用真实 TokenReview；验证短 TTL 续期、撤销后的清空/重新登录、 -错误 namespace/audience 拒绝以及凭据访问恢复。临时 TokenReview 入口仅允许对应 POST， +OpenBao 通过专用 reviewer 调用真实 TokenReview；集群外受限 kubeconfig 启动实际 manager， +验证同一配置申请 JWT、短 TTL 续期、RBAC 撤回/恢复与重新登录、跨 namespace/其他 SA 拒绝、 +错误 OpenBao audience 拒绝以及凭据访问恢复。临时 TokenReview 入口仅允许对应 POST， 两段连接均验证 TLS；其 Docker bridge 入口仅为隔离测试，不修改生产 OpenBao 或 Kubernetes。 -单元测试补充缺失/空文件无回退、投射 token 更新、无期限 lease 拒绝、并发生命周期与退出清理。 +单元测试补充 TokenRequest 失败/空 token 无回退、重新申请 JWT、无期限 lease 拒绝、 +并发生命周期、退出清理及 manager 显式参数不受 BAO 环境身份覆盖。 ## 设计入口 diff --git a/docs/database/deployment.md b/docs/database/deployment.md index 470dae5..83c1aa8 100644 --- a/docs/database/deployment.md +++ b/docs/database/deployment.md @@ -10,7 +10,7 @@ | 最后更新 | 2026-09-25 | 本文定义 v1alpha1 的运行依赖、启动顺序和部署级配置。Instance 观测已接入 manager； -OpenBao、ESO 与完整供应装配仍是后续实现合同。 +OpenBao 认证可显式启用；ESO 与完整供应装配仍是后续实现合同。 ## 依赖与顺序 @@ -34,18 +34,21 @@ OpenBao、ESO 与完整供应装配仍是后续实现合同。 Instance 观测）与 `--database-root-cert`（公开 PostgreSQL CA PEM 路径）。Deployment 通过 downward API 获取 namespace，Secret 权限由该 namespace 的 Role 授予。 -以下是尚待实现的供应/交付配置合同，不表示当前 manager 接受这些 CLI flags。 +以下表格区分已实现的认证参数与尚待实现的供应/交付参数。 必填项缺失、路径无效或 duration 不为正数时，进程必须在启动 manager 前失败； 不得等到 reconcile 时才逐个资源报告配置错误。 | CLI flag | 必填/默认 | 说明 | | --- | --- | --- | -| `--openbao-address` | 必填 | controller 可访问的 OpenBao API address | +| `--openbao-address` | 已实现，默认空 | HTTPS API 地址；为空时关闭认证会话 | | `--openbao-consumer-address` | 默认同 `--openbao-address` | 写入 Tenant status，必须能被预期外部消费者解析 | -| `--openbao-auth-mount` | `kubernetes` | Kubernetes auth mount 名称 | -| `--openbao-auth-role` | 必填 | controller ServiceAccount 对应 role | +| `--openbao-auth-mount` | 已实现，`kubernetes` | Kubernetes auth mount 名称 | +| `--openbao-auth-role` | 已实现，启用时必填 | OpenBao 登录 role | +| `--openbao-ca-cert` | 已实现，默认系统信任根 | OpenBao 公开 CA PEM 路径 | | `--openbao-kv-mount` | `kv` | KV v2 mount；开发可显式用 `secret` | -| `--openbao-service-account-token-path` | `/var/run/secrets/kubernetes.io/serviceaccount/token` | Kubernetes auth 使用的投射 token 文件 | +| `--openbao-service-account-namespace` | 已实现，启用时必填 | TokenRequest 目标 SA 的固定 namespace | +| `--openbao-service-account-name` | 已实现，启用时必填 | TokenRequest 目标 SA 名称 | +| `--openbao-token-audience` | 已实现，`openbao` | SA JWT audience，必须匹配 OpenBao role | | `--openbao-tenant-base-path` | 默认 `postgresql-tenants` | controller 专属 mount-relative 前缀 | | `--external-secret-store-name` | 必填 | controller 创建的 ExternalSecret 固定引用 | | `--database-root-cert` | 已实现 | 只读 PEM trust bundle，不含私钥；沿用 Instance 连接配置 | @@ -56,6 +59,35 @@ address 必须是绝对 `http` 或 `https` URL，不允许 userinfo、query 或 `/` 开头，不含空段、`.` 或 `..`；base path 还不得编码 KV v2 的 `data`/`metadata` API 层。生产环境的 `--openbao-address` 必须使用 HTTPS；HTTP 只用于明确的开发 fixture。 +### 集群内与 systemd 共用 Kubernetes 认证 + +controller 是 API 客户端，不要求部署为 Pod。OpenBao 认证直接复用 manager 已加载的 +Kubernetes 配置：集群外可用标准 `--kubeconfig`（或 `KUBECONFIG`），集群内可使用 +in-cluster 配置。禁止另要求 `/var/run/secrets/.../token` 文件或解析 kubeconfig 中的 bearer token。 +每次 Bao 登录前通过 TokenRequest 申请新的短期 SA JWT，Kubernetes JWT 与 Bao token 的 +生命周期分别由 API 签发和官方 SDK 续期管理；不把 kubeconfig 本身当成永久有效凭据。 + +管理员为 controller 的实际 Kubernetes 身份授予目标 namespace 内 +`create serviceaccounts/token`，用 `resourceNames` 限定目标 SA；示例见 +[最小 RBAC](../../config/samples/database_openbao_auth_rbac.yaml)。集群外 RoleBinding subject +对应 kubeconfig 的用户/组，集群内可绑定 manager SA；登录目标 SA 可以独立于调用者身份。 +controller 不创建 SA、Role/RoleBinding，也不向自己授予权限。不要求 `get secrets` 来获取 JWT。 +OpenBao role 还需限制 SA 名称、namespace 与 audience，TokenReview reviewer 身份由管理员配置。 + +示例启动参数（仅示意，不包含真实 kubeconfig 或凭据）： + +```sh +manager --kubeconfig=/etc/ayatori/controller.kubeconfig \ + --database-secret-namespace=ayatori-system \ + --openbao-address=https://bao.example:8200 \ + --openbao-auth-role=ayatori-database \ + --openbao-service-account-namespace=ayatori-system \ + --openbao-service-account-name=database-openbao-login +``` + +这里的认证成功只开放 manager readiness，不代表 Database 已具备供应或交付能力。 +实例管理 Secret 的 namespace 同样由参数指定，systemd 模式不依赖 `POD_NAMESPACE` 环境变量。 + Tenant 不能选择任意凭据路径。凭据必须能随 Database 保留并安全交付给被授权的新 Tenant； 原 `//` 定位规则不再直接作为新 API 合同。 动态供应位置使用 `/`；导入使用 Database 的显式 credentialRef， diff --git a/docs/database/security.md b/docs/database/security.md index 05ebbe0..ea4b79a 100644 --- a/docs/database/security.md +++ b/docs/database/security.md @@ -24,6 +24,9 @@ Kubernetes 管理员、OpenBao 管理员和 PostgreSQL 管理员是平台信任 ## 凭据处理 - controller 使用 Kubernetes auth 获取短期 OpenBao token，不配置长期静态 token。 +- Kubernetes auth 不等于部署在 Kubernetes 内：复用 manager 的 kubeconfig/in-cluster 身份， + 通过最小 RBAC 的指定 ServiceAccount TokenRequest 获取 JWT，不依赖 Pod 投射文件。 + kubeconfig 的签发、更新与撤销由部署管理负责；申请失败不回退其他机器身份。 - 管理凭据只从 Instance 引用的 controller namespace Secret 读取，不复制到 CR/status/Event/metric/trace；管理员维护 ExternalSecret，由 ESO 同步该 Secret。 - 动态供应密码使用密码学安全随机源；已有可靠关联时复用 OpenBao 现值，结果不确定时停止并报冲突。 diff --git a/internal/database/adapter/openbao/authentication.go b/internal/database/adapter/openbao/authentication.go index 40067d4..7b9a0ed 100644 --- a/internal/database/adapter/openbao/authentication.go +++ b/internal/database/adapter/openbao/authentication.go @@ -19,8 +19,6 @@ package openbao import ( "context" "errors" - "os" - "path/filepath" "strings" "sync" "sync/atomic" @@ -28,30 +26,52 @@ import ( kubernetesauth "github.com/openbao/openbao/api/auth/kubernetes/v2" bao "github.com/openbao/openbao/api/v2" + authenticationv1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" ) var ErrAuthenticationConfiguration = errors.New("invalid OpenBao Kubernetes authentication configuration") // KubernetesSession 为专用 SDK client 维护短期登录，不持久化或对外返回 token。 -// 登录失败或 lease 不再可用时清空 client token，依赖恢复后重新读取投射的 SA token。 +// 每次登录通过 manager 的 Kubernetes 身份申请新的 SA JWT，不依赖 controller 的部署位置。 // 续期调度由官方 LifetimeWatcher 负责，不实现自己的 lease 算法。 type KubernetesSession struct { - client *bao.Client - mount string - role string - tokenPath string - running sync.Mutex - ready atomic.Bool + client *bao.Client + mount string + role string + accounts corev1.ServiceAccountInterface + identity KubernetesIdentity + running sync.Mutex + ready atomic.Bool } -func NewKubernetesSession(client *bao.Client, mount, role, tokenPath string) (*KubernetesSession, error) { - if client == nil || !validPath(mount) || !pathSegment.MatchString(role) || !filepath.IsAbs(tokenPath) { +// KubernetesIdentity 是部署固定的登录目标，不由 Tenant 选择。 +type KubernetesIdentity struct { + Namespace string + ServiceAccount string + Audience string +} + +func NewKubernetesSession(client *bao.Client, config *rest.Config, mount, role string, identity KubernetesIdentity) (*KubernetesSession, error) { + if client == nil || config == nil || !validPath(mount) || !pathSegment.MatchString(role) || + len(validation.IsDNS1123Label(identity.Namespace)) != 0 || + len(validation.IsDNS1123Subdomain(identity.ServiceAccount)) != 0 || strings.TrimSpace(identity.Audience) == "" { + return nil, ErrAuthenticationConfiguration + } + api, err := corev1.NewForConfig(config) + if err != nil { return nil, ErrAuthenticationConfiguration } client.ClearToken() client.SetMaxRetries(0) client.SetClientTimeout(15 * time.Second) - return &KubernetesSession{client: client, mount: mount, role: role, tokenPath: tokenPath}, nil + return &KubernetesSession{ + client: client, mount: mount, role: role, + accounts: api.ServiceAccounts(identity.Namespace), identity: identity, + }, nil } // Ready 仅代表当前登录 lease 正由 SDK 管理，不保证下一次后端请求一定成功。 @@ -88,15 +108,25 @@ func (s *KubernetesSession) clear() { } func (s *KubernetesSession) login(ctx context.Context) *bao.Secret { - // 只读取指定投射位置；SDK 的空 token 选项会回退默认路径，必须在此拒绝空文件。 - data, err := os.ReadFile(s.tokenPath) - if err != nil || strings.TrimSpace(string(data)) == "" { + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + // JWT 只用于本次登录，不缓存或自行解析 kubeconfig 中的凭据。 + // client-go 负责 kubeconfig/in-cluster 身份与凭据更新；API server 按 RBAC 签发。 + expirationSeconds := int64(600) + token, err := s.accounts.CreateToken(ctx, s.identity.ServiceAccount, &authenticationv1.TokenRequest{ + Spec: authenticationv1.TokenRequestSpec{ + Audiences: []string{s.identity.Audience}, + ExpirationSeconds: &expirationSeconds, + }, + }, metav1.CreateOptions{}) + if err != nil || token == nil || strings.TrimSpace(token.Status.Token) == "" || + !token.Status.ExpirationTimestamp.After(time.Now()) { return nil } // helper 会缓存 token，不能跨登录轮次复用。 method, err := kubernetesauth.NewKubernetesAuth(s.role, kubernetesauth.WithMountPath(s.mount), - kubernetesauth.WithServiceAccountToken(strings.TrimSpace(string(data))), + kubernetesauth.WithServiceAccountToken(token.Status.Token), ) if err != nil { return nil diff --git a/internal/database/adapter/openbao/authentication_integration_test.go b/internal/database/adapter/openbao/authentication_integration_test.go index d969c26..bd40390 100644 --- a/internal/database/adapter/openbao/authentication_integration_test.go +++ b/internal/database/adapter/openbao/authentication_integration_test.go @@ -33,7 +33,6 @@ import ( "net/http/httputil" "net/url" "os/exec" - "path/filepath" "strconv" "strings" "testing" @@ -43,17 +42,23 @@ import ( authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/envtest" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "git.ddupan.top/panxiao81/ayatori/internal/database/adapter/openbao" ) const ( - authNamespace = "bao-controller" - authAudience = "openbao" + unrelatedAuthNamespace = "bao-unrelated" + tokenRequestRole = "request-openbao-token" + authNamespace = "bao-controller" + authAudience = "openbao" ) // Bao 容器通过 Docker bridge 访问这个仅转发 TokenReview 的临时入口。 @@ -127,7 +132,17 @@ func tokenReviewCertificate(t *testing.T, address net.IP) (tls.Certificate, stri return certificate, string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) } -func TestKubernetesSessionWithRealTokenReview(t *testing.T) { +type kubernetesAuthFixture struct { + config *rest.Config + controllerConfig *rest.Config + admin *kubernetes.Clientset + controller *kubernetes.Clientset + grant func() + requestToken func(string, []string) string +} + +func newKubernetesAuthFixture(t *testing.T) *kubernetesAuthFixture { + t.Helper() environment := &envtest.Environment{} config, err := environment.Start() if err != nil { @@ -143,7 +158,7 @@ func TestKubernetesSessionWithRealTokenReview(t *testing.T) { t.Fatal("cannot construct fixture API client") } ctx := t.Context() - for _, namespace := range []string{authNamespace, "bao-unrelated"} { + for _, namespace := range []string{authNamespace, unrelatedAuthNamespace} { if _, err := clientset.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{Name: namespace}, metav1.CreateOptions{}); err != nil { t.Fatal("cannot create fixture namespace") } @@ -151,9 +166,52 @@ func TestKubernetesSessionWithRealTokenReview(t *testing.T) { t.Fatal("cannot create fixture service account") } } + // 集群外身份只有指定 SA 的 TokenRequest 权限，不使用 envtest 管理员凭据运行会话。 + user, err := environment.AddUser(envtest.User{Name: "systemd-controller"}, config) + if err != nil { + t.Fatal("cannot create external controller identity") + } + kubeconfig, err := user.KubeConfig() + if err != nil { + t.Fatal("cannot build external kubeconfig") + } + externalConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig) + if err != nil { + t.Fatal("cannot load external kubeconfig") + } + if _, err := clientset.RbacV1().Roles(authNamespace).Create(ctx, &rbacv1.Role{ + Name: tokenRequestRole, + Rules: []rbacv1.PolicyRule{{APIGroups: []string{""}, Resources: []string{"serviceaccounts/token"}, + ResourceNames: []string{authRole}, Verbs: []string{"create"}}}, + }, metav1.CreateOptions{}); err != nil { + t.Fatal("cannot create TokenRequest Role") + } + permission := &rbacv1.RoleBinding{ + Name: tokenRequestRole, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: tokenRequestRole}, + Subjects: []rbacv1.Subject{{Kind: "User", APIGroup: rbacv1.GroupName, Name: "systemd-controller"}}, + } + grant := func() { + t.Helper() + if _, err := clientset.RbacV1().RoleBindings(authNamespace).Create(ctx, permission.DeepCopy(), metav1.CreateOptions{}); err != nil { + t.Fatal("cannot grant TokenRequest permission") + } + } + grant() + externalClient, err := kubernetes.NewForConfig(externalConfig) + if err != nil { + t.Fatal("cannot construct restricted controller client") + } + for _, target := range []struct{ namespace, name string }{{unrelatedAuthNamespace, authRole}, {authNamespace, "another-account"}} { + _, err := externalClient.CoreV1().ServiceAccounts(target.namespace).CreateToken(ctx, target.name, + &authenticationv1.TokenRequest{Spec: authenticationv1.TokenRequestSpec{Audiences: []string{authAudience}}}, metav1.CreateOptions{}) + if !apierrors.IsForbidden(err) { + t.Fatal("TokenRequest escaped Role namespace/resourceNames restriction") + } + } if _, err := clientset.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{ Name: "bao-fixture-reviewer", - RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: "system:auth-delegator"}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: "system:auth-delegator"}, Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Namespace: authNamespace, Name: authRole}}, }, metav1.CreateOptions{}); err != nil { t.Fatal("cannot authorize fixture TokenReview") @@ -167,12 +225,22 @@ func TestKubernetesSessionWithRealTokenReview(t *testing.T) { } return response.Status.Token } + + return &kubernetesAuthFixture{ + config: config, controllerConfig: externalConfig, admin: clientset, controller: externalClient, + grant: grant, requestToken: requestToken, + } +} + +func TestKubernetesSessionWithRealTokenReview(t *testing.T) { + api := newKubernetesAuthFixture(t) + ctx := t.Context() root := baoFixture(t) if err := root.Sys().EnableAuthWithOptionsWithContext(ctx, "kubernetes", &bao.EnableAuthOptions{Type: "kubernetes"}); err != nil { t.Fatal("cannot enable fixture Kubernetes auth") } - reviewerToken := requestToken(authNamespace, nil) - reviewURL, reviewCA := tokenReviewEndpoint(t, config) + reviewerToken := api.requestToken(authNamespace, nil) + reviewURL, reviewCA := tokenReviewEndpoint(t, api.config) if _, err := root.Logical().WriteWithContext(ctx, "auth/kubernetes/config", map[string]any{ "kubernetes_host": reviewURL, "kubernetes_ca_cert": reviewCA, @@ -189,18 +257,23 @@ func TestKubernetesSessionWithRealTokenReview(t *testing.T) { "bound_service_account_namespaces": []string{authNamespace}, "audience": authAudience, "token_policies": []string{authRole}, - "token_ttl": "3s", "token_max_ttl": "15s", + "token_ttl": "3s", "token_max_ttl": "60s", }); err != nil { t.Fatal("cannot configure fixture auth role") } - path := filepath.Join(t.TempDir(), "token") - writeAuthToken(t, path, requestToken(authNamespace, []string{authAudience})) client := fixtureClient(t, root.Address()) - session, err := openbao.NewKubernetesSession(client, "kubernetes", authRole, path) + manager, err := ctrl.NewManager(api.controllerConfig, ctrl.Options{Metrics: metricsserver.Options{BindAddress: "0"}, HealthProbeBindAddress: "0"}) + if err != nil { + t.Fatal("cannot construct external controller manager") + } + session, err := openbao.NewKubernetesSession(client, manager.GetConfig(), "kubernetes", authRole, testIdentity) if err != nil { t.Fatal(err) } - cancel := startSession(t, session) + if err := manager.Add(session); err != nil { + t.Fatal("cannot register authentication lifecycle") + } + cancel := startSession(t, manager) waitForAuthentication(t, session.Ready) store := fixtureStore(t, client) if err := store.Create(ctx, credentialPath, fixtureCredential(t)); err != nil { @@ -220,24 +293,34 @@ func TestKubernetesSessionWithRealTokenReview(t *testing.T) { namespace string audience string }{ - {"bao-unrelated", authAudience}, {authNamespace, "wrong-audience"}, + {unrelatedAuthNamespace, authAudience}, {authNamespace, "wrong-audience"}, } { - writeAuthToken(t, path, requestToken(invalid.namespace, []string{invalid.audience})) - if err := root.Auth().Token().RevokeOrphanWithContext(ctx, client.Token()); err != nil { - t.Fatal("cannot revoke fixture token") - } - waitForAuthentication(t, func() bool { return !session.Ready() && client.Token() == "" }) // 用相同真实登录入口直接确认 namespace/audience 拒绝，不依赖定时轮询推断。 if _, err := root.Logical().WriteWithContext(ctx, "auth/kubernetes/login", map[string]any{ - "role": authRole, "jwt": requestToken(invalid.namespace, []string{invalid.audience}), + "role": authRole, "jwt": api.requestToken(invalid.namespace, []string{invalid.audience}), }); err == nil { t.Fatal("invalid Kubernetes identity was accepted") } - writeAuthToken(t, path, requestToken(authNamespace, []string{authAudience})) - waitForAuthentication(t, session.Ready) - if _, err := store.Read(ctx, credentialPath); err != nil { - t.Fatal("credential access did not recover", err) - } + } + if err := api.admin.RbacV1().RoleBindings(authNamespace).Delete(ctx, tokenRequestRole, metav1.DeleteOptions{}); err != nil { + t.Fatal("cannot revoke TokenRequest permission") + } + if err := root.Auth().Token().RevokeOrphanWithContext(ctx, client.Token()); err != nil { + t.Fatal("cannot revoke fixture OpenBao token") + } + waitForAuthentication(t, func() bool { return !session.Ready() && client.Token() == "" }) + denied, err := api.controller.CoreV1().ServiceAccounts(authNamespace).CreateToken(ctx, authRole, + &authenticationv1.TokenRequest{Spec: authenticationv1.TokenRequestSpec{Audiences: []string{authAudience}}}, metav1.CreateOptions{}) + if !apierrors.IsForbidden(err) || (denied != nil && denied.Status.Token != "") { + t.Fatal("revoked caller still obtained a token") + } + api.grant() + waitForAuthentication(t, session.Ready) + if client.Token() == initialToken { + t.Fatal("reauthentication reused revoked OpenBao token") + } + if _, err := store.Read(ctx, credentialPath); err != nil { + t.Fatal("credential access did not recover", err) } cancel() waitForAuthentication(t, func() bool { return !session.Ready() && client.Token() == "" }) diff --git a/internal/database/adapter/openbao/authentication_test.go b/internal/database/adapter/openbao/authentication_test.go index 1d9ae5e..6c2bec5 100644 --- a/internal/database/adapter/openbao/authentication_test.go +++ b/internal/database/adapter/openbao/authentication_test.go @@ -21,12 +21,14 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "os" - "path/filepath" "sync/atomic" "testing" "time" + authenticationv1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "git.ddupan.top/panxiao81/ayatori/internal/database/adapter/openbao" ) @@ -34,27 +36,52 @@ const ( authRole = "controller" ) -func writeAuthToken(t *testing.T, path, token string) { +var testIdentity = openbao.KubernetesIdentity{Namespace: "bao-controller", ServiceAccount: authRole, Audience: "openbao"} + +func authenticationConfig(address string) *rest.Config { + return &rest.Config{Host: address, ContentType: "application/json"} +} + +func authenticationServer(t *testing.T, token func() (string, bool), login http.HandlerFunc) *httptest.Server { t.Helper() - if err := os.WriteFile(path, []byte(token), 0600); err != nil { - t.Fatal("cannot write temporary projected token") - } + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/namespaces/bao-controller/serviceaccounts/controller/token" { + login(w, r) + return + } + var request authenticationv1.TokenRequest + if json.NewDecoder(r.Body).Decode(&request) != nil || r.Method != http.MethodPost || + len(request.Spec.Audiences) != 1 || request.Spec.Audiences[0] != testIdentity.Audience || + request.Spec.ExpirationSeconds == nil || *request.Spec.ExpirationSeconds != 600 { + t.Error("unexpected TokenRequest target or lifetime") + w.WriteHeader(http.StatusBadRequest) + return + } + jwt, allowed := token() + if !allowed { + w.WriteHeader(http.StatusForbidden) + return + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(authenticationv1.TokenRequest{ + APIVersion: "authentication.k8s.io/v1", Kind: "TokenRequest", + Status: authenticationv1.TokenRequestStatus{Token: jwt, ExpirationTimestamp: metav1.NewTime(time.Now().Add(10 * time.Minute))}, + }); err != nil { + t.Error("cannot encode fixture TokenRequest response") + } + })) } func TestKubernetesSessionDoesNotFallbackFromMissingToken(t *testing.T) { var requests atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - requests.Add(1) - w.WriteHeader(http.StatusForbidden) - })) - defer server.Close() for _, missing := range []bool{true, false} { - path := filepath.Join(t.TempDir(), "token") - if !missing { - writeAuthToken(t, path, " \n") - } + server := authenticationServer(t, func() (string, bool) { return "", !missing }, func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusForbidden) + }) + defer server.Close() client := fixtureClient(t, server.URL) - session, err := openbao.NewKubernetesSession(client, "kubernetes", authRole, path) + session, err := openbao.NewKubernetesSession(client, authenticationConfig(server.URL), "kubernetes", authRole, testIdentity) if err != nil { t.Fatal(err) } @@ -62,7 +89,7 @@ func TestKubernetesSessionDoesNotFallbackFromMissingToken(t *testing.T) { err = session.Start(ctx) cancel() if err != nil || session.Ready() || client.Token() != "" || requests.Load() != 0 { - t.Fatal("missing or empty token must not fall back to another identity") + t.Fatal("denied or empty TokenRequest must not fall back to another identity") } } } @@ -80,7 +107,7 @@ func waitForAuthentication(t *testing.T, check func() bool) { } } -func startSession(t *testing.T, session *openbao.KubernetesSession) context.CancelFunc { +func startSession(t *testing.T, session interface{ Start(context.Context) error }) context.CancelFunc { t.Helper() ctx, cancel := context.WithCancel(t.Context()) done := make(chan error, 1) @@ -96,10 +123,16 @@ func startSession(t *testing.T, session *openbao.KubernetesSession) context.Canc return cancel } -func TestKubernetesSessionRereadsTokenAfterFailure(t *testing.T) { +func TestKubernetesSessionRequestsNewTokenAfterFailure(t *testing.T) { var attempts atomic.Int32 var accepted atomic.Bool - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var rotated atomic.Bool + server := authenticationServer(t, func() (string, bool) { + if rotated.Load() { + return "rotated-test-jwt", true + } + return "expired-test-jwt", true + }, func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/v1/auth/kubernetes/login" { w.WriteHeader(http.StatusNotFound) return @@ -121,12 +154,10 @@ func TestKubernetesSessionRereadsTokenAfterFailure(t *testing.T) { }}); err != nil { t.Error("cannot encode authentication fixture response") } - })) + }) defer server.Close() - path := filepath.Join(t.TempDir(), "token") - writeAuthToken(t, path, "expired-test-jwt") client := fixtureClient(t, server.URL) - session, err := openbao.NewKubernetesSession(client, "kubernetes", authRole, path) + session, err := openbao.NewKubernetesSession(client, authenticationConfig(server.URL), "kubernetes", authRole, testIdentity) if err != nil { t.Fatal(err) } @@ -138,7 +169,7 @@ func TestKubernetesSessionRereadsTokenAfterFailure(t *testing.T) { if session.Ready() || client.Token() != "" { t.Fatal("failed login retained credentials") } - writeAuthToken(t, path, "rotated-test-jwt") + rotated.Store(true) waitForAuthentication(t, func() bool { return accepted.Load() && session.Ready() }) if client.Token() != fixtureToken { t.Fatal("successful login did not configure the client") @@ -152,19 +183,17 @@ func TestKubernetesSessionRereadsTokenAfterFailure(t *testing.T) { func TestKubernetesSessionRejectsUnboundedLease(t *testing.T) { var attempts atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + server := authenticationServer(t, func() (string, bool) { return "test-jwt", true }, func(w http.ResponseWriter, _ *http.Request) { attempts.Add(1) if err := json.NewEncoder(w).Encode(map[string]any{"auth": map[string]any{ "client_token": fixtureToken, "lease_duration": 0, }}); err != nil { t.Error("cannot encode authentication fixture response") } - })) + }) defer server.Close() - path := filepath.Join(t.TempDir(), "token") - writeAuthToken(t, path, "test-jwt") client := fixtureClient(t, server.URL) - session, err := openbao.NewKubernetesSession(client, "kubernetes", authRole, path) + session, err := openbao.NewKubernetesSession(client, authenticationConfig(server.URL), "kubernetes", authRole, testIdentity) if err != nil { t.Fatal(err) }