Author SHA1 Message Date
panxiao81 0e3564bf72 feat: 部署 OpenSandbox API 与双运行时 Pool
yaml / yaml (pull_request) Successful in 20s
2026-09-18 01:10:07 +00:00
116 changed files with 638 additions and 3203 deletions
+4 -2
View File
@@ -16,6 +16,9 @@ on:
- '.ansible-lint'
- '.gitea/workflows/ansible.yml'
env:
ANSIBLE_COLLECTIONS_PATH: /root/.ansible/collections
jobs:
lint:
runs-on: [self-hosted, pod]
@@ -27,7 +30,6 @@ jobs:
python3 -m pip install --user --break-system-packages \
--index-url https://pypi.org/simple --quiet uv==0.11.7
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
echo "ANSIBLE_COLLECTIONS_PATH=$HOME/.ansible/collections" >> "$GITHUB_ENV"
- name: Install ansible-lint and collections
run: |
@@ -48,7 +50,7 @@ jobs:
- name: ansible-lint
run: |
export PATH="$HOME/.local/bin:$PATH"
export ANSIBLE_COLLECTIONS_PATH="$PWD/infrastructure/samba-ad/ansible/collections:$HOME/.ansible/collections"
export ANSIBLE_COLLECTIONS_PATH="$PWD/infrastructure/samba-ad/ansible/collections:/root/.ansible/collections"
# 静态检查不应依赖生产 vault 凭据。一次性 checkout 可以去掉加密变量文件;
# syntax-check 只验证结构,不需要解析变量的运行时值。
rm -f \
-22
View File
@@ -1,22 +0,0 @@
name: hydra-login
on:
pull_request:
paths:
- 'apps/hydra/login-consent/**'
- '.gitea/workflows/hydra.yml'
workflow_dispatch:
jobs:
verify:
runs-on: [self-hosted, pod]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: apps/hydra/login-consent/go.mod
cache-dependency-path: apps/hydra/login-consent/go.sum
- name: Test authentication boundaries
working-directory: apps/hydra/login-consent
run: |
go test -race ./...
go vet ./...
CGO_ENABLED=0 go build -trimpath .
-20
View File
@@ -1,20 +0,0 @@
# Backstage
Backstage 作为 homelab 的只读开发者门户运行。应用源码、插件、测试和镜像构建归
`panxiao81/backstage` 仓库管理;本目录只保存 Kubernetes/Flux 部署声明,并通过
OCI digest 固定镜像。
## 外部前置
- OpenBao `kv/k8s/backstage` 必须包含以下与容器环境变量同名的字段:
`BACKEND_SECRET`、`AUTH_SESSION_SECRET`、`AUTH_OIDC_CLIENT_ID`、
`AUTH_OIDC_CLIENT_SECRET`、`POSTGRES_PASSWORD`、`GITEA_TOKEN`。
- PostgreSQL 需要在共享集群中预先创建由 `backstage` 角色拥有的 `backstage`
数据库;密码必须与 OpenBao 中的 `POSTGRES_PASSWORD` 一致。
- Authelia OIDC 客户端回调地址为
`https://backstage.ad.ddupan.top/api/auth/oidc/handler/frame`。
- AD DNS 需要将 `backstage.ad.ddupan.top` 指向 Envoy Gateway
`192.168.10.127`。
Flux 等待 ExternalSecret 和 Deployment 就绪;任何前置缺失都会使该
Kustomization 保持 NotReady,而不会回退到明文 Secret。
-85
View File
@@ -1,85 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: backstage
namespace: backstage
labels:
app.kubernetes.io/name: backstage
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: backstage
template:
metadata:
labels:
app.kubernetes.io/name: backstage
spec:
serviceAccountName: backstage
securityContext:
fsGroup: 1000
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
containers:
- name: backstage
image: zot.ad.ddupan.top/panxiao81/backstage@sha256:6d6a11f8611a85237e3b0ffd30a2a6e119b1e5258b2ca6c193d60a8c3ec52553
imagePullPolicy: IfNotPresent
env:
- name: BACKSTAGE_BASE_URL
value: https://backstage.ad.ddupan.top
- name: POSTGRES_HOST
value: shared-postgresql-rw.shared-db.svc.cluster.local
- name: POSTGRES_PORT
value: "5432"
- name: POSTGRES_USER
value: backstage
- name: POSTGRES_DATABASE
value: backstage
- name: GITEA_HOST
value: git.ddupan.top
- name: GITEA_ORG
value: panxiao81
envFrom:
- secretRef:
name: backstage
ports:
- containerPort: 7007
name: http
protocol: TCP
readinessProbe:
httpGet:
path: /healthcheck
port: http
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 3
livenessProbe:
httpGet:
path: /healthcheck
port: http
initialDelaySeconds: 30
periodSeconds: 20
timeoutSeconds: 3
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
volumeMounts:
- mountPath: /tmp
name: tmp
volumes:
- emptyDir: {}
name: tmp
-16
View File
@@ -1,16 +0,0 @@
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: backstage
namespace: backstage
spec:
refreshInterval: 1h
secretStoreRef:
kind: ClusterSecretStore
name: openbao
target:
creationPolicy: Owner
name: backstage
dataFrom:
- extract:
key: k8s/backstage
-20
View File
@@ -1,20 +0,0 @@
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: backstage
namespace: backstage
spec:
parentRefs:
- name: eg
namespace: envoy-gateway-system
sectionName: https
hostnames:
- backstage.ad.ddupan.top
rules:
- matches:
- path:
type: PathPrefix
value: /
backendRefs:
- name: backstage
port: 7007
-10
View File
@@ -1,10 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- serviceaccount.yaml
- external-secret.yaml
- deployment.yaml
- service.yaml
- networkpolicy.yaml
- httproute.yaml
-7
View File
@@ -1,7 +0,0 @@
apiVersion: v1
kind: Namespace
metadata:
name: backstage
labels:
pod-security.kubernetes.io/enforce: restricted
pod-security.kubernetes.io/enforce-version: v1.36
-24
View File
@@ -1,24 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: backstage-ingress
namespace: backstage
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: backstage
policyTypes: [Ingress]
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: envoy-gateway-system
ports:
- port: 7007
protocol: TCP
- from:
- ipBlock:
cidr: 192.168.10.127/32
ports:
- port: 7007
protocol: TCP
-13
View File
@@ -1,13 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: backstage
namespace: backstage
spec:
selector:
app.kubernetes.io/name: backstage
ports:
- name: http
port: 7007
protocol: TCP
targetPort: http
-12
View File
@@ -44,15 +44,3 @@ API、OIDC、Git/Flux 和 runner 均已验证。第二跳按明确决定跳过
结构,或者 release notes 指出相关 breaking migration 时,才恢复停机一致备份、分阶段
suspend、详细日志审计和扩展验收。出现启动失败或 migration error 时也立即升级为完整
故障流程。
## Hydra 人类登录 PoC
新增 `hydra` OIDC 登录源,旧 `authelia` 入口保留。Hydra 通过通用 OIDC Login/Consent
适配器转到现有 Authelia 完成人类认证;不是 Gitea 直接验证 LDAP 或 SPIFFE。
入口为 `https://git.ddupan.top/user/oauth2/hydra`,需 LAN/Tailscale 可达 Hydra 内网域名。
新 client secret 通过 `ExternalSecret/gitea-hydra-oidc` 从 OpenBao 投射。沿用
preferred_username、已验证邮箱与 groups;当前仍映射 gitea-admins,不在本轮切换组模型。
先部署并验证 Hydra discovery 后再接入本配置,避免 Gitea init 因上游不可达而失败。
实际登录验收与部署状态见 wiki;依赖和回退见 [Hydra README](../hydra/README.md)。
-7
View File
@@ -116,13 +116,6 @@ gitea:
scopes: openid profile email groups
groupClaimName: groups
adminGroup: gitea-admins
- name: hydra
provider: openidConnect
existingSecret: gitea-hydra-oidc
autoDiscoverUrl: https://hydra.ad.ddupan.top/.well-known/openid-configuration
scopes: openid profile email groups
groupClaimName: groups
adminGroup: gitea-admins
persistence:
size: 20Gi
-22
View File
@@ -1,22 +0,0 @@
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: gitea-hydra-oidc
namespace: gitea
spec:
refreshInterval: 1h
secretStoreRef:
kind: ClusterSecretStore
name: openbao
target:
name: gitea-hydra-oidc
creationPolicy: Owner
template:
data:
key: gitea
secret: "{{ .client_secret }}"
data:
- secretKey: client_secret
remoteRef:
key: k8s/hydra
property: gitea_client_secret
-1
View File
@@ -16,4 +16,3 @@ resources:
- helmrepository.yaml
- helmrelease.yaml
- httproute.yaml
- hydra-external-secret.yaml
-102
View File
@@ -1,102 +0,0 @@
# Hydra 与 OIDC Login/Consent PoC
本目录提供独立 Hydra 签发服务,以及一个薄的 **OIDC 上游适配器**。当前上游配置为
Authelia;适配器不连接 LDAP,也不管理用户目录。Samba AD、密码和 MFA 继续由现有
Authelia 链路负责。第一轮只接入人类和 Gitea,不实现 agent 动态授权。
```text
Gitea → Hydra → OIDC Login/Consent → Authelia → Samba AD
← OIDC ← 经验证的上游身份 ← OIDC callback
```
目标入口:
- `https://hydra.ad.ddupan.top`:Hydra 公共 OAuth2/OIDC endpoint。
- `https://hydra-login.ad.ddupan.top`:上游 OIDC 登录及 consent 适配器。
- `hydra-admin.hydra.svc.cluster.local:4445`:仅集群内管理接口,无 HTTPRoute。
均为 LAN/Tailscale 入口,复用 Envoy `eg/https` wildcard TLS。没有增加公网 tunnel。
部署及实际验收状态以 wiki 和对应 PR 为准,文件存在不表示登录已验收。
## 首次使用与边界
在 Gitea 登录页选择 `hydra`,跳转到 Authelia 完成现有人类认证,再返回原有 Gitea
账号。旧 `authelia` 登录源保留。Gitea 的账号关联和资源权限仍由 Gitea 维护。
适配器要求验证上游 issuer、audience、签名、过期时间和 nonce,使用 PKCE S256,
并把单次 state 绑定到 Secure/HttpOnly/SameSite=Lax cookie。短期登录事务只存内存,
最多 1024 个、10 分钟过期;单副本重启后正在登录的用户需重试,不存人类密码或 token。
Hydra subject 为上游 `(issuer, sub)` 的 SHA-256 加 `human:` 前缀,与可变邮箱/用户名
分离。第一轮要求上游返回经过验证的 email 及 preferred_username;这些 claims 必须
明确配置进 ID token。更换 issuer 会改变本 PoC 的 subject,正式迁移前需要身份绑定设计。
仅为显式 `ALLOWED_CLIENTS=gitea` 自动 consent,scope 限于 openid/profile/email/groups;
拒绝额外 access-token audience,不发 refresh token。只按实际请求 scope 释放 claims。
这不是通用的无人确认授权服务。组当前透传,沿用 Gitea 的 gitea-admins 映射;统一组
模型和 agent 认证均在后续阶段。不存在对 Authelia 专有协议的调用。
NetworkPolicy 限制公共端口只接收 Envoy 流量,Hydra admin 只允许适配器访问。
Hydra 使用正式模式,TLS 由 Envoy 终止;不使用 `--dev`。管理操作使用受控
`kubectl port-forward`,不要将 admin 接口暴露到 Gateway。
## 依赖、秘密与初始化
依赖共享 CloudNativePG、OpenBao/ESO、Authelia OIDC、Envoy、Samba DNS、zot 镜像仓库。
Hydra 使用独立 `hydra` database/role,不与其他应用共享数据库角色。
`kv/k8s/hydra` 保存 dsn、system_secret、upstream_client_secret、upstream_client_digest、
gitea_client_secret;通过 ExternalSecret 投射,值不写入 Git。Bootstrap 创建角色及数据库
后才启动 Hydra migration。system_secret 必须持久保存,不得在重启时随机重建。
Authelia 中新增 confidential client `hydra-login`:
- redirect URI:`https://hydra-login.ad.ddupan.top/callback`;
- authorization policy:two_factor;grant:authorization_code;PKCE:S256;
- token endpoint auth:client_secret_basic;scope:openid/profile/email/groups;
- claims policy:把 preferred_username、name、email、email_verified、groups 放入 ID token;
- client secret 的 PBKDF2 digest 存入 Authelia,原值仅供适配器使用。
Authelia 尚非 Flux 管理。修改 Helm values 时保留所有已有 clients 与 secret 引用,
通过 `--reuse-values` 和最小 overlay 增加客户端,不能以本目录配置覆盖其完整 values。
Hydra 中注册 confidential client `gitea`,redirect URI 为
`https://git.ddupan.top/user/oauth2/hydra/callback`,grant/response 为 authorization_code/code,
scope 为 openid/profile/email/groups,token endpoint auth 为 client_secret_basic。
Gitea 启动时读取 OIDC discovery,所以应先确认 Hydra 健康和 discovery 可达,再接入 Gitea。
## 构建与检查
```bash
cd apps/hydra/login-consent
go test -race ./...
go vet ./...
CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o login-consent .
docker build -t hydra-login-consent:VERSION .
```
Go module 独立,依赖由 go.sum 锁定;Dockerfile 固定基础镜像 digest。
使用已授权的短期 SPIFFE zot 凭据发布镜像,部署使用匿名拉取入口与不可变 digest。
不把 registry 凭据写入源码或 build args。
```bash
kubectl kustomize apps/hydra
sudo k3s kubectl -n hydra get deployment,pod,externalsecret,httproute
sudo k3s kubectl -n hydra logs deployment/hydra -c migrate
sudo k3s kubectl -n hydra logs deployment/hydra-login
```
日志不输出上游 token、授权 code、challenge 或秘密。登录失败先查两端 Pod 状态、
DNS/discovery 连通性、client redirect URI 和 scope,再由用户重新发起登录。
不要在故障排查中关闭签名验证、MFA 或 state/nonce 校验。
## 恢复与回退
保留共享 PostgreSQL 中 Hydra 数据及 OpenBao 秘密;数据库持有 clients、会话及签名密钥,
单独重建 Deployment 不能替代恢复数据库。先恢复依赖,再启动 Hydra 和适配器。
当前恢复仍依赖 homelab 共享基础设施,不能声称已完成独立灾备。
第一轮不切换 Authelia 的主入口。撤回 Gitea 的新增 Hydra 登录源即可回到旧入口;
先撤消费者,再考虑停用 Hydra。不要删除旧 Authelia 登录源、用户或数据库作为回退手段。
跨服务设计见 [独立 IAM 草案](https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/architecture/independent-iam-draft.md)。
-95
View File
@@ -1,95 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: hydra
namespace: hydra
spec:
replicas: 1
selector:
matchLabels:
app: hydra
template:
metadata:
labels:
app: hydra
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
seccompProfile:
type: RuntimeDefault
initContainers:
- name: migrate
image: docker.io/oryd/hydra:v26.2.0@sha256:ff67c7fb5f95074fa53374d41151713554960504b340cd3f95b09e65deaea2a9
args:
- migrate
- sql
- -e
- --yes
env:
- name: DSN
valueFrom:
secretKeyRef:
name: hydra
key: dsn
securityContext: &id002
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources: &id001
requests:
cpu: 50m
memory: 64Mi
limits:
memory: 256Mi
containers:
- name: hydra
image: docker.io/oryd/hydra:v26.2.0@sha256:ff67c7fb5f95074fa53374d41151713554960504b340cd3f95b09e65deaea2a9
args:
- serve
- all
- --config
- /etc/hydra/hydra.yaml
- --sqa-opt-out
env:
- name: DSN
valueFrom:
secretKeyRef:
name: hydra
key: dsn
- name: SECRETS_SYSTEM
valueFrom:
secretKeyRef:
name: hydra
key: system_secret
ports:
- name: public
containerPort: 4444
- name: admin
containerPort: 4445
resources: *id001
securityContext: *id002
volumeMounts:
- name: config
mountPath: /etc/hydra
readOnly: true
readinessProbe:
httpGet:
path: /health/ready
port: admin
initialDelaySeconds: 5
periodSeconds: 5
livenessProbe:
httpGet:
path: /health/alive
port: admin
initialDelaySeconds: 20
periodSeconds: 20
volumes:
- name: config
configMap:
name: hydra-config
-16
View File
@@ -1,16 +0,0 @@
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: hydra
namespace: hydra
spec:
refreshInterval: 1h
secretStoreRef:
kind: ClusterSecretStore
name: openbao
target:
name: hydra
creationPolicy: Owner
dataFrom:
- extract:
key: k8s/hydra
-33
View File
@@ -1,33 +0,0 @@
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: hydra-public
namespace: hydra
spec:
parentRefs:
- name: eg
namespace: envoy-gateway-system
sectionName: https
hostnames:
- hydra.ad.ddupan.top
rules:
- backendRefs:
- name: hydra-public
port: 4444
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: hydra-login
namespace: hydra
spec:
parentRefs:
- name: eg
namespace: envoy-gateway-system
sectionName: https
hostnames:
- hydra-login.ad.ddupan.top
rules:
- backendRefs:
- name: hydra-login
port: 8080
-25
View File
@@ -1,25 +0,0 @@
serve:
public:
port: 4444
admin:
port: 4445
tls:
allow_termination_from:
- 10.42.0.0/16
cookies:
same_site_mode: Lax
urls:
self:
issuer: https://hydra.ad.ddupan.top
public: https://hydra.ad.ddupan.top
login: https://hydra-login.ad.ddupan.top/login
consent: https://hydra-login.ad.ddupan.top/consent
ttl:
access_token: 15m
id_token: 15m
auth_code: 5m
log:
level: info
leak_sensitive_values: false
oauth2:
expose_internal_errors: false
-15
View File
@@ -1,15 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- external-secret.yaml
- deployment.yaml
- login-deployment.yaml
- services.yaml
- httproutes.yaml
- networkpolicy.yaml
configMapGenerator:
- name: hydra-config
namespace: hydra
files:
- hydra.yaml
-1
View File
@@ -1 +0,0 @@
/login-consent
-4
View File
@@ -1,4 +0,0 @@
FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab
COPY login-consent /login-consent
USER 65532:65532
ENTRYPOINT ["/login-consent"]
-13
View File
@@ -1,13 +0,0 @@
module git.ddupan.top/panxiao81/homelab-infra/apps/hydra/login-consent
go 1.26.0
require (
github.com/coreos/go-oidc/v3 v3.14.1
golang.org/x/oauth2 v0.37.0
)
require (
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
golang.org/x/crypto v0.36.0 // indirect
)
-18
View File
@@ -1,18 +0,0 @@
github.com/coreos/go-oidc/v3 v3.14.1 h1:9ePWwfdwC4QKRlCXsJGou56adA/owXczOzwKdOumLqk=
github.com/coreos/go-oidc/v3 v3.14.1/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE=
github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-285
View File
@@ -1,285 +0,0 @@
// Login/Consent adapter for a single trusted upstream and first-party clients.
package main
import (
"bytes"
"context"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"strings"
"sync"
"time"
"github.com/coreos/go-oidc/v3/oidc"
"golang.org/x/oauth2"
)
const cookieName = "__Host-hydra-login"
type pending struct {
Challenge, Nonce, Verifier string
Expires time.Time
}
type claims struct {
Username string `json:"preferred_username"`
Email string `json:"email"`
EmailVerified bool `json:"email_verified"`
Name string `json:"name"`
Groups []string `json:"groups"`
}
type flowRequest struct {
Client struct {
ID string `json:"client_id"`
} `json:"client"`
Subject string `json:"subject"`
Scopes []string `json:"requested_scope"`
Audience []string `json:"requested_access_token_audience"`
Context claims `json:"context"`
}
type app struct {
admin, public string
client *http.Client
oauth oauth2.Config
verifier *oidc.IDTokenVerifier
allowed map[string]bool
mu sync.Mutex
pending map[string]pending
}
func required(key string) string {
v := os.Getenv(key)
if v == "" {
log.Fatalf("missing %s", key)
}
return v
}
func random() string {
b := make([]byte, 32)
if _, err := rand.Read(b); err != nil {
panic(err)
}
return base64.RawURLEncoding.EncodeToString(b)
}
func (a *app) api(ctx context.Context, method, path string, in, out any) error {
var body io.Reader
if in != nil {
b, err := json.Marshal(in)
if err != nil {
return err
}
body = bytes.NewReader(b)
}
req, err := http.NewRequestWithContext(ctx, method, a.admin+path, body)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := a.client.Do(req)
if err != nil {
return errors.New("Hydra unavailable")
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("Hydra status %d", resp.StatusCode)
}
if out != nil {
return json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(out)
}
return nil
}
func (a *app) request(r *http.Request, kind, challenge string) (flowRequest, error) {
var f flowRequest
if challenge == "" || len(challenge) > 8192 {
return f, errors.New("missing or invalid challenge")
}
err := a.api(r.Context(), http.MethodGet, "/admin/oauth2/auth/requests/"+kind+"?"+kind+"_challenge="+url.QueryEscape(challenge), nil, &f)
if err != nil {
return f, err
}
if !a.allowed[f.Client.ID] {
return f, errors.New("client not allowed")
}
return f, nil
}
func (a *app) accept(w http.ResponseWriter, r *http.Request, kind, challenge string, body any) {
var result struct {
Redirect string `json:"redirect_to"`
}
if err := a.api(r.Context(), http.MethodPut, "/admin/oauth2/auth/requests/"+kind+"/accept?"+kind+"_challenge="+url.QueryEscape(challenge), body, &result); err != nil {
fail(w, 502)
return
}
// Only Hydra's own authorization endpoint can receive a challenge verifier.
u, err := url.Parse(result.Redirect)
p, _ := url.Parse(a.public)
if err != nil || u.Scheme != p.Scheme || u.Host != p.Host || u.User != nil || u.Path != "/oauth2/auth" {
fail(w, 502)
return
}
http.Redirect(w, r, result.Redirect, http.StatusSeeOther)
}
func fail(w http.ResponseWriter, status int) { http.Error(w, http.StatusText(status), status) }
func (a *app) login(w http.ResponseWriter, r *http.Request) {
challenge := r.URL.Query().Get("login_challenge")
if _, err := a.request(r, "login", challenge); err != nil {
fail(w, 403)
return
}
state := random()
p := pending{challenge, random(), oauth2.GenerateVerifier(), time.Now().Add(10 * time.Minute)}
a.mu.Lock()
for k, v := range a.pending {
if time.Now().After(v.Expires) {
delete(a.pending, k)
}
}
if len(a.pending) >= 1024 {
a.mu.Unlock()
fail(w, 503)
return
}
a.pending[state] = p
a.mu.Unlock()
http.SetCookie(w, &http.Cookie{Name: cookieName, Value: state, Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteLaxMode, MaxAge: 600})
http.Redirect(w, r, a.oauth.AuthCodeURL(state, oidc.Nonce(p.Nonce), oauth2.S256ChallengeOption(p.Verifier)), http.StatusSeeOther)
}
func (a *app) take(r *http.Request) (pending, error) {
state := r.URL.Query().Get("state")
cookie, err := r.Cookie(cookieName)
if err != nil || state == "" || subtle.ConstantTimeCompare([]byte(cookie.Value), []byte(state)) != 1 {
return pending{}, errors.New("state mismatch")
}
a.mu.Lock()
defer a.mu.Unlock()
p, ok := a.pending[state]
delete(a.pending, state)
if !ok || time.Now().After(p.Expires) {
return pending{}, errors.New("expired or used state")
}
return p, nil
}
func (a *app) callback(w http.ResponseWriter, r *http.Request) {
p, err := a.take(r)
if err != nil {
fail(w, 403)
return
}
http.SetCookie(w, &http.Cookie{Name: cookieName, Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteLaxMode, MaxAge: -1})
if r.URL.Query().Get("error") != "" || r.URL.Query().Get("code") == "" {
fail(w, 403)
return
}
ctx := oidc.ClientContext(r.Context(), a.client)
token, err := a.oauth.Exchange(ctx, r.URL.Query().Get("code"), oauth2.VerifierOption(p.Verifier))
if err != nil {
fail(w, 502)
return
}
raw, ok := token.Extra("id_token").(string)
if !ok {
fail(w, 502)
return
}
id, err := a.verifier.Verify(ctx, raw)
if err != nil || id.Nonce != p.Nonce || id.Subject == "" {
fail(w, 403)
return
}
var c claims
if id.Claims(&c) != nil || c.Username == "" || c.Email == "" || !c.EmailVerified {
fail(w, 403)
return
}
if _, err := a.request(r, "login", p.Challenge); err != nil {
fail(w, 403)
return
}
// Stable identity is tied to the verified upstream issuer+subject, never email.
sum := sha256.Sum256([]byte(id.Issuer + "\x00" + id.Subject))
a.accept(w, r, "login", p.Challenge, map[string]any{"subject": "human:" + hex.EncodeToString(sum[:]), "remember": false, "context": c})
}
func consentSession(f flowRequest) (map[string]any, error) {
if !strings.HasPrefix(f.Subject, "human:") || f.Context.Username == "" || f.Context.Email == "" || !f.Context.EmailVerified {
return nil, errors.New("invalid identity context")
}
allowed := map[string]bool{"openid": true, "profile": true, "email": true, "groups": true}
session := map[string]any{"principal_type": "human"}
for _, scope := range f.Scopes {
if !allowed[scope] {
return nil, errors.New("scope not allowed")
}
switch scope {
case "profile":
session["preferred_username"] = f.Context.Username
session["name"] = f.Context.Name
case "email":
session["email"] = f.Context.Email
session["email_verified"] = true
case "groups":
session["groups"] = f.Context.Groups
}
}
if len(f.Audience) > 0 {
return nil, errors.New("access token audience not allowed")
}
return session, nil
}
func (a *app) consent(w http.ResponseWriter, r *http.Request) {
challenge := r.URL.Query().Get("consent_challenge")
f, err := a.request(r, "consent", challenge)
if err != nil {
fail(w, 403)
return
}
session, err := consentSession(f)
if err != nil {
fail(w, 403)
return
}
// Explicit policy for pre-approved first-party clients only; no generic auto-consent.
a.accept(w, r, "consent", challenge, map[string]any{"grant_scope": f.Scopes, "remember": false, "session": map[string]any{"id_token": session}})
}
func (a *app) handler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })
mux.HandleFunc("GET /login", a.login)
mux.HandleFunc("GET /callback", a.callback)
mux.HandleFunc("GET /consent", a.consent)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Referrer-Policy", "no-referrer")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("Content-Security-Policy", "default-src 'none'; frame-ancestors 'none'")
mux.ServeHTTP(w, r)
})
}
func main() {
client := &http.Client{Timeout: 15 * time.Second, CheckRedirect: func(r *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }}
issuer := required("UPSTREAM_ISSUER")
ctx := oidc.ClientContext(context.Background(), client)
provider, err := oidc.NewProvider(ctx, issuer)
if err != nil {
log.Fatal("upstream discovery failed")
}
clientID := required("UPSTREAM_CLIENT_ID")
a := &app{admin: required("HYDRA_ADMIN_URL"), public: required("HYDRA_PUBLIC_URL"), client: client, allowed: map[string]bool{}, pending: map[string]pending{},
oauth: oauth2.Config{ClientID: clientID, ClientSecret: required("UPSTREAM_CLIENT_SECRET"), RedirectURL: required("CALLBACK_URL"), Endpoint: provider.Endpoint(), Scopes: []string{"openid", "profile", "email", "groups"}},
verifier: provider.Verifier(&oidc.Config{ClientID: clientID})}
for _, id := range strings.Split(required("ALLOWED_CLIENTS"), ",") {
a.allowed[id] = true
}
s := http.Server{Addr: ":8080", Handler: a.handler(), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 20 * time.Second, WriteTimeout: 45 * time.Second, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 16384}
log.Print("login/consent adapter listening on :8080")
log.Fatal(s.ListenAndServe())
}
-107
View File
@@ -1,107 +0,0 @@
package main
import (
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
"golang.org/x/oauth2"
)
func TestStateBoundToCookieSingleUseAndExpiry(t *testing.T) {
a := &app{pending: map[string]pending{"valid": {Challenge: "challenge", Expires: time.Now().Add(time.Minute)}, "expired": {Expires: time.Now().Add(-time.Minute)}}}
request := func(state, cookie string) *http.Request {
r := httptest.NewRequest("GET", "https://login.example/callback?state="+state, nil)
if cookie != "" {
r.AddCookie(&http.Cookie{Name: cookieName, Value: cookie})
}
return r
}
for _, r := range []*http.Request{request("valid", ""), request("valid", "other"), request("expired", "expired")} {
if _, err := a.take(r); err == nil {
t.Fatal("invalid state accepted")
}
}
if p, err := a.take(request("valid", "valid")); err != nil || p.Challenge != "challenge" {
t.Fatal("valid state rejected")
}
if _, err := a.take(request("valid", "valid")); err == nil {
t.Fatal("replayed state accepted")
}
}
func TestConsentRejectsPrivilegeExpansionAndFiltersClaims(t *testing.T) {
f := flowRequest{Subject: "human:known", Scopes: []string{"openid", "email"}, Context: claims{Username: "alice", Email: "[email protected]", EmailVerified: true, Groups: []string{"operators"}}}
s, err := consentSession(f)
if err != nil {
t.Fatal(err)
}
if _, ok := s["groups"]; ok {
t.Fatal("groups leaked without scope")
}
if _, ok := s["preferred_username"]; ok {
t.Fatal("profile leaked without scope")
}
for _, scope := range []string{"admin", "offline_access", "unknown"} {
bad := f
bad.Scopes = append([]string{"openid"}, scope)
if _, err := consentSession(bad); err == nil {
t.Fatalf("accepted %s", scope)
}
}
f.Audience = []string{"other-service"}
if _, err := consentSession(f); err == nil {
t.Fatal("unexpected audience accepted")
}
f.Audience = nil
f.Context.EmailVerified = false
if _, err := consentSession(f); err == nil {
t.Fatal("unverified email accepted")
}
}
func TestLoginValidatesClientAndUsesPKCEAndNonce(t *testing.T) {
admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]any{"client": map[string]string{"client_id": r.URL.Query().Get("login_challenge")}})
}))
defer admin.Close()
a := &app{admin: admin.URL, client: admin.Client(), allowed: map[string]bool{"gitea": true}, pending: map[string]pending{}, oauth: oauth2.Config{ClientID: "hydra-login", RedirectURL: "https://login.example/callback", Endpoint: oauth2.Endpoint{AuthURL: "https://upstream.example/authorize"}}}
w := httptest.NewRecorder()
a.handler().ServeHTTP(w, httptest.NewRequest("GET", "https://login.example/login?login_challenge=rogue", nil))
if w.Code != 403 {
t.Fatal("unknown client accepted")
}
w = httptest.NewRecorder()
a.handler().ServeHTTP(w, httptest.NewRequest("GET", "https://login.example/login?login_challenge=gitea", nil))
if w.Code != 303 {
t.Fatalf("status %d", w.Code)
}
u, _ := url.Parse(w.Header().Get("Location"))
q := u.Query()
if q.Get("code_challenge_method") != "S256" || q.Get("code_challenge") == "" || q.Get("nonce") == "" || q.Get("state") == "" {
t.Fatal("missing protocol binding")
}
cookies := w.Result().Cookies()
if len(cookies) != 1 || !cookies[0].Secure || !cookies[0].HttpOnly || cookies[0].SameSite != http.SameSiteLaxMode || cookies[0].Value != q.Get("state") {
t.Fatal("unsafe cookie")
}
if w.Header().Get("Cache-Control") != "no-store" {
t.Fatal("missing cache protection")
}
}
func TestHydraRedirectCannotLeaveTrustedOrigin(t *testing.T) {
for _, target := range []string{"https://evil.example/oauth2/auth", "https://[email protected]/oauth2/auth", "https://hydra.example/other"} {
admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]string{"redirect_to": target})
}))
a := &app{admin: admin.URL, public: "https://hydra.example", client: admin.Client()}
w := httptest.NewRecorder()
a.accept(w, httptest.NewRequest("GET", "https://login.example/login", nil), "login", "challenge", map[string]string{"subject": "human:test"})
if w.Code != 502 || strings.Contains(w.Header().Get("Location"), "evil") {
t.Fatal("untrusted redirect accepted")
}
admin.Close()
}
}
-69
View File
@@ -1,69 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: hydra-login
namespace: hydra
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: hydra-login
template:
metadata:
labels:
app: hydra-login
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 65532
runAsGroup: 65532
seccompProfile:
type: RuntimeDefault
containers:
- name: login-consent
image: zot.ad.ddupan.top/iam/oidc-login-consent@sha256:fede9b9e93c457c4b7a8a6022d9df86ff5d5900d3f6b6c4f3439851a0ae1e944
env:
- name: HYDRA_ADMIN_URL
value: http://hydra-admin.hydra.svc.cluster.local:4445
- name: HYDRA_PUBLIC_URL
value: https://hydra.ad.ddupan.top
- name: UPSTREAM_ISSUER
value: https://auth.ddupan.top
- name: UPSTREAM_CLIENT_ID
value: hydra-login
- name: CALLBACK_URL
value: https://hydra-login.ad.ddupan.top/callback
- name: ALLOWED_CLIENTS
value: gitea
- name: UPSTREAM_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: hydra
key: upstream_client_secret
ports:
- name: http
containerPort: 8080
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
memory: 256Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
readinessProbe:
httpGet:
path: /healthz
port: http
livenessProbe:
httpGet:
path: /healthz
port: http
initialDelaySeconds: 10
-6
View File
@@ -1,6 +0,0 @@
apiVersion: v1
kind: Namespace
metadata:
name: hydra
labels:
pod-security.kubernetes.io/enforce: restricted
-46
View File
@@ -1,46 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hydra
namespace: hydra
spec:
podSelector:
matchLabels:
app: hydra
policyTypes:
- Ingress
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: envoy-gateway-system
ports:
- port: 4444
protocol: TCP
- from:
- podSelector:
matchLabels:
app: hydra-login
ports:
- port: 4445
protocol: TCP
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hydra-login
namespace: hydra
spec:
podSelector:
matchLabels:
app: hydra-login
policyTypes:
- Ingress
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: envoy-gateway-system
ports:
- port: 8080
protocol: TCP
-35
View File
@@ -1,35 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: hydra-public
namespace: hydra
spec:
selector:
app: hydra
ports:
- port: 4444
targetPort: 4444
---
apiVersion: v1
kind: Service
metadata:
name: hydra-admin
namespace: hydra
spec:
selector:
app: hydra
ports:
- port: 4445
targetPort: 4445
---
apiVersion: v1
kind: Service
metadata:
name: hydra-login
namespace: hydra
spec:
selector:
app: hydra-login
ports:
- port: 8080
targetPort: 8080
-157
View File
@@ -1,157 +0,0 @@
# Nexus Repository POC
本目录声明一个 Nexus Repository Community Edition POC,用来验证一次性 CI runner 通过
网络服务复用 Ansible Galaxy、Go Modules 与 OCI/BuildKit 缓存。Nexus 固定为 `3.96.1`,
镜像固定到官方 multi-arch index digest;LAN 入口为
`https://nexus.ad.ddupan.top`。
## POC 边界
- 单副本 Deployment,`Recreate` 更新,避免一个 RWO 卷被两个 Pod 同时挂载。
- `/nexus-data` 使用 `localpv-zfs-ceph` 上的 50 GiB RWO PVC。
- 资源预算为 250m/2 GiB request、2 CPU/4 GiB limit;JVM heap 上限 2 GiB。
- 使用容器默认的 embedded H2。它只用于 POC;正式接管 OCI 制品前必须迁移到外部
PostgreSQL,并验证备份恢复。
- 入口只在 LAN wildcard Gateway 上发布,不创建公网 DNS 或 Cloudflare route。
- 不套 Authelia forward-auth;它会破坏 Go、Ansible 与 OCI 非浏览器客户端。
- 现有 zot 保持不变。Nexus 完成 OCI、BuildKit cache 和恢复验收前不得迁移或删除 zot。
Terraform provider 创建 `ansible-public`、`go-public`、最小匿名权限与 OCI Bearer Token
Realm。Nexus 3.94 才加入的原生 OCI repository 已有 REST API,但当前锁定的 community
provider 尚未暴露 OCI resource;`terraform/reconcile-oci.sh` 因此根据 3.96.1 实例 Swagger
固定的 JSON schema,幂等调和 `oci-hosted`、`oci-proxy` 与 `oci-public`。不得绕过该入口在
UI 中创建无人管理的长期 repository。
## 部署
Flux 从 `clusters/homelab/apps/nexus.yaml` 协调本目录,并依赖 Envoy Gateway 与 OpenEBS。
合并前只渲染配置,不直接 apply:
```bash
kubectl kustomize apps/nexus
```
合并并由 Flux 部署后检查:
```bash
kubectl -n flux-system get kustomization nexus
kubectl -n nexus get pod,pvc,service,httproute
kubectl -n nexus logs deployment/nexus --tail=100
```
启动可能需要数分钟,startup probe 允许最多十分钟。不要因初次启动较慢反复删除 Pod;
先确认 PVC 已 Bound、Pod 没有 OOM,以及日志仍在推进。
DNS 期望状态已加入 `infrastructure/dns/records.yml`,需从 Samba AD Ansible root 以
`--check --diff` 核对后再按其 README 应用 DNS tag。没有 DNS 时可先用 port-forward
验证应用,但不能据此宣称 Gateway 路径已通过。
## 首次初始化与 Terraform
初始管理员密码生成在 PVC 的 `/nexus-data/admin.password`。只在交互式终端中读取并立即
完成首次密码轮换;不得把密码复制进 shell tracing、工单、Git 或命令参数。随后将
Terraform 管理账号的凭据存入 OpenBao,由 CI 通过 Terraform input variable 注入以下
环境变量:
```text
TF_VAR_nexus_url=https://nexus.ad.ddupan.top
TF_VAR_nexus_username=admin
TF_VAR_nexus_password=<OpenBao kv/infra/nexus 的 admin_password 字段>
```
`terraform/` 使用 `sonatype-nexus-community/sonatyperepo` 1.17.0,当前声明:
- `ansible-galaxy-proxy` → `https://galaxy.ansible.com`
- `ansible-public` group
- `go-proxy` → `https://proxy.golang.org`
- `go-public` group
provider credential 不写入 HCL 或 tfvars。正式 apply 前还必须为这个独立 Terraform root
配置远端 backend;本地 state 只允许用于可丢弃的 POC,不提交。验证命令:
```bash
terraform -chdir=apps/nexus/terraform init -backend=false
terraform -chdir=apps/nexus/terraform validate
```
先以 `--check` 查看 OCI repository 漂移,再明确 apply;脚本只从上述环境变量取得凭据,
用临时 `0600` netrc 调用 REST API,退出时删除:
```bash
apps/nexus/terraform/reconcile-oci.sh --check
apps/nexus/terraform/reconcile-oci.sh --apply
```
Terraform 同时把内置 `anonymous` 用户从默认的全仓库 `nx-anonymous` 角色收窄到
`ansible-public`、其返回制品 URL 使用的 `ansible-galaxy-proxy`、`go-public`,以及
`oci-public`/`oci-proxy` 的 `browse/read` 权限。`oci-hosted` 不向匿名用户开放。首次接管
已有实例时先执行
`terraform import sonatyperepo_user.anonymous anonymous,default`,再 apply;不要先启用默认的
全仓库匿名读取。
2026-09-20 的 POC 现场验收已确认:Flux 与 Pod Ready、PVC Bound、HTTPRoute 通过 HTTPS
返回 Nexus 状态 200,Samba DNS 已幂等收敛。全新客户端目录通过匿名入口下载
`community.general:11.2.0` 时冷缓存为 8.49 秒、热缓存为 1.89 秒,两次 tarball SHA-256
一致;`golang.org/x/[email protected]` 为 2.92 秒与 1.51 秒。
## 客户端验收
先验证冷缓存,再原样重复命令验证热缓存;记录 Nexus 请求、上游流量和耗时,不只观察
命令成功。Ansible 配置的 URL 必须以 `/` 结尾:
```ini
[galaxy]
server_list = nexus
[galaxy_server.nexus]
url = https://nexus.ad.ddupan.top/repository/ansible-public/
```
```bash
ansible-galaxy collection install -r collections/requirements.yml \
-p .ansible/collections
```
Go POC 使用:
```bash
GOPROXY=https://nexus.ad.ddupan.top/repository/go-public/ go mod download
```
私有 `git.ddupan.top/*` module 的 `GOPRIVATE`、凭据与是否允许 `direct` fallback 在实际
workflow 中单独决定;不要让私有 module path 意外发往公共 proxy。
OCI 使用 path-based routing:匿名公共拉取地址形如
`nexus.ad.ddupan.top/oci-public/library/alpine:3.22`,认证写入地址形如
`nexus.ad.ddupan.top/oci-hosted/<namespace>/<image>:<tag>`。2026-09-20 现场验收结果:
- `oci-public` 匿名代理拉取 Alpine 冷缓存 4.75 秒、热缓存 0.80 秒,digest 一致;
- `oci-hosted` 认证 push/pull 成功,匿名 pull 返回 401;
- amd64/arm64 OCI image index push 成功,两个平台 manifest 可见;
- Helm chart push/pull digest 与本地 tarball SHA-256 一致;
- Cosign 3.1.3 使用一次性密钥签名并验证成功,OCI 1.1 referrers API 返回一个
`application/vnd.dev.sigstore.bundle.v0.3+json` artifact;
- BuildKit `registry` cache 以 `mode=max` 导出成功,销毁首个 builder 后由新 builder 导入,
两个 `RUN` step 均明确命中 `CACHED`。
本机安装的测试客户端包括 `/usr/local/bin/cosign` 3.1.3;安装时核对官方 Linux amd64
binary SHA-256 `4629c757b7618056f8ddd7e2625ae9fdd94c0372a65049520bc7d9df9efc7f71`。
上述结果仍不代表备份恢复、外部 PostgreSQL 或正式 publisher service account 已完成;
这些项目通过前不得迁移或删除 zot。
## 数据与恢复
POC 的数据库、配置、blob、初始管理员状态都位于 `nexus-data` PVC。删除 Deployment
不会删除 PVC;删除 PVC 会永久删除整个 POC。当前没有独立备份,不能将它用于唯一副本的
正式制品。
恢复验证至少包括:停止写入、取得一致备份、在独立 PVC/实例恢复、登录、列出 repository、
拉取已缓存的 Ansible/Go 制品,并核对 OCI digest/referrers。正式化时再把数据库迁移至
外部 PostgreSQL,并分别定义数据库与 blob 的备份、恢复顺序和 RPO。
参考:
- [Nexus OCI repositories](https://help.sonatype.com/en/oci-repositories.html)
- [Nexus Ansible repositories](https://help.sonatype.com/en/ansible-repositories.html)
- [Nexus Go repositories](https://help.sonatype.com/en/go-repositories.html)
- [官方容器镜像](https://hub.docker.com/r/sonatype/nexus3)
-82
View File
@@ -1,82 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: nexus
namespace: nexus
labels:
app.kubernetes.io/name: nexus
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: nexus
template:
metadata:
labels:
app.kubernetes.io/name: nexus
spec:
automountServiceAccountToken: false
securityContext:
fsGroup: 200
fsGroupChangePolicy: OnRootMismatch
runAsGroup: 200
runAsNonRoot: true
runAsUser: 200
seccompProfile:
type: RuntimeDefault
terminationGracePeriodSeconds: 120
containers:
- name: nexus
image: docker.io/sonatype/nexus3:3.96.1@sha256:56142f13432cf072e017aebb2025f201e42ae36ff40bb82618c702504c61f7dd
imagePullPolicy: IfNotPresent
env:
- name: INSTALL4J_ADD_VM_PARAMS
value: >-
-Xms1024m -Xmx2048m -XX:MaxDirectMemorySize=1024m
-Djava.util.prefs.userRoot=/nexus-data/javaprefs
ports:
- name: http
containerPort: 8081
protocol: TCP
resources:
requests:
cpu: 250m
memory: 2Gi
limits:
cpu: "2"
memory: 4Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
startupProbe:
httpGet:
path: /service/rest/v1/status
port: http
failureThreshold: 60
periodSeconds: 10
timeoutSeconds: 5
readinessProbe:
httpGet:
path: /service/rest/v1/status
port: http
failureThreshold: 6
periodSeconds: 10
timeoutSeconds: 5
livenessProbe:
httpGet:
path: /service/rest/v1/status
port: http
failureThreshold: 6
periodSeconds: 30
timeoutSeconds: 5
volumeMounts:
- name: data
mountPath: /nexus-data
volumes:
- name: data
persistentVolumeClaim:
claimName: nexus-data
-16
View File
@@ -1,16 +0,0 @@
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: nexus
namespace: nexus
spec:
parentRefs:
- name: eg
namespace: envoy-gateway-system
sectionName: https
hostnames:
- nexus.ad.ddupan.top
rules:
- backendRefs:
- name: nexus
port: 8081
-9
View File
@@ -1,9 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- pvc.yaml
- deployment.yaml
- service.yaml
- httproute.yaml
- networkpolicy.yaml
-8
View File
@@ -1,8 +0,0 @@
apiVersion: v1
kind: Namespace
metadata:
name: nexus
labels:
pod-security.kubernetes.io/enforce: restricted
pod-security.kubernetes.io/audit: restricted
pod-security.kubernetes.io/warn: restricted
-23
View File
@@ -1,23 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: nexus-ingress
namespace: nexus
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: nexus
policyTypes:
- Ingress
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: envoy-gateway-system
podSelector:
matchLabels:
gateway.envoyproxy.io/owning-gateway-name: eg
gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway-system
ports:
- protocol: TCP
port: 8081
-12
View File
@@ -1,12 +0,0 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: nexus-data
namespace: nexus
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 50Gi
storageClassName: localpv-zfs-ceph
-14
View File
@@ -1,14 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: nexus
namespace: nexus
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: nexus
ports:
- name: http
port: 8081
protocol: TCP
targetPort: http
-6
View File
@@ -1,6 +0,0 @@
.terraform/
*.tfstate
*.tfstate.*
*.tfplan
crash.log
crash.*.log
-24
View File
@@ -1,24 +0,0 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/sonatype-nexus-community/sonatyperepo" {
version = "1.17.0"
constraints = "1.17.0"
hashes = [
"h1:uKhvhhhI7B+HBsh0zq/ybqKt+EnOGyI6rjcRCtj79ZA=",
"zh:0dde99e7b343fa01f8eefc378171fb8621bedb20f59157d6cc8e3d46c738105f",
"zh:3315929df254a3a6ac27c8c846c2006f7d2a91fadc014351bc4d617f948e5bf9",
"zh:36be5a455af3ce4e187de26753de63e78c1ee9a32dba0135c6cf96a6c1fff25f",
"zh:3f73f7ff57b8c339a7c7ac37653e2dc0b2dd9dcc3f3a538788e7e3ac838337b2",
"zh:40286ecca4c22ab7ae90618ac6d2743f5055199dac81cf5204a4a397c784d439",
"zh:4d24e5c0195fb3155b1967583ee64cfeda402d7cc7f3c73369438f6c69f4245b",
"zh:828a9d7aceaac36af7f9c07af43ec8d20a89148780645d170ffb1c68b2da792d",
"zh:a5ab04de3fe626ec57c832618c6f990abd6610f81e132621651e0b180b970cff",
"zh:a959fa6090a8c0f53739879184e7346423494aee598003df0d1ab4a22b2eee91",
"zh:bdda26c2f03f918bbe59e75abea44868fafda019c3a543725331195df126350b",
"zh:d8048e149ee97ba62971e6a79355d59887bc6d10fcf72cc2feff3d0a2582670c",
"zh:dd36f9988af4e1ca5b1ca7b7bb6f658df9a220dfcda7fec7392fedfe9064f652",
"zh:dda2688d46c7e539fe97e8fe9d3ec81fb364170e018d9c6a681364c8955d4e9d",
"zh:e6b519afe2dea1c0434f766eb6bc9ba78cc5b6ef2c311c2ca3c65cb24744f31f",
]
}
-17
View File
@@ -1,17 +0,0 @@
{
"name": "oci-hosted",
"online": true,
"storage": {
"blobStoreName": "default",
"strictContentTypeValidation": true,
"writePolicy": "ALLOW",
"latestPolicy": false
},
"oci": {
"v1Enabled": false,
"forceBasicAuth": false,
"pathEnabled": true
},
"component": { "proprietaryComponents": false },
"cosign": { "enforcement": "NONE" }
}
-27
View File
@@ -1,27 +0,0 @@
{
"name": "oci-proxy",
"online": true,
"storage": {
"blobStoreName": "default",
"strictContentTypeValidation": true
},
"oci": {
"v1Enabled": false,
"forceBasicAuth": false,
"pathEnabled": true
},
"ociProxy": {
"indexType": "HUB",
"cacheForeignLayers": false,
"foreignLayerUrlWhitelist": []
},
"proxy": {
"remoteUrl": "https://registry-1.docker.io",
"contentMaxAge": 1440,
"metadataMaxAge": 60,
"preserveEncodedCharacters": false
},
"negativeCache": { "enabled": true, "timeToLive": 60 },
"httpClient": { "blocked": false, "autoBlock": true },
"cosign": { "enforcement": "NONE" }
}
-15
View File
@@ -1,15 +0,0 @@
{
"name": "oci-public",
"online": true,
"storage": {
"blobStoreName": "default",
"strictContentTypeValidation": true
},
"group": { "memberNames": ["oci-proxy"] },
"oci": {
"v1Enabled": false,
"forceBasicAuth": false,
"pathEnabled": true
},
"cosign": { "enforcement": "NONE" }
}
-80
View File
@@ -1,80 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
mode="${1:---check}"
case "$mode" in
--check | --apply) ;;
*) echo "usage: $0 [--check|--apply]" >&2; exit 2 ;;
esac
: "${TF_VAR_nexus_url:?set TF_VAR_nexus_url}"
: "${TF_VAR_nexus_username:?set TF_VAR_nexus_username}"
: "${TF_VAR_nexus_password:?set TF_VAR_nexus_password}"
script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
auth_file="$(mktemp /tmp/nexus-oci-auth.XXXXXX)"
trap 'rm -f -- "$auth_file"' EXIT
chmod 0600 "$auth_file"
printf 'machine %s\nlogin %s\npassword %s\n' \
"${TF_VAR_nexus_url#*://}" "$TF_VAR_nexus_username" \
"$TF_VAR_nexus_password" >"$auth_file"
drift=0
for entry in \
"hosted:$script_dir/oci/oci-hosted.json" \
"proxy:$script_dir/oci/oci-proxy.json" \
"group:$script_dir/oci/oci-public.json"; do
repository_type="${entry%%:*}"
desired_file="${entry#*:}"
repository_name="$(jq -er '.name' "$desired_file")"
endpoint="$TF_VAR_nexus_url/service/rest/v1/repositories/oci/$repository_type"
current_file="$(mktemp /tmp/nexus-oci-current.XXXXXX)"
status="$(curl --silent --show-error --netrc-file "$auth_file" \
--output "$current_file" --write-out '%{http_code}' \
"$endpoint/$repository_name")"
if [[ "$status" == 404 ]]; then
drift=1
if [[ "$mode" == --apply ]]; then
curl --fail --silent --show-error --netrc-file "$auth_file" \
--header 'Content-Type: application/json' \
--data-binary "@$desired_file" "$endpoint"
echo "created $repository_name"
else
echo "missing $repository_name" >&2
fi
elif [[ "$status" == 200 ]]; then
if jq -e --slurpfile desired "$desired_file" '
def subset($actual; $wanted):
if ($wanted | type) == "object" then
all($wanted | keys[];
($actual[.] != null) and subset($actual[.]; $wanted[.]))
else
$actual == $wanted
end;
subset(.; $desired[0])
' "$current_file" >/dev/null; then
echo "in sync $repository_name"
else
drift=1
if [[ "$mode" == --apply ]]; then
curl --fail --silent --show-error --netrc-file "$auth_file" \
--request PUT --header 'Content-Type: application/json' \
--data-binary "@$desired_file" "$endpoint/$repository_name"
echo "updated $repository_name"
else
echo "drifted $repository_name" >&2
fi
fi
else
cat "$current_file" >&2
echo "unexpected HTTP $status for $repository_name" >&2
exit 1
fi
rm -f -- "$current_file"
done
if [[ "$mode" == --check && "$drift" -ne 0 ]]; then
exit 1
fi
-64
View File
@@ -1,64 +0,0 @@
locals {
proxy_http_client = {
auto_block = true
blocked = false
}
proxy_negative_cache = {
enabled = true
time_to_live = 60
}
repository_storage = {
blob_store_name = "default"
strict_content_type_validation = true
}
}
resource "sonatyperepo_repository_ansiblegalaxy_proxy" "galaxy" {
name = "ansible-galaxy-proxy"
online = true
http_client = local.proxy_http_client
negative_cache = local.proxy_negative_cache
proxy = {
remote_url = "https://galaxy.ansible.com"
content_max_age = 1440
metadata_max_age = 60
}
storage = local.repository_storage
}
resource "sonatyperepo_repository_ansiblegalaxy_group" "public" {
name = "ansible-public"
online = true
group = {
member_names = [sonatyperepo_repository_ansiblegalaxy_proxy.galaxy.name]
}
storage = local.repository_storage
}
resource "sonatyperepo_repository_go_proxy" "public" {
name = "go-proxy"
online = true
http_client = local.proxy_http_client
negative_cache = local.proxy_negative_cache
proxy = {
remote_url = "https://proxy.golang.org"
content_max_age = 1440
metadata_max_age = 60
}
storage = local.repository_storage
}
resource "sonatyperepo_repository_go_group" "public" {
name = "go-public"
online = true
group = {
member_names = [sonatyperepo_repository_go_proxy.public.name]
}
storage = local.repository_storage
}
-75
View File
@@ -1,75 +0,0 @@
resource "sonatyperepo_privilege_repository_view" "anonymous_ansible" {
name = "ci-anonymous-ansible-read"
description = "Anonymous read access to the Ansible Galaxy group"
actions = ["BROWSE", "READ"]
format = "ansiblegalaxy"
repository = sonatyperepo_repository_ansiblegalaxy_group.public.name
}
resource "sonatyperepo_privilege_repository_view" "anonymous_ansible_proxy" {
name = "ci-anonymous-ansible-proxy-read"
description = "Anonymous artifact read access to the Ansible Galaxy proxy"
actions = ["BROWSE", "READ"]
format = "ansiblegalaxy"
repository = sonatyperepo_repository_ansiblegalaxy_proxy.galaxy.name
}
resource "sonatyperepo_privilege_repository_view" "anonymous_go" {
name = "ci-anonymous-go-read"
description = "Anonymous read access to the Go module group"
actions = ["BROWSE", "READ"]
format = "go"
repository = sonatyperepo_repository_go_group.public.name
}
resource "sonatyperepo_privilege_repository_view" "anonymous_oci_public" {
name = "ci-anonymous-oci-public-read"
description = "Anonymous read access to the public OCI group"
actions = ["BROWSE", "READ"]
format = "oci"
repository = "oci-public"
}
resource "sonatyperepo_privilege_repository_view" "anonymous_oci_proxy" {
name = "ci-anonymous-oci-proxy-read"
description = "Anonymous read access to the OCI proxy member"
actions = ["BROWSE", "READ"]
format = "oci"
repository = "oci-proxy"
}
resource "sonatyperepo_role" "anonymous_ci" {
id = "ci-anonymous-read"
name = "CI anonymous read"
description = "Read-only access to public CI dependency proxy groups"
privileges = [
sonatyperepo_privilege_repository_view.anonymous_ansible.name,
sonatyperepo_privilege_repository_view.anonymous_ansible_proxy.name,
sonatyperepo_privilege_repository_view.anonymous_go.name,
sonatyperepo_privilege_repository_view.anonymous_oci_public.name,
sonatyperepo_privilege_repository_view.anonymous_oci_proxy.name,
]
roles = []
}
resource "sonatyperepo_user" "anonymous" {
user_id = "anonymous"
first_name = "Anonymous"
last_name = "User"
email_address = "[email protected]"
status = "active"
roles = [sonatyperepo_role.anonymous_ci.id]
}
resource "sonatyperepo_system_anonymous_access" "ci" {
enabled = true
user_id = sonatyperepo_user.anonymous.user_id
realm_name = "NexusAuthorizingRealm"
}
resource "sonatyperepo_security_realms" "active" {
active = [
"NexusAuthenticatingRealm",
"OciBearerToken",
]
}
-33
View File
@@ -1,33 +0,0 @@
terraform {
required_version = ">= 1.11.0"
required_providers {
sonatyperepo = {
source = "sonatype-nexus-community/sonatyperepo"
version = "1.17.0"
}
}
}
provider "sonatyperepo" {
url = var.nexus_url
username = var.nexus_username
password = var.nexus_password
}
variable "nexus_url" {
description = "Nexus Repository base URL"
type = string
}
variable "nexus_username" {
description = "Nexus Terraform management username"
type = string
sensitive = true
}
variable "nexus_password" {
description = "Nexus Terraform management password"
type = string
sensitive = true
}
+4 -28
View File
@@ -70,11 +70,11 @@ configFiles:
},
"accessControl": {
"repositories": {
"panxiao81/backstage": {
"panxiao81/gitea-dynamic-runner-controller": {
"policies": [
{
"users": [
"spiffe://ddupan.top/ci/panxiao81/backstage/image",
"spiffe://ddupan.top/ci/panxiao81/gitea-dynamic-runner/publish-images",
"spiffe://ddupan.top/dev/panxiao81"
],
"actions": [
@@ -88,42 +88,18 @@ configFiles:
"read"
]
},
"panxiao81/gitea-dynamic-runner-controller": {
"policies": [
{
"users": [
"spiffe://ddupan.top/ci/panxiao81/gitea-dynamic-runner/publish-images"
],
"actions": [
"read",
"create",
"update"
]
},
{
"users": ["spiffe://ddupan.top/dev/panxiao81"],
"actions": ["read", "create", "update", "delete"]
}
],
"defaultPolicy": [
"read"
]
},
"panxiao81/gitea-dynamic-runner-runner": {
"policies": [
{
"users": [
"spiffe://ddupan.top/ci/panxiao81/gitea-dynamic-runner/publish-images"
"spiffe://ddupan.top/ci/panxiao81/gitea-dynamic-runner/publish-images",
"spiffe://ddupan.top/dev/panxiao81"
],
"actions": [
"read",
"create",
"update"
]
},
{
"users": ["spiffe://ddupan.top/dev/panxiao81"],
"actions": ["read", "create", "update", "delete"]
}
],
"defaultPolicy": [
-2
View File
@@ -53,6 +53,4 @@ sudo k3s kubectl -n flux-system get gitrepositories,kustomizations
- SPIRE 已按 hardened chart 内部 fork `0.30.2-ddupan.1`(基于上游 `0.30.2`,SPIRE
`1.15.3`)声明,使用共享
PostgreSQL 与独立 signing-key PVC;首次上线和 OpenBao JWT-SVID PoC 尚待合并后验证;
- Nexus Repository CE POC 已加入 GitOps 声明,计划验证 Ansible Galaxy、Go Modules 与
OCI/BuildKit 缓存;尚未部署或完成现场验收,现有 zot 保持不变;
- root Kustomization 与所有 brownfield 子 Kustomization 继续保持 `prune: false`。
-22
View File
@@ -1,22 +0,0 @@
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: backstage
namespace: flux-system
spec:
dependsOn:
- name: envoy-gateway
- name: external-secrets
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: backstage
namespace: backstage
interval: 10m
path: ./apps/backstage
prune: true
sourceRef:
kind: GitRepository
name: flux-system
timeout: 5m
wait: true
+2 -3
View File
@@ -6,12 +6,11 @@ metadata:
spec:
dependsOn:
- name: external-secrets
- name: nats
- name: spire
interval: 10m
path: ./platform/dynamic-runner
# The runner backends are replaceable. Prune is required when a retired
# worker is removed from the component; otherwise it keeps consuming work.
prune: true
prune: false
sourceRef:
kind: GitRepository
name: flux-system
@@ -1,17 +1,14 @@
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hydra
name: gitea-actions
namespace: flux-system
spec:
dependsOn:
- name: envoy-gateway
- name: external-secrets
interval: 10m
path: ./apps/hydra
path: ./platform/gitea-runner
prune: false
sourceRef:
kind: GitRepository
name: flux-system
timeout: 5m
wait: true
timeout: 3m
wait: false
-22
View File
@@ -1,22 +0,0 @@
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: nexus
namespace: flux-system
spec:
dependsOn:
- name: envoy-gateway
- name: openebs
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: nexus
namespace: nexus
interval: 10m
path: ./apps/nexus
prune: false
sourceRef:
kind: GitRepository
name: flux-system
timeout: 15m
wait: true
+1 -3
View File
@@ -7,7 +7,7 @@ resources:
- apps/envoy-gateway.yaml
- apps/external-secrets.yaml
- apps/gitea.yaml
- apps/backstage.yaml
- apps/gitea-actions.yaml
- apps/http-echo.yaml
- apps/openebs.yaml
- apps/nats.yaml
@@ -15,5 +15,3 @@ resources:
- apps/spire.yaml
- apps/observability.yaml
- apps/zot.yaml
- apps/nexus.yaml
- apps/hydra.yaml
+4 -15
View File
@@ -15,8 +15,7 @@ Root bootstrap 已完成。后续按依赖顺序分别引入:
1. 监控 CRD、kube-state-metrics 以及 kubelet/cAdvisor 抓取配置;
2. SPIRE Agent、SPIFFE CSI Driver 与 workload registration;
3. Kata Containers、`block-plain` RuntimeClass;
4. 独立 External Secrets Operator 与 sandbox 专用 OpenBao auth backend;
5. OpenSandbox controller/server;CI Pool 与 runner 调度器随后独立接入。
4. OpenSandbox operator/server 及 `ci-pod`、`ci-vm` Pools。
每一阶段单独合并并等待对应 Flux Kustomization Ready,不在 bootstrap 时一次性部署。
第一阶段监控拆为 `monitoring-operator` 与依赖它的 `monitoring`,防止 VM CR 在
@@ -43,25 +42,15 @@ attestation。Server 使用 external bundle publisher 持续维护 sandbox
显式关闭 Server 与 OIDC Provider,只部署 Agent DaemonSet 和 SPIFFE CSI Driver;因此
不会产生第二个 trust root。
`spire-smoke` namespace、ServiceAccount 和 `sandbox-spire-smoke` ClusterSPIFFEID 只用于
普通 Pod 的 CSI 回归夹具,稳定身份为 `spiffe://ddupan.top/sandbox/smoke`。Kata guest
不能复用 node Agent 暴露的 Unix socket;virtio-fs 只能呈现 socket 路径,不能把连接
跨过 VM 边界。Kata workload 必须使用 guest 内 Agent,具体约束见
`platform/sandbox-kata/README.md`。测试 Pod 临时创建并在验收后删除,普通 Pod 的身份
声明保留。
`spire-smoke` namespace、ServiceAccount 和 `sandbox-spire-smoke` ClusterSPIFFEID 是
普通 Pod 与后续 Kata guest 的回归夹具,稳定身份为
`spiffe://ddupan.top/sandbox/smoke`。测试 Pod 临时创建并在验收后删除,身份声明保留。
Kata 阶段使用官方 4.1.0 `kata-deploy` chart 的短生命周期 `job` 模式,逐节点安装并
重启 K3s。只启用 `kata-clh-runtime-rs`,不创建默认 `kata` 别名;该 handler 的
`emptyDir` 固定使用 `block-plain`,为 Docker/BuildKit overlay2 与 kind 提供 guest
内块设备文件系统。详细限制与上线验收见 `platform/sandbox-kata/README.md`。
Sandbox 的 ESO 通过独立 `auth/kubernetes-sandbox` 向 OpenBao 证明 ServiceAccount
身份,只能读取共享的 `kv/k8s/opensandbox-api`。它不保存 reviewer JWT 或长期 Bao token;相关
Terraform 与 Flux 边界见 `platform/sandbox-external-secrets/README.md`。
OpenSandbox 阶段固定官方源码 commit 与 umbrella chart `0.2.2`,只部署 controller、
ClusterIP server 和 CRD。API key 由 ESO 从 OpenBao 投影,明文不进入 Git。
## 监控边界
这里只管理 sandbox LXC 内的 Kubernetes 监控,不负责 PVE 宿主监控。LXC 与宿主共享
@@ -1,17 +0,0 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: external-secrets-operator
namespace: flux-system
spec:
dependsOn:
- name: monitoring-operator
interval: 10m
path: ./platform/sandbox-external-secrets/operator
prune: true
sourceRef:
kind: GitRepository
name: flux-system
timeout: 10m
wait: true
@@ -1,17 +0,0 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: external-secrets
namespace: flux-system
spec:
dependsOn:
- name: external-secrets-operator
interval: 10m
path: ./platform/sandbox-external-secrets/config
prune: true
sourceRef:
kind: GitRepository
name: flux-system
timeout: 10m
wait: true
@@ -2,14 +2,13 @@
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: ci-runners
name: opensandbox-pools
namespace: flux-system
spec:
dependsOn:
- name: opensandbox
- name: spire-agents
interval: 10m
path: ./platform/sandbox-ci-runners
path: ./platform/sandbox-opensandbox-pools
prune: true
sourceRef:
kind: GitRepository
+2 -3
View File
@@ -6,14 +6,13 @@ metadata:
namespace: flux-system
spec:
dependsOn:
- name: external-secrets
- name: kata
- name: monitoring-operator
- name: spire-agents
interval: 10m
path: ./platform/sandbox-opensandbox
prune: true
sourceRef:
kind: GitRepository
name: flux-system
timeout: 15m
timeout: 20m
wait: true
+1 -3
View File
@@ -7,7 +7,5 @@ resources:
- apps/spire-bootstrap.yaml
- apps/spire-agents.yaml
- apps/kata.yaml
- apps/external-secrets-operator.yaml
- apps/external-secrets.yaml
- apps/opensandbox.yaml
- apps/ci-runners.yaml
- apps/opensandbox-pools.yaml
-3
View File
@@ -18,10 +18,7 @@ homelab_dns:
- { zone: ad.ddupan.top, name: metrics-write, type: A, values: [192.168.10.127] }
- { zone: ad.ddupan.top, name: netbox, type: A, values: [192.168.10.127] }
- { zone: ad.ddupan.top, name: nats, type: A, values: [192.168.10.127] }
- { zone: ad.ddupan.top, name: nexus, type: A, values: [192.168.10.127] }
- { zone: ad.ddupan.top, name: s3, type: A, values: [192.168.10.127] }
- { zone: ad.ddupan.top, name: hydra, type: A, values: [192.168.10.127] }
- { zone: ad.ddupan.top, name: hydra-login, type: A, values: [192.168.10.127] }
- { zone: ad.ddupan.top, name: spire-oidc, type: A, values: [192.168.10.127] }
- { zone: ad.ddupan.top, name: spire-server, type: A, values: [192.168.10.127] }
- { zone: ad.ddupan.top, name: zot, type: A, values: [192.168.10.127] }
+1 -8
View File
@@ -51,7 +51,7 @@ few things Terraform must not own.**
| secrets engine mounts (`kv`, `pki`, `ssh-client-signer`) | the daemon, Raft, TLS files, systemd |
| PKI role, issuing/CRL URLs, cluster paths, **ACME** | `bao operator init` / unseal (manual, PGP-wrapped) |
| SSH signing role (`ai-agent`) | **PKI root CA + SSH CA signing key** |
| OIDC/Kubernetes auth mount、config 与 role | OIDC **client secret** (`auth/oidc/config`) |
| OIDC auth *mount* and *role* | OIDC **client secret** (`auth/oidc/config`) |
| all policies | snapshot token + script + systemd timer |
| | host-level CA trust distribution (`openbao_ssh_ca_trust`) |
@@ -92,13 +92,6 @@ then `VAULT_ADDR`/`VAULT_TOKEN`), mirroring how `smtp-relay/terraform` uses
native `openbao/openbao` provider is published only to the OpenTofu registry and
cannot be resolved by the HashiCorp `terraform` CLI.
Sandbox 集群使用独立的 `auth/kubernetes-sandbox`。其 API 地址、公开 Kubernetes CA、
ESO role 与只读 `kv/k8s/opensandbox-api` policy 全部由 Terraform 管理;CA 位于
`terraform/certs/sandbox-kubernetes-ca.crt`。集群重建并轮换 CA 后,先更新该文件并
apply,再让 Flux 恢复 ESO reconciliation。该 backend 不保存 reviewer JWT,而是使用
ESO 的短期登录 JWT 执行 TokenReview。该组资源已于 2026-09-18 apply,随后复验 plan
为 zero-diff。
## DNS
`bao.ad.ddupan.top` is an **internal-only** name — not a public Cloudflare record and
-30
View File
@@ -59,33 +59,3 @@ resource "vault_kubernetes_auth_backend_role" "external_secrets" {
# in a long TTL and every extra hour is a longer-lived credential in memory.
token_ttl = 3600
}
# A Kubernetes auth mount can validate identities from only the API server it is
# configured against. The sandbox cluster therefore cannot reuse auth/kubernetes,
# whose TokenReview endpoint belongs to homelab.
resource "vault_auth_backend" "sandbox_kubernetes" {
type = "kubernetes"
path = "kubernetes-sandbox"
}
resource "vault_kubernetes_auth_backend_config" "sandbox" {
backend = vault_auth_backend.sandbox_kubernetes.path
kubernetes_host = "https://10.60.0.13:6443"
kubernetes_ca_cert = file("${path.module}/certs/sandbox-kubernetes-ca.crt")
disable_local_ca_jwt = true
# Deliberately omit token_reviewer_jwt. OpenBao uses the login JWT for
# TokenReview; the sandbox external-secrets ServiceAccount is bound only to
# system:auth-delegator and all issued JWTs remain short-lived.
}
resource "vault_kubernetes_auth_backend_role" "sandbox_external_secrets" {
backend = vault_auth_backend.sandbox_kubernetes.path
role_name = "external-secrets"
bound_service_account_names = ["external-secrets"]
bound_service_account_namespaces = ["external-secrets"]
token_policies = [vault_policy.sandbox_external_secrets.name]
token_ttl = 3600
}
@@ -1,10 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIBdjCCAR2gAwIBAgIBADAKBggqhkjOPQQDAjAjMSEwHwYDVQQDDBhrM3Mtc2Vy
dmVyLWNhQDE3ODk2NTI4NTcwHhcNMjYwOTE3MTI0NzM3WhcNMzYwOTE0MTI0NzM3
WjAjMSEwHwYDVQQDDBhrM3Mtc2VydmVyLWNhQDE3ODk2NTI4NTcwWTATBgcqhkjO
PQIBBggqhkjOPQMBBwNCAAR4SbqzTXZnlZdUPz7viN6+dYbB1Maw44Qiepn9r5XG
sOzkYkN8t1aG3Ugo8TqQ3xJaKkM89n1Rluj0vbOhiNajo0IwQDAOBgNVHQ8BAf8E
BAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUyLSGoKAKAJuiniuRdBLG
XYaDQC8wCgYIKoZIzj0EAwIDRwAwRAIgFkVzyUZexk/ynnxBEOg+3foJv3WKqAei
hTSRjO1gL0UCIFbBKR7BMrJJAgW3DJFeeBM+b+tTg93jNx55qZACbFOL
-----END CERTIFICATE-----
@@ -35,10 +35,3 @@ resource "vault_policy" "external_secrets" {
name = "external-secrets"
policy = file("${path.module}/policies/external-secrets.hcl")
}
# The sandbox cluster has its own Kubernetes auth backend and a deliberately
# narrower KV view than the homelab ESO instance.
resource "vault_policy" "sandbox_external_secrets" {
name = "sandbox-external-secrets"
policy = file("${path.module}/policies/sandbox-external-secrets.hcl")
}
@@ -1,9 +0,0 @@
# Read only the shared OpenSandbox control-plane API key. The same Bao object is
# consumed by the server in sandbox and, later, by the scheduler in homelab.
path "kv/data/k8s/opensandbox-api" {
capabilities = ["read"]
}
path "kv/metadata/k8s/opensandbox-api" {
capabilities = ["read"]
}
@@ -83,11 +83,6 @@ vyos_postgresql_primary_address: "10.60.0.11"
vyos_sandbox_k3s_api_address: "10.60.0.13"
vyos_sandbox_k3s_api_interface: eth1
vyos_sandbox_k3s_api_port: 6443
vyos_opensandbox_api_port: 8080
vyos_opensandbox_node_port: 30080
# Lifecycle create is synchronous and a cold Kata image pull can exceed the
# HAProxy 50-second default. Keep this below OpenSandbox's overall timeout.
vyos_opensandbox_api_timeout: 600
vyos_sandbox_k3s_servers:
- name: sandbox1
address: "10.60.0.11"
@@ -15,20 +15,6 @@ set interfaces ethernet {{ i.iface }} address {{ i.address }}
set interfaces ethernet {{ i.iface }} description '{{ i.description }}'
{% endfor %}
{# OpenSandbox stays on a NodePort; VyOS owns the stable routed frontend. #}
set load-balancing haproxy service opensandbox-api listen-address {{ vyos_sandbox_k3s_api_address }}
set load-balancing haproxy service opensandbox-api port {{ vyos_opensandbox_api_port }}
set load-balancing haproxy service opensandbox-api mode 'tcp'
set load-balancing haproxy service opensandbox-api backend 'opensandbox-api'
set load-balancing haproxy service opensandbox-api timeout client {{ vyos_opensandbox_api_timeout }}
set load-balancing haproxy backend opensandbox-api mode 'tcp'
set load-balancing haproxy backend opensandbox-api timeout server {{ vyos_opensandbox_api_timeout }}
{% for server in vyos_sandbox_k3s_servers %}
set load-balancing haproxy backend opensandbox-api server {{ server.name }} address {{ server.address }}
set load-balancing haproxy backend opensandbox-api server {{ server.name }} port {{ vyos_opensandbox_node_port }}
set load-balancing haproxy backend opensandbox-api server {{ server.name }} check
{% endfor %}
{# --- default route out; OSPF carries the rest --- #}
set protocols static route 0.0.0.0/0 next-hop {{ vyos_lan_gateway }}
+2 -12
View File
@@ -18,8 +18,7 @@ Flux 管理以下 Kubernetes 资源:
- Kata Containers 和 CI 专用的 `block-plain` RuntimeClass;
- SPIRE Agent、SPIFFE CSI Driver 与 workload identity 声明;
- vmagent、kube-state-metrics、kubelet/cAdvisor scrape 配置和告警;
- sandbox External Secrets Operator、OpenSandbox controller/server;CI Pool 与 runner
调度器由 runner 项目接入。
- OpenSandbox operator/server、`ci-pod` 与 `ci-vm` Pools。
同一个对象只能有一个 owner。Ansible 不直接部署上述集群内 workload;Flux 不管理
LXC、K3s datastore 或 K3s 本身。
@@ -93,13 +92,6 @@ ansible-playbook site.yml
ansible-playbook k3s.yml
```
只 reconcile Flux controllers 与 root sync manifest(不触碰 LXC、PostgreSQL 或
K3s lifecycle):
```bash
ansible-playbook flux.yml
```
K3s 外部 datastore URI 由运行时 `SANDBOX_K3S_DB_PASSWORD` 生成,密码在 URI 中
进行 URL 编码,最终仅持久化于节点 root 可读的 `/etc/rancher/k3s/config.yaml`
(mode `0600`)。首节点生成的 K3s join token 仅在同一次 Ansible run 内传给第二节点;
@@ -114,9 +106,7 @@ ansible-playbook verify.yml
当前已经声明 LXC 生命周期、最小 OS baseline、PostgreSQL 和 K3s,包括系统级
homelab CA trust。Flux `v2.9.5` controllers 与 root sync 也由 Ansible 通过 K3s
server manifests 管理;root 使用 homelab CA 访问公开 Gitea 仓库,不保存 Git token。
集群内 workload 由 `clusters/sandbox/` 分阶段纳入 Flux。root Kustomization 的健康检查
timeout 为 40 分钟,用于覆盖 Kata 等首次安装时会逐节点重启 K3s 的子
Kustomization;各子项仍保留自己的更短 timeout,故障会在对应子项先行暴露。
集群内 workload 由 `clusters/sandbox/` 分阶段纳入 Flux。
## SPIRE 跨集群 bootstrap
@@ -1,6 +0,0 @@
---
- name: Reconcile Flux controllers and root sync
hosts: sandbox1
gather_facts: false
roles:
- sandbox_flux
@@ -35,5 +35,5 @@ spec:
sourceRef:
kind: GitRepository
name: flux-system
timeout: 40m
timeout: 3m
wait: true
+36 -92
View File
@@ -1,107 +1,51 @@
# Gitea dynamic runner
# Gitea dynamic runner controller
本目录部署单副本 Go controller,在同一进程运行 RunnerService scheduler、原生
Kubernetes Pod worker、OpenSandbox VM worker 和 SPIFFE mTLS facade:
此目录只管理 homelab 中的 controller 部署。controller、worker、Cloud Hypervisor
launcher 和 guest runner 的源码与发布位于独立仓库
`panxiao81/gitea-dynamic-runner`。
当前 bootstrap controller 接收 Gitea `workflow_job` webhook,将 `[self-hosted, pod]` 和
`[self-hosted, vm]` 的 queued job 分别发布到 NATS。Pod worker 在本集群创建一次性
privileged host runner;Docker、BuildKit 和 kind 由 workflow 自行 setup。内部
endpoint:
```text
Gitea RunnerService -> scheduler -> JetStream ci.runner.pod
|
v
homelab Kubernetes Pod
|
SPIFFE mTLS RunnerService facade
|
v
Gitea
http://dynamic-runner-controller.dynamic-runner.svc.cluster.local:8787/webhook
```
Pod backend 不经过 OpenSandbox。VM backend 后续启用时才访问 VyOS 暴露的
OpenSandbox Lifecycle API;本目录不修改 sandbox 平台侧 ESO、Bao Terraform 或
OpenSandbox chart 所有权边界。
OpenBao 路径:
## Canary 安全边界
- `kv/k8s/nats.ci_producer_password`:已有 NATS producer 密码。
- `kv/k8s/nats.ci_worker_password`:已有 NATS worker 密码。
- `kv/k8s/dynamic-runner.webhook_secret`:Gitea webhook HMAC secret。
- `kv/k8s/gitea-runner.token`:现有 instance runner registration token。
- Deployment 为单副本,滚动策略固定 `maxSurge: 1`、`maxUnavailable: 0`,保证新
facade Ready 后才终止旧实例。scheduler 必须通过 Kubernetes Lease 保持单 leader,
不能依赖 Recreate 避免重复领取。
- scheduler 使用一个 runner registration,并按总容量启动并发 `FetchTask` goroutine;
`POD_CAPACITY=4` 与 `VM_CAPACITY=1` 分别限制两个 durable consumer 和 backend
admission pool。池满时 assignment 保持 JetStream pending,任一 backend 不占用
另一方的执行槽位,也不会创建超出容量的 workload。
- rollout 重叠期间只有持有 `Lease/dynamic-runner-scheduler` 的 controller 执行
`FetchTask`;所有 Ready 实例都可通过 backend metadata 恢复 claim 并服务 facade。
- executor 镜像使用 digest;Pod 以 UID 2000 运行,SPIRE `ClusterStaticEntry` 同时绑定
具体 Pod UID 与 `unix:uid:2000`。
- facade 通过仅内网可路由的 `192.168.10.127:30443` NodePort 提供给 sandbox executor;
双方使用 Workload API X509-SVID mTLS,并按 SPIFFE ID 而不是 IP/DNS 名验证服务端。
该入口不经过公网或 Cloudflare Tunnel。
- controller 的 SPIFFE ID 固定为
`spiffe://ddupan.top/ns/dynamic-runner/sa/dynamic-runner-controller`。
- NATS 保留既有最小权限分离:`ci-producer` 仅 publish,`ci-worker` 仅 pull/ACK。
首期 controller 与 runner 镜像由 laptop 本机构建后导入 k3s containerd,作为 CI
发布链路建立前的 bootstrap。部署使用 `imagePullPolicy: Never`。正式发布 workflow
获得专用 SPIFFE ID 后,必须将 image 改为 zot digest 并移除本地导入步骤。
facade claim registry 在启动时从 Pod/OpenSandbox metadata 恢复;scheduler leadership
由 Kubernetes Lease 持久化协调。Deployment 仍保持 `replicas: 1`,滚动更新期间允许
一个额外 Pod 提供 facade 连续性。
## 身份绑定
## Secret 边界
queued webhook 只负责创建没有业务身份的 Pod。runner 实际领取任务后,Gitea 的
`in_progress` webhook 会携带实际 `runner_name`;controller 将 binding 消息发布到
NATS,Pod worker 再给对应 Pod 添加:
`ExternalSecret/dynamic-runner` 从既有 `ClusterSecretStore/openbao` 读取:
- `kv/k8s/nats`:producer/worker 密码;
- `kv/k8s/opensandbox-api:api_key`:保留给后续 VM worker;
- `kv/k8s/gitea-runner:token`:保留的 runner registration token;
- `kv/k8s/dynamic-runner`:scheduler UUID/token、facade HMAC key 和回滚所需 webhook secret。
scheduler credential 由官方 Gitea Runner v3.5.0 一次注册生成;它只挂载到 controller,
不会进入 executor。facade capability key 至少 32 字节,controller 为每个 assignment
确定性生成独立 capability。不要打印 Secret、创建静态 Bao token或把 credential 写入
Git。OpenBao 写入使用本机 SPIFFE JWT 换取的短期 `local-development` token。
## Workflow 身份与依赖配置
workflow 可复用 [`panxiao81/ci-actions`](https://git.ddupan.top/panxiao81/ci-actions)
中的 `spiffe-openbao-login@v1` 和 `setup-nexus@v1`。这不改变 runner 的权限边界:
runner 仅提供 Node.js 20、`spire-agent` 与 Workload API socket,workflow 负责声明 Bao
role、audience 和具体用途,目标服务 policy 决定是否授权。短期 Bao token 会进入
Actions job 临时文件,因此这些 Action 只允许在本目录管理的一次性 Pod/VM executor
中使用,不能迁移到共享或持久 runner。
匿名读取 Nexus public repository 只需 `setup-nexus@v1`,不应为了依赖下载额外申请
Bao 凭据;需要发布制品时再为对应 repository 建立独立 service account 与最小权限
policy。
## 首次验收
合并后先观察 Flux 与 controller,不要立即开启 VM:
```bash
flux reconcile kustomization dynamic-runner --with-source
kubectl -n dynamic-runner wait externalsecret/dynamic-runner \
--for=condition=Ready --timeout=2m
kubectl -n dynamic-runner rollout status deploy/dynamic-runner-controller --timeout=5m
kubectl -n dynamic-runner logs deploy/dynamic-runner-controller -f
```text
ci.ddupan.top/identity-bound=true
ci.ddupan.top/spiffe-path=<owner>/<repository>/<percent-encoded-job-name>
```
确认 scheduler 只领取一条 `[self-hosted,pod]` task,然后验证:
`ClusterSPIFFEID/gitea-dynamic-runner` 只匹配已经绑定的 Pod,并签发
`spiffe://ddupan.top/ci/<owner>/<repository>/<job-name>`。runner 的 job-start hook 在
SVID 可用之前不会放行第一步,因此不能根据 queued 事件错配身份。
1. assignment message 进入并离开 durable `pod` consumer;
2. `gitea-task-<task-id>` Pod 创建,取得实际 Pod UID;
3. 同名 `ClusterStaticEntry` 的 parent ID 包含该 UID,SPIFFE ID 使用 repository/job key;
4. 官方 runner v3.5.0 经 facade claim 精确 task,Gitea 实时收到日志和终态;
5. Pod consumer 达到 capacity 时 VM consumer 仍可独立接受任务。
每个 runner Pod 使用 `gitea-dynamic-runner` ServiceAccount。该 ServiceAccount 没有
Kubernetes API 权限;只有 `dynamic-runner-pod-worker` ServiceAccount 能在本 namespace
create/get/patch/delete Pod。
Pod 与 VM 都在 Gitea 接受终态后先把 terminal marker 写入各自 backend metadata,再由
lifecycle reconciler 删除执行器。首次 VM 测试仍须观察 BatchSandbox、Pod 与
ClusterStaticEntry 全部消失;完整自动清理通过前不得提高 `POD_CAPACITY` 或
`VM_CAPACITY`。
长期实现将由兼容 Gitea RunnerService 的 scheduler 直接领取 task,再交给 Pod/VM
executor;届时删除 webhook、临时 runner 注册和 identity binding 消息。跟踪见
`panxiao81/gitea-dynamic-runner` issue #7。
VM backend 已通过 `vm-dev` canary 完成 Docker、kind、SPIFFE 和完整生命周期测试。
生产调度标签为 `[self-hosted, vm]`,初始保持 `VM_CAPACITY=1`;扩容前先观察实际任务的
资源水位、等待时间以及 OpenSandbox 是否存在 terminal sandbox 残留。
## 回滚
若 controller 在领取 task 前失败,回滚到前一 commit 的 Python controller/Pod worker
manifests。若已经创建 `gitea-task-*` Pod,先保留现场并核对 Gitea task 状态,不能直接
重启 scheduler 造成重复执行。scheduler registration 和 capability key 保留在 Bao,
回滚不需要删除或打印它们。
Gitea webhook 只订阅 `workflow_job`,content type 使用 JSON,secret 与 Bao 中值
一致。不要启用 `send_everything`,否则 controller 会收到无关仓库事件。
+5 -4
View File
@@ -1,16 +1,17 @@
apiVersion: spire.spiffe.io/v1alpha1
kind: ClusterSPIFFEID
metadata:
name: dynamic-runner-controller
name: gitea-dynamic-runner
spec:
className: spire-mgmt-spire
spiffeIDTemplate: spiffe://ddupan.top/ns/dynamic-runner/sa/dynamic-runner-controller
spiffeIDTemplate: 'spiffe://{{ .TrustDomain }}/ci/{{ index .PodMeta.Annotations "ci.ddupan.top/spiffe-path" }}'
namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: dynamic-runner
podSelector:
matchLabels:
app.kubernetes.io/name: dynamic-runner-controller
app.kubernetes.io/name: gitea-dynamic-runner
ci.ddupan.top/identity-bound: "true"
workloadSelectorTemplates:
- k8s:ns:dynamic-runner
- k8s:sa:dynamic-runner-controller
- k8s:sa:gitea-dynamic-runner
+28 -82
View File
@@ -5,11 +5,6 @@ metadata:
namespace: dynamic-runner
spec:
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
selector:
matchLabels:
app.kubernetes.io/name: dynamic-runner-controller
@@ -19,10 +14,17 @@ spec:
app.kubernetes.io/name: dynamic-runner-controller
spec:
serviceAccountName: dynamic-runner-controller
automountServiceAccountToken: false
initContainers:
- name: fetch-internal-ca
image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6
args: [--fail, --silent, --show-error, --output, /trust/ca.pem, https://bao.ad.ddupan.top:8200/v1/pki/ca/pem]
args:
- --fail
- --silent
- --show-error
- --output
- /trust/ca.pem
- https://bao.ad.ddupan.top:8200/v1/pki/ca/pem
securityContext:
allowPrivilegeEscalation: false
capabilities:
@@ -38,89 +40,40 @@ spec:
mountPath: /trust
containers:
- name: controller
image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:7374a08a238ca1c76a9aee33c57520ae0edaa9b5b443708487d5b3a69a61da4e
imagePullPolicy: IfNotPresent
args: [controller]
# Bootstrap import on laptop. Replace with a zot digest after the
# repository's image publishing workflow has a dedicated identity.
image: gitea-dynamic-runner-controller:0.3.0-bootstrap
imagePullPolicy: Never
env:
- name: COMPONENTS
value: scheduler,pod-worker,vm-worker
- name: GITEA_INSTANCE_URL
value: https://git.ddupan.top
- name: GITEA_RUNNER_UUID_FILE
value: /run/dynamic-runner-secrets/scheduler-uuid
- name: GITEA_RUNNER_TOKEN_FILE
value: /run/dynamic-runner-secrets/scheduler-token
- name: NATS_URL
value: tls://nats.ad.ddupan.top:4222
- name: NATS_CA_FILE
value: /run/trust/ca.pem
- name: NATS_PRODUCER_PASSWORD_FILE
- name: NATS_PASSWORD_FILE
value: /run/dynamic-runner-secrets/nats-password
- name: NATS_WORKER_PASSWORD_FILE
value: /run/dynamic-runner-secrets/nats-worker-password
- name: RUNNER_FACADE_CAPABILITY_KEY_FILE
value: /run/dynamic-runner-secrets/facade-capability-key
- name: RUNNER_FACADE_LISTEN
value: :8443
- name: RUNNER_FACADE_URL
value: https://192.168.10.127:30443
- name: RUNNER_FACADE_SPIFFE_ID
value: spiffe://ddupan.top/ns/dynamic-runner/sa/dynamic-runner-controller
- name: SPIFFE_ENDPOINT_SOCKET
value: unix:///run/spire/agent-sockets/spire-agent.sock
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_EXECUTOR_IMAGE
value: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:325d75a208b1a1c6f3b4d4705e47bbc58b34733edcd12f689c60769ab6772a59
- name: POD_SERVICE_ACCOUNT
value: gitea-dynamic-runner
- name: POD_EXECUTOR_UID
value: "2000"
- name: POD_CAPACITY
value: "4"
- name: OPENSANDBOX_API
value: http://10.60.0.13:8080
- name: OPENSANDBOX_API_KEY_FILE
value: /run/dynamic-runner-secrets/opensandbox-api-key
- name: OPENSANDBOX_POOL
value: ci-vm
- name: VM_CAPACITY
value: "1"
- name: VM_RUNNER_LABEL
value: vm
- name: VM_TIMEOUT_SECONDS
value: "14400"
- name: SPIRE_CLUSTER
value: homelab
- name: SPIRE_CLASS
value: spire-mgmt-spire
- name: SPIRE_AGENT_ID
value: spiffe://ddupan.top/spire/agent/k8s_psat/homelab/cd2d0233-c4ea-4031-8327-e7e359e766dd
- name: SSL_CERT_FILE
value: /run/trust/ca.pem
- name: WEBHOOK_SECRET_FILE
value: /run/dynamic-runner-secrets/webhook-secret
ports:
- name: facade
containerPort: 8443
- name: http
containerPort: 8787
readinessProbe:
httpGet: null
tcpSocket:
port: facade
httpGet:
path: /healthz
port: http
periodSeconds: 5
livenessProbe:
httpGet: null
tcpSocket:
port: facade
initialDelaySeconds: 15
httpGet:
path: /healthz
port: http
initialDelaySeconds: 10
periodSeconds: 10
resources:
requests:
cpu: 50m
memory: 64Mi
cpu: 25m
memory: 32Mi
limits:
cpu: 500m
memory: 256Mi
cpu: 250m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
@@ -138,9 +91,6 @@ spec:
- name: trust
mountPath: /run/trust
readOnly: true
- name: spire-agent-socket
mountPath: /run/spire/agent-sockets
readOnly: true
securityContext:
fsGroup: 65532
fsGroupChangePolicy: OnRootMismatch
@@ -154,7 +104,3 @@ spec:
- name: trust
emptyDir:
sizeLimit: 1Mi
- name: spire-agent-socket
csi:
driver: csi.spiffe.io
readOnly: true
@@ -20,10 +20,6 @@ spec:
remoteRef:
key: k8s/nats
property: ci_worker_password
- secretKey: opensandbox-api-key
remoteRef:
key: k8s/opensandbox-api
property: api_key
- secretKey: webhook-secret
remoteRef:
key: k8s/dynamic-runner
@@ -32,15 +28,3 @@ spec:
remoteRef:
key: k8s/gitea-runner
property: token
- secretKey: scheduler-uuid
remoteRef:
key: k8s/dynamic-runner
property: scheduler_uuid
- secretKey: scheduler-token
remoteRef:
key: k8s/dynamic-runner
property: scheduler_token
- secretKey: facade-capability-key
remoteRef:
key: k8s/dynamic-runner
property: facade_capability_key
@@ -6,4 +6,5 @@ resources:
- rbac.yaml
- clusterspiffeid.yaml
- deployment.yaml
- pod-worker-deployment.yaml
- service.yaml
@@ -0,0 +1,100 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dynamic-runner-pod-worker
namespace: dynamic-runner
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dynamic-runner-pod-worker
template:
metadata:
labels:
app.kubernetes.io/name: dynamic-runner-pod-worker
spec:
serviceAccountName: dynamic-runner-pod-worker
initContainers:
- name: fetch-internal-ca
image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6
args:
- --fail
- --silent
- --show-error
- --output
- /trust/ca.pem
- https://bao.ad.ddupan.top:8200/v1/pki/ca/pem
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 101
runAsGroup: 102
seccompProfile:
type: RuntimeDefault
volumeMounts:
- name: trust
mountPath: /trust
containers:
- name: pod-worker
image: gitea-dynamic-runner-controller:0.3.0-bootstrap
imagePullPolicy: Never
command: [/venv/bin/gitea-dynamic-runner-pod-worker]
env:
- name: NATS_URL
value: tls://nats.ad.ddupan.top:4222
- name: NATS_CA_FILE
value: /run/trust/ca.pem
- name: NATS_PASSWORD_FILE
value: /run/dynamic-runner-secrets/nats-worker-password
- name: RUNNER_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: RUNNER_IMAGE
value: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:4c61f6315453d68a827ee9542f1345ed86576eaaf62325eb803c8fe3f06ddf2a
- name: RUNNER_SERVICE_ACCOUNT
value: gitea-dynamic-runner
- name: RUNNER_TOKEN_SECRET
value: dynamic-runner
- name: RUNNER_CAPACITY
value: "4"
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 250m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 65532
runAsGroup: 65532
seccompProfile:
type: RuntimeDefault
volumeMounts:
- name: secret
mountPath: /run/dynamic-runner-secrets
readOnly: true
- name: trust
mountPath: /run/trust
readOnly: true
securityContext:
fsGroup: 65532
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
volumes:
- name: secret
secret:
secretName: dynamic-runner
defaultMode: 0400
- name: trust
emptyDir:
sizeLimit: 1Mi
+11 -30
View File
@@ -6,6 +6,12 @@ metadata:
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: dynamic-runner-pod-worker
namespace: dynamic-runner
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: gitea-dynamic-runner
namespace: dynamic-runner
@@ -13,48 +19,23 @@ metadata:
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: dynamic-runner-controller
name: dynamic-runner-pod-worker
namespace: dynamic-runner
rules:
- apiGroups: [""]
resources: [pods]
verbs: [create, get, list, watch, patch, delete]
- apiGroups: [coordination.k8s.io]
resources: [leases]
verbs: [create, get, list, watch, update, patch]
verbs: [create, get, patch, delete]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: dynamic-runner-controller
name: dynamic-runner-pod-worker
namespace: dynamic-runner
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: dynamic-runner-controller
name: dynamic-runner-pod-worker
subjects:
- kind: ServiceAccount
name: dynamic-runner-controller
namespace: dynamic-runner
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: dynamic-runner-spiffe-entries
rules:
- apiGroups: [spire.spiffe.io]
resources: [clusterstaticentries]
verbs: [create, get, delete]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: dynamic-runner-spiffe-entries
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: dynamic-runner-spiffe-entries
subjects:
- kind: ServiceAccount
name: dynamic-runner-controller
name: dynamic-runner-pod-worker
namespace: dynamic-runner
+3 -5
View File
@@ -4,11 +4,9 @@ metadata:
name: dynamic-runner-controller
namespace: dynamic-runner
spec:
type: NodePort
selector:
app.kubernetes.io/name: dynamic-runner-controller
ports:
- name: facade
port: 8443
targetPort: facade
nodePort: 30443
- name: http
port: 8787
targetPort: http
+114
View File
@@ -0,0 +1,114 @@
# Gitea Actions runner
This is the bootstrap runner for Gitea Actions. One persistent runner Pod accepts
up to four jobs; each job runs in a dynamically created container inside a
Docker-in-Docker daemon. The official chart runs DinD privileged. Rootless DinD
would still be privileged and is blocked by the node's AppArmor user-namespace
policy, so this deployment uses regular DinD instead of weakening that host-wide
policy. Only trusted workflows may target this runner.
DinD 同时使用 `--mtu=1450` 和
`--default-network-opt=bridge=com.docker.network.driver.mtu=1450`,与 k3s Pod 的
`eth0` 一致。前者只覆盖 Docker 默认 bridge;act 为每个 job 创建 user-defined
bridge,必须由后者设置默认 MTU。不要在未验证节点 Pod MTU 的情况下删除或修改这
两个参数:MTU 1500 的 job 容器虽然能够解析 GitHub、甚至建立 TCP 连接,但较大的
TLS 数据包会在嵌套网络路径中丢失,表现为 `github.com` / `api.github.com` 超时或
`setup-go` 每次请求卡满 6 分钟后重试。Pod 网络和默认 Docker bridge 正常不代表
Actions job bridge 正常。
The runner is registered at instance scope so it is available to every repository
on this Gitea instance. Repository permissions and protected-branch review are
therefore the security boundary; do not enable Actions for untrusted repositories.
The runner registration token is authoritative in OpenBao at
`kv/k8s/gitea-runner`. External Secrets Operator projects its `token` property to
the `gitea-runner-token` Secret. Never put the token in this directory or a Helm
command line.
## SPIRE 与 OCI 发布
runner Pod 使用专用 ServiceAccount `gitea-actions`,并由精确匹配 namespace、
ServiceAccount 隐含的 Pod、以及 chart labels 的 `ClusterSPIFFEID` 获得:
```text
spiffe://ddupan.top/ci/gitea-actions
```
SPIFFE CSI socket 同时只读挂载到 runner 和 DinD。act 的 volume allowlist 只允许
`/run/spire/agent-sockets`;workflow 仍必须在 job container 中显式请求该 bind
mount。原因是 bind mount 由 DinD 内的 dockerd 解析,只挂 runner 容器无法让 job
访问 Workload API。
该身份不是通用 registry 管理员。zot 只对明确列出的 CI 镜像仓库授予
`read/create/update`,不授予 delete 或其他仓库写入。workflow 应获取
`aud=zot` 的短期 JWT-SVID,并经 stdin 传给 registry client,不得把 JWT、X.509
SVID 或 Docker auth 写入 workspace/artifact。
## Flux 接管状态
该 release 最初通过下述 review-first 流程手动 bootstrap。下一个 GitOps 阶段将
使用 Flux `HelmRelease` 接管它,并首先固定现有 chart `0.1.1`,不在接管 PR 中升级。
迁移前审计发现:Helm 保存的 user-supplied values 和 release manifest 仍描述失败的
rootless DinD 尝试,但 live StatefulSet 与本目录 `values.yaml` 都已经使用 regular
DinD。首次 reconcile 的验收条件是修正 Helm 存储状态,同时 live Pod spec、PVC
identity、runner capacity 和在线状态保持不变。接管稳定后再用独立 PR 升级 chart。
接管分两阶段:第一阶段提交 `suspend: true` 的 HelmRelease、HelmRepository 和由
`values.yaml` 生成的 ConfigMap。Flux 只登记这些对象,不执行 Helm action。合并后
检查 HelmRepository Ready,并用固定 chart 重复比较期望清单与 live StatefulSet;
第二阶段解除 suspend。第一阶段已经确认 source Ready、完整 chart render 与 live
资源零差异,且登记过程中现有 runner 没有 rollout。失败重试使用
`RetryOnFailure`,不会用 stored rootless release 做 rollback。
## 历史 review-first bootstrap
这是 Flux 安装前执行过的一次性手动部署流程,保留用于恢复和审计:
1. Merge the reviewed PR.
2. As a Gitea site administrator, create an instance-scoped runner registration
token under **Site Administration → Actions → Runners**.
3. Store it as the `token` property at `kv/k8s/gitea-runner` without exposing it
in shell history:
```bash
read -rsp 'Runner token: ' runner_token
printf '%s' "$runner_token" | bao kv put kv/k8s/gitea-runner token=-
unset runner_token
```
4. From the updated `main`, create the namespace and ExternalSecret, then wait
for `SecretSynced=True`:
```bash
KUBECONFIG="$HOME/.kube/config" k3s kubectl apply \
-f platform/gitea-runner/namespace.yaml
KUBECONFIG="$HOME/.kube/config" k3s kubectl apply \
-f platform/gitea-runner/external-secret.yaml
KUBECONFIG="$HOME/.kube/config" k3s kubectl wait \
--namespace gitea-actions \
--for=condition=Ready externalsecret/gitea-runner-token \
--timeout=60s
```
5. Install chart `actions` version `0.1.1` from
`https://dl.gitea.com/charts/` with this `values.yaml`:
```bash
helm repo add gitea-charts https://dl.gitea.com/charts/
helm repo update gitea-charts
helm upgrade --install gitea-actions gitea-charts/actions \
--namespace gitea-actions \
--version 0.1.1 \
--values platform/gitea-runner/values.yaml \
--wait --timeout 10m
```
6. Confirm the runner is online, then re-run the queued lint workflow.
Do not deploy from an unmerged feature branch. Do not use `--set` for the token.
The 1 GiB PVC preserves `.runner` identity. Docker image layers are ephemeral;
the Pod has a 20 GiB ephemeral-storage limit. Terraform apply jobs must use a
workflow concurrency group because runner capacity does not serialize access to
a shared state.
@@ -0,0 +1,17 @@
apiVersion: spire.spiffe.io/v1alpha1
kind: ClusterSPIFFEID
metadata:
name: gitea-actions
spec:
className: spire-mgmt-spire
spiffeIDTemplate: spiffe://{{ .TrustDomain }}/ci/gitea-actions
namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: gitea-actions
podSelector:
matchLabels:
app.kubernetes.io/instance: gitea-actions
app.kubernetes.io/name: actions-runner
workloadSelectorTemplates:
- k8s:ns:gitea-actions
- k8s:sa:gitea-actions
@@ -1,19 +1,18 @@
---
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: opensandbox-api-key
namespace: opensandbox-system
name: gitea-runner-token
namespace: gitea-actions
spec:
refreshInterval: 1h
secretStoreRef:
kind: ClusterSecretStore
name: openbao
target:
name: opensandbox-api-key
creationPolicy: Owner
name: gitea-runner-token
data:
- secretKey: api-key
- secretKey: token
remoteRef:
key: k8s/opensandbox-api
property: api_key
key: k8s/gitea-runner
property: token
@@ -1,35 +1,31 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: external-secrets
namespace: external-secrets
name: gitea-actions
namespace: gitea-actions
spec:
chart:
spec:
chart: external-secrets
chart: actions
interval: 1h
sourceRef:
kind: HelmRepository
name: external-secrets
version: 2.8.0
name: gitea-charts
version: 0.1.1
driftDetection:
mode: enabled
install:
crds: CreateReplace
strategy:
name: RetryOnFailure
retryInterval: 5m
interval: 30m
releaseName: external-secrets
targetNamespace: external-secrets
releaseName: gitea-actions
targetNamespace: gitea-actions
timeout: 10m
upgrade:
crds: CreateReplace
strategy:
name: RetryOnFailure
retryInterval: 5m
valuesFrom:
- kind: ConfigMap
name: external-secrets-values
valuesKey: values.yaml
name: gitea-actions-values
@@ -0,0 +1,8 @@
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: gitea-charts
namespace: gitea-actions
spec:
interval: 1h
url: https://dl.gitea.com/charts/
+18
View File
@@ -0,0 +1,18 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
generatorOptions:
disableNameSuffixHash: true
labels:
reconcile.fluxcd.io/watch: Enabled
configMapGenerator:
- name: gitea-actions-values
namespace: gitea-actions
files:
- values.yaml=values.yaml
resources:
- namespace.yaml
- serviceaccount.yaml
- clusterspiffeid.yaml
- external-secret.yaml
- helmrepository.yaml
- helmrelease.yaml
@@ -1,5 +1,4 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: external-secrets
name: gitea-actions
@@ -1,6 +1,6 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: backstage
namespace: backstage
name: gitea-actions
namespace: gitea-actions
automountServiceAccountToken: false
+81
View File
@@ -0,0 +1,81 @@
enabled: true
giteaRootURL: http://gitea-http.gitea.svc.cluster.local:3000
existingSecret: gitea-runner-token
existingSecretKey: token
statefulset:
replicas: 1
timezone: Etc/UTC
serviceAccountName: gitea-actions
extraVolumes:
- name: spiffe-workload-api
csi:
driver: csi.spiffe.io
readOnly: true
securityContext:
fsGroup: 1000
# Chart 0.1.1 applies this block to both runner and DinD containers.
resources:
requests:
cpu: 250m
memory: 512Mi
ephemeral-storage: 2Gi
limits:
cpu: "4"
memory: 6Gi
ephemeral-storage: 20Gi
persistence:
size: 1Gi
runner:
registry: docker.io
repository: gitea/runner
tag: 2.3.0
pullPolicy: IfNotPresent
extraVolumeMounts:
- name: spiffe-workload-api
mountPath: /run/spire/agent-sockets
readOnly: true
config: |
log:
level: info
runner:
file: .runner
capacity: 4
timeout: 3h
shutdown_timeout: 3h
labels:
- self-hosted:docker://docker.gitea.com/runner-images:ubuntu-latest
cache:
enabled: false
container:
require_docker: true
docker_timeout: 300s
# Workflows must still request this exact bind mount explicitly. The
# allowlist prevents arbitrary host paths from reaching job containers.
valid_volumes:
- /run/spire/agent-sockets
dind:
# The node enforces AppArmor's unprivileged-userns restriction, which blocks
# rootlesskit even though this chart must run DinD privileged either way.
rootless: false
registry: docker.io
repository: docker
tag: 29.7.1-dind
pullPolicy: IfNotPresent
# Bind mounts are resolved by dockerd, so the CSI socket must exist in the
# DinD container as well as in the runner container.
extraVolumeMounts:
- name: spiffe-workload-api
mountPath: /run/spire/agent-sockets
readOnly: true
# k3s uses a 1450-byte pod MTU. Without matching it here, nested Actions
# networks advertise 1500 and GitHub TLS packets disappear on the outer
# overlay path while direct pod traffic remains healthy.
extraArgs:
- --mtu=1450
# --mtu only changes Docker's default bridge. act creates a user-defined
# bridge per job, so give every new bridge the same explicit default.
- --default-network-opt=bridge=com.docker.network.driver.mtu=1450
-84
View File
@@ -1,84 +0,0 @@
# OpenSandbox Gitea runner Pools
本目录只部署零预热的 `ci-vm` Pool、sandbox 内的 SPIFFE identity controller,以及仅供
内网 VyOS 转发的 OpenSandbox NodePort。`ci-vm` 使用 `kata-clh-runtime-rs`,每个 VM
执行单一任务并在结束后删除。普通 Pod job 直接运行在 homelab,不进入 OpenSandbox。
Pool 中 task-executor 接收 Lifecycle API 下发的进程环境。guest-local SPIRE Agent 用
Pod-bound PSAT 向中央 SPIRE 注册;identity controller 从 BatchSandbox allocation
取得真实 Pod UID,再创建精确的 `ClusterStaticEntry`。runner 只有拿到请求中的完整
repository/task SVID 后才领取一次性 Gitea registration token。
VM 与 Pod backend 使用同一个 `gitea-dynamic-runner-runner` executor 镜像。Pool 不运行
常驻 `docker:dind` sidecar;需要 Docker 的 workflow 在 privileged executor 内按任务启动
daemon。Kata VM 中的 workflow 必须先把稀疏 ext4 镜像 loop-mount 到
`/var/lib/docker`,并完成 cgroup v2 nesting 初始化。
`ci-vm` executor 固定请求并限制为 2 CPU/3GiB。真实 kind canary 表明 1 CPU/约 2GiB
虽然能最终启动全部 control-plane 容器,但无法在 kubeadm 超时前提供可用的 API server;
该规格是 nested Kubernetes 任务的容量下限,不是用资源掩盖存储阻塞。
这些 `ClusterStaticEntry` 位于 sandbox 集群,由 central SPIRE Server 内的
`spire-controller-manager-sandbox` 通过受限 external kubeconfig reconcile。必须在
`platform/spire/values.yaml` 显式启用 external controller-manager 的
`reconcile.clusterStaticEntries`(chart 默认关闭);仅看到 CR 存在但没有 status,不算
身份链路就绪。
## 部署依赖
- OpenSandbox chart 和 CRD 已 Ready;
- RuntimeClass `kata-clh-runtime-rs` 已存在;
- 中央 SPIRE 已发布 `ConfigMap/opensandbox/spire-bundle-pem`;
- VyOS `10.60.0.13:8080` 转发 sandbox1/2 的 NodePort `30080`;
- runner/controller 镜像均使用 Zot digest,而不是可变 tag。
本目录不读取 OpenBao,也不修改 OpenSandbox 平台侧 ExternalSecret、ClusterSecretStore
或 Bao policy。OpenSandbox API key 只存在于平台 server Secret 和 homelab controller
Secret,两边由各自身份读取同一 Bao 资源。
## 上线验收
先确认 Secret 和 API 认证,命令不得输出 key:
```bash
kubectl -n dynamic-runner wait externalsecret/dynamic-runner --for=condition=Ready --timeout=2m
kubectl -n dynamic-runner exec deploy/dynamic-runner-controller -- \
wget -qO- http://10.60.0.13:8080/health
```
然后触发一个 `runs-on: [self-hosted, vm]` 的最小 workflow,并同时观察:
```bash
kubectl -n dynamic-runner logs deploy/dynamic-runner-controller -f
kubectl -n opensandbox get pool,batchsandbox,pod -w
kubectl get clusterstaticentry -l app.kubernetes.io/component=opensandbox-identity -w
kubectl -n opensandbox logs deploy/opensandbox-identity -f
```
合格证据必须同时包含:
1. Lifecycle create 成功并选择 `ci-vm`;
2. 分配 Pod 的 `runtimeClassName` 为 `kata-clh-runtime-rs`;
3. entry 的 parent ID 包含该 Pod 的 UID,SPIFFE ID 使用 repository/task;
4. Gitea 显示临时 runner 带 `self-hosted,vm` labels 并完成真实任务;
5. 任务后 BatchSandbox、Pod、ClusterStaticEntry 和临时 runner 均消失。
清理超时对象时只调用 Lifecycle DELETE,不直接删除 Pool Pod。若 controller 已不可用,
可从 OpenSandbox API 按 metadata 定位 sandbox 后执行 DELETE;不要绕过 API 伪造状态。
## 故障定位
- `401`:检查 homelab ExternalSecret Ready 和文件挂载,不打印 Secret;
- `PoolCapacityExhausted`:检查 `ci-vm` 的 `poolMax` 及残留 BatchSandbox;
- runner 等待 SVID:核对 allocation Pod UID、ClusterStaticEntry parentID、guest Agent 日志;
- runner 等待 token:核对 `192.168.10.127:8787` 的 sandbox 到 homelab 路由;
- Docker 任务失败:检查 workflow 的 job-local dockerd 日志及
`/var/run/docker.sock`;Pool 不提供共享或常驻 daemon。
- kind node 的 systemd 报 `Failed to create /init.scope` 或 `Structure needs
cleaning`:确认 workflow 启动 dockerd 前完成 cgroup v2 nesting 初始化。否则
子容器的私有 cgroup namespace 根会变成 `threaded`,systemd 无法创建 domain
cgroup。
- kind 的 kubeadm 卡在 `CreateContainer`,而 nested containerd goroutine 停在
`bbolt` 的 `fdatasync`:不要直接把 Kata shared mount 用作 Docker 数据目录。
workflow 必须在 guest 内创建稀疏 ext4 镜像并 loop-mount 到
`/var/lib/docker`;该镜像随 Sandbox 删除,不得在节点侧遗留。
@@ -1,31 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: gitea-ci-spire-agent
namespace: opensandbox
data:
agent.conf: |
agent {
data_dir = "/run/spire/data"
log_level = "INFO"
server_address = "spire-server.ad.ddupan.top"
server_port = "8081"
socket_path = "/run/spire/agent-sockets/spire-agent.sock"
trust_bundle_path = "/run/spire/bundle/bundle.pem"
trust_domain = "ddupan.top"
}
plugins {
NodeAttestor "k8s_psat" {
plugin_data {
cluster = "sandbox-kata"
token_path = "/run/spire/tokens/token"
}
}
KeyManager "memory" {
plugin_data {}
}
WorkloadAttestor "unix" {
plugin_data {}
}
}
@@ -1,41 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: opensandbox-identity
namespace: opensandbox
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: opensandbox-identity
template:
metadata:
labels:
app.kubernetes.io/name: opensandbox-identity
spec:
serviceAccountName: opensandbox-identity
containers:
- name: controller
image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:dfbfaf2a7aa5951d1dc4e7f941a8cd05ceb4a860ac1ab23eb8be234861edcb55
imagePullPolicy: IfNotPresent
command: [/venv/bin/gitea-dynamic-runner-opensandbox-identity]
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 100m
memory: 96Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 65532
runAsGroup: 65532
seccompProfile:
type: RuntimeDefault
securityContext:
seccompProfile:
type: RuntimeDefault
@@ -1,22 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- service.yaml
- agent-config.yaml
- rbac.yaml
- identity-controller.yaml
- pools.yaml
patches:
# Pod jobs run directly in the homelab cluster. Keep OpenSandbox VM-only.
- target:
group: sandbox.opensandbox.io
version: v1alpha1
kind: Pool
name: ci-pod
patch: |-
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: ci-pod
namespace: opensandbox
$patch: delete
-293
View File
@@ -1,293 +0,0 @@
---
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: ci-vm
namespace: opensandbox
labels:
app.kubernetes.io/name: gitea-ci-vm
app.kubernetes.io/component: runner-pool
spec:
capacitySpec:
bufferMax: 0
bufferMin: 0
poolMax: 2
poolMin: 0
recycleStrategy:
type: Delete
template:
metadata:
labels:
app.kubernetes.io/name: gitea-ci-vm
ci.ddupan.top/backend: vm
spec:
runtimeClassName: kata-clh-runtime-rs
serviceAccountName: gitea-ci
restartPolicy: Never
terminationGracePeriodSeconds: 30
shareProcessNamespace: true
securityContext:
fsGroup: 2000
fsGroupChangePolicy: OnRootMismatch
initContainers:
- name: task-executor-installer
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/task-executor:v0.1.0
command: [/bin/sh, -c]
args:
- cp /workspace/server /opt/opensandbox/task-executor && chmod 0755 /opt/opensandbox/task-executor
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox
- name: execd-installer
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22
command: [/bin/sh, -c]
args:
- cp ./execd /opt/opensandbox/execd && cp ./bootstrap.sh /opt/opensandbox/bootstrap.sh && chmod 0755 /opt/opensandbox/execd /opt/opensandbox/bootstrap.sh
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox
containers:
- name: sandbox
image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:325d75a208b1a1c6f3b4d4705e47bbc58b34733edcd12f689c60769ab6772a59
imagePullPolicy: IfNotPresent
command: [/opt/opensandbox/task-executor]
args:
- -listen-addr=0.0.0.0:5758
- -log-dir=/tmp
- -data-dir=/tmp/tasks
env:
- name: SANDBOX_MAIN_CONTAINER
value: sandbox
- name: EXECD_ENVS
value: /opt/opensandbox/.env
- name: EXECD
value: /opt/opensandbox/execd
- name: GITEA_INSTANCE_URL
value: https://git.ddupan.top
- name: HOME
value: /data
ports:
- name: task-executor
containerPort: 5758
securityContext:
privileged: true
runAsNonRoot: true
runAsUser: 2000
runAsGroup: 2000
resources:
requests:
cpu: "2"
memory: 3Gi
limits:
cpu: "2"
memory: 3Gi
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox
- name: runner-data
mountPath: /data
- name: runner-workspace
mountPath: /workspace
- name: spire-socket
mountPath: /run/spire/agent-sockets
- name: spire-agent
image: ghcr.io/spiffe/spire-agent:1.15.3@sha256:41b0dcd8b258a69db9e2768292a060766fb76fd866e4bc925849981ea1b825ff
args: [-config, /run/spire/config/agent.conf]
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
volumeMounts:
- name: spire-config
mountPath: /run/spire/config
readOnly: true
- name: spire-bundle
mountPath: /run/spire/bundle
readOnly: true
- name: spire-token
mountPath: /run/spire/tokens
readOnly: true
- name: spire-data
mountPath: /run/spire/data
- name: spire-socket
mountPath: /run/spire/agent-sockets
volumes:
- name: opensandbox-bin
emptyDir: {}
- name: runner-data
emptyDir: {}
- name: runner-workspace
emptyDir: {}
- name: spire-data
emptyDir:
medium: Memory
- name: spire-socket
emptyDir:
medium: Memory
- name: spire-config
configMap:
name: gitea-ci-spire-agent
- name: spire-bundle
configMap:
name: spire-bundle-pem
- name: spire-token
projected:
sources:
- serviceAccountToken:
audience: spire-server
expirationSeconds: 3600
path: token
---
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: ci-pod
namespace: opensandbox
labels:
app.kubernetes.io/name: gitea-ci-pod
app.kubernetes.io/component: runner-pool
spec:
capacitySpec:
bufferMax: 0
bufferMin: 0
poolMax: 4
poolMin: 0
recycleStrategy:
type: Delete
template:
metadata:
labels:
app.kubernetes.io/name: gitea-ci-pod
ci.ddupan.top/backend: pod
spec:
serviceAccountName: gitea-ci
restartPolicy: Never
terminationGracePeriodSeconds: 30
shareProcessNamespace: true
securityContext:
fsGroup: 2000
fsGroupChangePolicy: OnRootMismatch
initContainers:
- name: task-executor-installer
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/task-executor:v0.1.0
command: [/bin/sh, -c]
args:
- cp /workspace/server /opt/opensandbox/task-executor && chmod 0755 /opt/opensandbox/task-executor
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox
- name: execd-installer
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22
command: [/bin/sh, -c]
args:
- cp ./execd /opt/opensandbox/execd && cp ./bootstrap.sh /opt/opensandbox/bootstrap.sh && chmod 0755 /opt/opensandbox/execd /opt/opensandbox/bootstrap.sh
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox
containers:
- name: sandbox
image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:325d75a208b1a1c6f3b4d4705e47bbc58b34733edcd12f689c60769ab6772a59
imagePullPolicy: IfNotPresent
command: [/opt/opensandbox/task-executor]
args:
- -listen-addr=0.0.0.0:5758
- -log-dir=/tmp
- -data-dir=/tmp/tasks
env:
- name: SANDBOX_MAIN_CONTAINER
value: sandbox
- name: EXECD_ENVS
value: /opt/opensandbox/.env
- name: EXECD
value: /opt/opensandbox/execd
- name: GITEA_INSTANCE_URL
value: https://git.ddupan.top
- name: HOME
value: /data
- name: DOCKER_HOST
value: unix:///run/docker/docker.sock
ports:
- name: task-executor
containerPort: 5758
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
runAsNonRoot: true
runAsUser: 2000
runAsGroup: 2000
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox
- name: runner-data
mountPath: /data
- name: runner-workspace
mountPath: /workspace
- name: docker-socket
mountPath: /run/docker
- name: spire-socket
mountPath: /run/spire/agent-sockets
- name: spire-agent
image: ghcr.io/spiffe/spire-agent:1.15.3@sha256:41b0dcd8b258a69db9e2768292a060766fb76fd866e4bc925849981ea1b825ff
args: [-config, /run/spire/config/agent.conf]
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
volumeMounts:
- name: spire-config
mountPath: /run/spire/config
readOnly: true
- name: spire-bundle
mountPath: /run/spire/bundle
readOnly: true
- name: spire-token
mountPath: /run/spire/tokens
readOnly: true
- name: spire-data
mountPath: /run/spire/data
- name: spire-socket
mountPath: /run/spire/agent-sockets
- name: docker
image: docker.io/library/docker:29.1.5-dind
command: [/bin/sh, -c]
args:
- test -e /dev/kmsg || mknod /dev/kmsg c 1 11; exec dockerd --host=unix:///run/docker/docker.sock --group=2000 --storage-driver=overlay2
securityContext:
privileged: true
volumeMounts:
- name: docker-socket
mountPath: /run/docker
- name: docker-data
mountPath: /var/lib/docker
volumes:
- name: opensandbox-bin
emptyDir: {}
- name: runner-data
emptyDir: {}
- name: runner-workspace
emptyDir: {}
- name: docker-data
emptyDir: {}
- name: docker-socket
emptyDir:
medium: Memory
- name: spire-data
emptyDir:
medium: Memory
- name: spire-socket
emptyDir:
medium: Memory
- name: spire-config
configMap:
name: gitea-ci-spire-agent
- name: spire-bundle
configMap:
name: spire-bundle-pem
- name: spire-token
projected:
sources:
- serviceAccountToken:
audience: spire-server
expirationSeconds: 3600
path: token
-87
View File
@@ -1,87 +0,0 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: gitea-ci
namespace: opensandbox
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: opensandbox-identity
namespace: opensandbox
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: opensandbox-identity
namespace: opensandbox
rules:
- apiGroups: [""]
resources: [pods]
verbs: [get, list, watch]
- apiGroups: [sandbox.opensandbox.io]
resources: [batchsandboxes]
verbs: [get, list, watch]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: opensandbox-identity
namespace: opensandbox
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: opensandbox-identity
subjects:
- kind: ServiceAccount
name: opensandbox-identity
namespace: opensandbox
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: opensandbox-identity
rules:
- apiGroups: [spire.spiffe.io]
resources: [clusterstaticentries]
verbs: [create, delete, get, list, watch]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: opensandbox-identity
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: opensandbox-identity
subjects:
- kind: ServiceAccount
name: opensandbox-identity
namespace: opensandbox
---
# The central SPIRE external controller uses this existing sandbox credential
# to publish a PEM bundle for guest-local Agents. It cannot read Secrets here.
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: spire-runner-bundle-publisher
namespace: opensandbox
rules:
- apiGroups: [""]
resources: [configmaps]
verbs: [create, delete, get, list, patch, update, watch]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: spire-runner-bundle-publisher
namespace: opensandbox
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: spire-runner-bundle-publisher
subjects:
- kind: ServiceAccount
name: spire-controller-manager
namespace: spire-system
-15
View File
@@ -1,15 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: opensandbox-server-internal
namespace: opensandbox-system
spec:
type: NodePort
selector:
app.kubernetes.io/instance: opensandbox
app.kubernetes.io/name: opensandbox-server
ports:
- name: http
port: 80
targetPort: http
nodePort: 30080
@@ -1,56 +0,0 @@
# Sandbox External Secrets Operator
本目录在 sandbox 集群部署独立的 External Secrets Operator `2.8.0`,并通过
`ClusterSecretStore/openbao` 读取 OpenBao KV v2 中共享的 OpenSandbox API key。它不复用
homelab 集群的 ESO Pod、ServiceAccount 或 Kubernetes auth backend。
## 当前状态
- OpenBao `auth/kubernetes-sandbox`、backend config、`external-secrets` role 与
`sandbox-external-secrets` policy 已于 2026-09-18 由 Terraform 创建;apply 后 plan
为 zero-diff;
- `kv/k8s/opensandbox-api` 已由本机 `spiffe://ddupan.top/dev/panxiao81` 身份生成并写入,
值未输出或落盘;
- sandbox ESO operator、`ClusterSecretStore/openbao` 与 OpenSandbox `ExternalSecret`
由 Flux 管理;
- 线上 `ClusterSecretStore/openbao` 为 `Valid/Ready`,`ExternalSecret/opensandbox-api-key`
为 `SecretSynced/Ready`;
- OpenSandbox 已切换到 API key:无 key 请求返回 `401`,正确 key 请求返回 `200`;
- homelab runner 对同一 key 的投影不在本目录,留给 runner 项目管理。
OpenBao 的 `auth/kubernetes-sandbox`、对应 role、policy、sandbox API 地址与公开 CA
完全由 `infrastructure/openbao/terraform/` 管理。CA 文件提交到 Git 是刻意设计:CA
是公开信任材料,版本化后集群重建造成的 trust root 变化会产生可审计的 Terraform diff。
ESO 使用 TokenRequest 生成短期 ServiceAccount JWT。OpenBao 未配置长期
`token_reviewer_jwt`,而是使用登录 JWT 调用 sandbox TokenReview;因此
`external-secrets` ServiceAccount 仅额外绑定内建 `system:auth-delegator`。
## 重建顺序
1. 在 OpenBao 写入 OpenSandbox API key:
```bash
openssl rand -hex 32 | bao kv put kv/k8s/opensandbox-api api_key=-
```
2. 在 `infrastructure/openbao/terraform` 执行 `terraform plan` 和 `terraform apply`,
创建 `kubernetes-sandbox` auth mount、backend config、role 与只允许读取
`kv/k8s/opensandbox-api` 的最小权限 policy。
3. 合并 Flux 变更。依次等待 `flux-system/external-secrets-operator`、
`flux-system/external-secrets` Ready,再等待 `flux-system/opensandbox` 滚动完成。
operator 与配置拆成两个 Flux Kustomization,确保全新集群先安装 CRD,再声明
`ClusterSecretStore`;不要为了减少目录而把两层重新合并。
## 验收
```bash
kubectl get clustersecretstore openbao
kubectl -n opensandbox-system get externalsecret opensandbox-api-key
kubectl -n opensandbox-system get secret opensandbox-api-key
```
只检查 Secret 是否存在及 key 名,不输出 `data`。`ClusterSecretStore` 或
`ExternalSecret` 不 Ready 时,先检查 `auth/kubernetes-sandbox`,不要临时创建静态
Bao token Secret。
@@ -1,13 +0,0 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: sandbox-external-secrets-token-review
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: external-secrets
namespace: external-secrets
@@ -1,18 +0,0 @@
---
apiVersion: external-secrets.io/v1
kind: ClusterSecretStore
metadata:
name: openbao
spec:
provider:
vault:
server: https://bao.ad.ddupan.top:8200
path: kv
version: v2
auth:
kubernetes:
mountPath: kubernetes-sandbox
role: external-secrets
serviceAccountRef:
name: external-secrets
namespace: external-secrets
@@ -1,8 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- repository.yaml
- values.yaml
- helmrelease.yaml
@@ -1,9 +0,0 @@
---
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: external-secrets
namespace: external-secrets
spec:
interval: 1h
url: https://charts.external-secrets.io

Some files were not shown because too many files have changed in this diff Show More