From c11e1d5e6ff8a39d7f4a79cb9eea3d37ab426948 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Wed, 16 Sep 2026 12:00:24 +0000 Subject: [PATCH] =?UTF-8?q?=E9=83=A8=E7=BD=B2=20NATS=20JetStream=20?= =?UTF-8?q?=E6=B6=88=E6=81=AF=E5=9F=BA=E7=A1=80=E8=AE=BE=E6=96=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- clusters/homelab/apps/nats.yaml | 18 +++++++ clusters/homelab/kustomization.yaml | 1 + infrastructure/dns/records.yml | 1 + platform/nats/README.md | 43 +++++++++++++++++ platform/nats/certificate.yaml | 20 ++++++++ platform/nats/external-secret.yaml | 16 +++++++ platform/nats/helmrelease.yaml | 31 ++++++++++++ platform/nats/helmrepository.yaml | 8 ++++ platform/nats/kustomization.yaml | 17 +++++++ platform/nats/namespace.yaml | 4 ++ platform/nats/values.yaml | 74 +++++++++++++++++++++++++++++ 11 files changed, 233 insertions(+) create mode 100644 clusters/homelab/apps/nats.yaml create mode 100644 platform/nats/README.md create mode 100644 platform/nats/certificate.yaml create mode 100644 platform/nats/external-secret.yaml create mode 100644 platform/nats/helmrelease.yaml create mode 100644 platform/nats/helmrepository.yaml create mode 100644 platform/nats/kustomization.yaml create mode 100644 platform/nats/namespace.yaml create mode 100644 platform/nats/values.yaml diff --git a/clusters/homelab/apps/nats.yaml b/clusters/homelab/apps/nats.yaml new file mode 100644 index 0000000..53483f8 --- /dev/null +++ b/clusters/homelab/apps/nats.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: nats + namespace: flux-system +spec: + dependsOn: + - name: cert-manager + - name: external-secrets + - name: openebs + interval: 10m + path: ./platform/nats + prune: false + sourceRef: + kind: GitRepository + name: flux-system + timeout: 10m + wait: true diff --git a/clusters/homelab/kustomization.yaml b/clusters/homelab/kustomization.yaml index 5da14e8..4bdcde0 100644 --- a/clusters/homelab/kustomization.yaml +++ b/clusters/homelab/kustomization.yaml @@ -10,6 +10,7 @@ resources: - apps/gitea-actions.yaml - apps/http-echo.yaml - apps/openebs.yaml + - apps/nats.yaml - apps/spire.yaml - apps/observability.yaml - apps/zot.yaml diff --git a/infrastructure/dns/records.yml b/infrastructure/dns/records.yml index 71303d1..c88843d 100644 --- a/infrastructure/dns/records.yml +++ b/infrastructure/dns/records.yml @@ -12,6 +12,7 @@ homelab_dns: - { zone: ad.ddupan.top, name: pve3, type: A, values: [192.168.10.9] } - { zone: ad.ddupan.top, name: retrolab, type: A, values: [10.60.0.10] } - { zone: ad.ddupan.top, name: netbox, type: A, values: [192.168.10.127] } + - { zone: ad.ddupan.top, name: nats, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: s3, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: spire-oidc, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: zot, type: A, values: [192.168.10.127] } diff --git a/platform/nats/README.md b/platform/nats/README.md new file mode 100644 index 0000000..645922a --- /dev/null +++ b/platform/nats/README.md @@ -0,0 +1,43 @@ +# NATS + +共享的轻量消息基础设施。首期为 Gitea microVM runner 提供 JetStream work queue， +但 Account、subject 与部署位置均不与 CI controller 绑定，其他服务可按独立 Account +复用。 + +## 当前拓扑 + +- 单节点 NATS；当前 homelab 没有资源运行有意义的三副本 JetStream quorum。 +- JetStream file store 使用 `localpv-zfs-ceph`，PVC 2 GiB。 +- 服务通过 k3s ServiceLB 在 `nats.ad.ddupan.top:4222` 暴露给内网；集群内客户端 + 使用 `nats.nats.svc.cluster.local:4222`。访问控制由 TLS、Account 与用户权限负责， + 不额外维护易漂移的源 IP 白名单。 +- TLS 证书由 `bao-acme` 签发。PVE 节点已信任内部 CA。 +- `SYS` Account 用于管理；`CI` Account 启用 JetStream，存储上限 1 GiB。 + +首期使用静态用户，密码只存在 OpenBao `kv/k8s/nats`： + +```text +sys_password +ci_producer_password +ci_worker_password +``` + +`ci-producer` 只能发布 `ci.runner.>` 并调用必要的 JetStream API；`ci-worker` +只能调用 JetStream pull/ACK API。二者都不能读取另一个 Account 的 subject。 + +后续 SPIRE/Auth Callout 动态认证见 homelab-infra issue #56。该迁移只替换连接 +凭据，不改变 Account、stream、subject 或 consumer。 + +## CI stream 约定 + +controller 首次启动时幂等创建 `CI_RUNNER` stream：`ci.runner.*`、 +`WorkQueuePolicy`、file storage、24h/10000 条/256 MiB 上限。每类 runner 使用独立 +subject 和 durable pull consumer；同类型的多个 worker 共享 durable consumer。 +ACK 后消息立即删除，不保存 CI 历史。 + +## 验证 + +```bash +kubectl -n nats get helmrelease,pod,pvc,certificate,externalsecret +kubectl -n nats logs statefulset/nats -c nats +``` diff --git a/platform/nats/certificate.yaml b/platform/nats/certificate.yaml new file mode 100644 index 0000000..441b6ff --- /dev/null +++ b/platform/nats/certificate.yaml @@ -0,0 +1,20 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: nats-ad-ddupan-top + namespace: nats +spec: + secretName: nats-server-tls + issuerRef: + name: bao-acme + kind: ClusterIssuer + group: cert-manager.io + commonName: nats.ad.ddupan.top + dnsNames: + - nats.ad.ddupan.top + duration: 720h + renewBefore: 168h + privateKey: + algorithm: ECDSA + size: 256 + rotationPolicy: Always diff --git a/platform/nats/external-secret.yaml b/platform/nats/external-secret.yaml new file mode 100644 index 0000000..8aa0b08 --- /dev/null +++ b/platform/nats/external-secret.yaml @@ -0,0 +1,16 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: nats-auth + namespace: nats +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: openbao + target: + creationPolicy: Owner + name: nats-auth + dataFrom: + - extract: + key: k8s/nats diff --git a/platform/nats/helmrelease.yaml b/platform/nats/helmrelease.yaml new file mode 100644 index 0000000..02ec615 --- /dev/null +++ b/platform/nats/helmrelease.yaml @@ -0,0 +1,31 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: nats + namespace: nats +spec: + chart: + spec: + chart: nats + interval: 1h + sourceRef: + kind: HelmRepository + name: nats + version: 2.14.2 + driftDetection: + mode: enabled + install: + strategy: + name: RetryOnFailure + retryInterval: 5m + interval: 30m + releaseName: nats + targetNamespace: nats + timeout: 10m + upgrade: + strategy: + name: RetryOnFailure + retryInterval: 5m + valuesFrom: + - kind: ConfigMap + name: nats-values diff --git a/platform/nats/helmrepository.yaml b/platform/nats/helmrepository.yaml new file mode 100644 index 0000000..3368f94 --- /dev/null +++ b/platform/nats/helmrepository.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: nats + namespace: nats +spec: + interval: 1h + url: https://nats-io.github.io/k8s/helm/charts/ diff --git a/platform/nats/kustomization.yaml b/platform/nats/kustomization.yaml new file mode 100644 index 0000000..e27c99a --- /dev/null +++ b/platform/nats/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +generatorOptions: + disableNameSuffixHash: true + labels: + reconcile.fluxcd.io/watch: Enabled +configMapGenerator: + - name: nats-values + namespace: nats + files: + - values.yaml=values.yaml +resources: + - namespace.yaml + - helmrepository.yaml + - external-secret.yaml + - certificate.yaml + - helmrelease.yaml diff --git a/platform/nats/namespace.yaml b/platform/nats/namespace.yaml new file mode 100644 index 0000000..bcd77fe --- /dev/null +++ b/platform/nats/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: nats diff --git a/platform/nats/values.yaml b/platform/nats/values.yaml new file mode 100644 index 0000000..19ea515 --- /dev/null +++ b/platform/nats/values.yaml @@ -0,0 +1,74 @@ +config: + jetstream: + enabled: true + fileStore: + enabled: true + maxSize: 2Gi + pvc: + enabled: true + size: 2Gi + storageClassName: localpv-zfs-ceph + memoryStore: + enabled: true + maxSize: 64Mi + nats: + tls: + enabled: true + secretName: nats-server-tls + merge: + system_account: SYS + accounts: + SYS: + users: + - user: sys + password: "<< $NATS_SYS_PASSWORD >>" + CI: + jetstream: + max_memory: 32Mi + max_file: 1Gi + max_streams: 16 + max_consumers: 64 + max_bytes_required: true + users: + - user: ci-producer + password: "<< $NATS_CI_PRODUCER_PASSWORD >>" + permissions: + publish: + allow: [ci.runner.>, $JS.API.>] + subscribe: + allow: [_INBOX.>] + - user: ci-worker + password: "<< $NATS_CI_WORKER_PASSWORD >>" + permissions: + publish: + allow: [$JS.API.>, $JS.ACK.>] + subscribe: + allow: [_INBOX.>] + +container: + env: + NATS_SYS_PASSWORD: + valueFrom: + secretKeyRef: {name: nats-auth, key: sys_password} + NATS_CI_PRODUCER_PASSWORD: + valueFrom: + secretKeyRef: {name: nats-auth, key: ci_producer_password} + NATS_CI_WORKER_PASSWORD: + valueFrom: + secretKeyRef: {name: nats-auth, key: ci_worker_password} + resources: + requests: {cpu: 25m, memory: 64Mi} + limits: {memory: 192Mi} + +natsBox: + enabled: false + +promExporter: + enabled: true + podMonitor: + enabled: true + +service: + merge: + spec: + type: LoadBalancer