Merge pull request:部署 NATS JetStream 消息基础设施
首期使用静态 Account 凭据;SPIRE Auth Callout 后续见 #56。YAML 与 collection tests 已通过,Ansible lint 卡在无关的 Galaxy 依赖下载。
This commit was merged in pull request #57.
This commit is contained in:
@@ -0,0 +1,18 @@
|
|||||||
|
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||||
|
kind: Kustomization
|
||||||
|
metadata:
|
||||||
|
name: nats
|
||||||
|
namespace: flux-system
|
||||||
|
spec:
|
||||||
|
dependsOn:
|
||||||
|
- name: cert-manager
|
||||||
|
- name: external-secrets
|
||||||
|
- name: openebs
|
||||||
|
interval: 10m
|
||||||
|
path: ./platform/nats
|
||||||
|
prune: false
|
||||||
|
sourceRef:
|
||||||
|
kind: GitRepository
|
||||||
|
name: flux-system
|
||||||
|
timeout: 10m
|
||||||
|
wait: true
|
||||||
@@ -10,6 +10,7 @@ resources:
|
|||||||
- apps/gitea-actions.yaml
|
- apps/gitea-actions.yaml
|
||||||
- apps/http-echo.yaml
|
- apps/http-echo.yaml
|
||||||
- apps/openebs.yaml
|
- apps/openebs.yaml
|
||||||
|
- apps/nats.yaml
|
||||||
- apps/spire.yaml
|
- apps/spire.yaml
|
||||||
- apps/observability.yaml
|
- apps/observability.yaml
|
||||||
- apps/zot.yaml
|
- apps/zot.yaml
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ homelab_dns:
|
|||||||
- { zone: ad.ddupan.top, name: pve3, type: A, values: [192.168.10.9] }
|
- { zone: ad.ddupan.top, name: pve3, type: A, values: [192.168.10.9] }
|
||||||
- { zone: ad.ddupan.top, name: retrolab, type: A, values: [10.60.0.10] }
|
- { zone: ad.ddupan.top, name: retrolab, type: A, values: [10.60.0.10] }
|
||||||
- { zone: ad.ddupan.top, name: netbox, type: A, values: [192.168.10.127] }
|
- { zone: ad.ddupan.top, name: netbox, type: A, values: [192.168.10.127] }
|
||||||
|
- { zone: ad.ddupan.top, name: nats, type: A, values: [192.168.10.127] }
|
||||||
- { zone: ad.ddupan.top, name: s3, type: A, values: [192.168.10.127] }
|
- { zone: ad.ddupan.top, name: s3, type: A, values: [192.168.10.127] }
|
||||||
- { zone: ad.ddupan.top, name: spire-oidc, type: A, values: [192.168.10.127] }
|
- { zone: ad.ddupan.top, name: spire-oidc, type: A, values: [192.168.10.127] }
|
||||||
- { zone: ad.ddupan.top, name: zot, type: A, values: [192.168.10.127] }
|
- { zone: ad.ddupan.top, name: zot, type: A, values: [192.168.10.127] }
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
# NATS
|
||||||
|
|
||||||
|
共享的轻量消息基础设施。首期为 Gitea microVM runner 提供 JetStream work queue,
|
||||||
|
但 Account、subject 与部署位置均不与 CI controller 绑定,其他服务可按独立 Account
|
||||||
|
复用。
|
||||||
|
|
||||||
|
## 当前拓扑
|
||||||
|
|
||||||
|
- 单节点 NATS;当前 homelab 没有资源运行有意义的三副本 JetStream quorum。
|
||||||
|
- JetStream file store 使用 `localpv-zfs-ceph`,PVC 2 GiB。
|
||||||
|
- 服务通过 k3s ServiceLB 在 `nats.ad.ddupan.top:4222` 暴露给内网;集群内客户端
|
||||||
|
使用 `nats.nats.svc.cluster.local:4222`。访问控制由 TLS、Account 与用户权限负责,
|
||||||
|
不额外维护易漂移的源 IP 白名单。
|
||||||
|
- TLS 证书由 `bao-acme` 签发。PVE 节点已信任内部 CA。
|
||||||
|
- `SYS` Account 用于管理;`CI` Account 启用 JetStream,存储上限 1 GiB。
|
||||||
|
|
||||||
|
首期使用静态用户,密码只存在 OpenBao `kv/k8s/nats`:
|
||||||
|
|
||||||
|
```text
|
||||||
|
sys_password
|
||||||
|
ci_producer_password
|
||||||
|
ci_worker_password
|
||||||
|
```
|
||||||
|
|
||||||
|
`ci-producer` 只能发布 `ci.runner.>` 并调用必要的 JetStream API;`ci-worker`
|
||||||
|
只能调用 JetStream pull/ACK API。二者都不能读取另一个 Account 的 subject。
|
||||||
|
|
||||||
|
后续 SPIRE/Auth Callout 动态认证见 homelab-infra issue #56。该迁移只替换连接
|
||||||
|
凭据,不改变 Account、stream、subject 或 consumer。
|
||||||
|
|
||||||
|
## CI stream 约定
|
||||||
|
|
||||||
|
controller 首次启动时幂等创建 `CI_RUNNER` stream:`ci.runner.*`、
|
||||||
|
`WorkQueuePolicy`、file storage、24h/10000 条/256 MiB 上限。每类 runner 使用独立
|
||||||
|
subject 和 durable pull consumer;同类型的多个 worker 共享 durable consumer。
|
||||||
|
ACK 后消息立即删除,不保存 CI 历史。
|
||||||
|
|
||||||
|
## 验证
|
||||||
|
|
||||||
|
```bash
|
||||||
|
kubectl -n nats get helmrelease,pod,pvc,certificate,externalsecret
|
||||||
|
kubectl -n nats logs statefulset/nats -c nats
|
||||||
|
```
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
apiVersion: cert-manager.io/v1
|
||||||
|
kind: Certificate
|
||||||
|
metadata:
|
||||||
|
name: nats-ad-ddupan-top
|
||||||
|
namespace: nats
|
||||||
|
spec:
|
||||||
|
secretName: nats-server-tls
|
||||||
|
issuerRef:
|
||||||
|
name: bao-acme
|
||||||
|
kind: ClusterIssuer
|
||||||
|
group: cert-manager.io
|
||||||
|
commonName: nats.ad.ddupan.top
|
||||||
|
dnsNames:
|
||||||
|
- nats.ad.ddupan.top
|
||||||
|
duration: 720h
|
||||||
|
renewBefore: 168h
|
||||||
|
privateKey:
|
||||||
|
algorithm: ECDSA
|
||||||
|
size: 256
|
||||||
|
rotationPolicy: Always
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
apiVersion: external-secrets.io/v1
|
||||||
|
kind: ExternalSecret
|
||||||
|
metadata:
|
||||||
|
name: nats-auth
|
||||||
|
namespace: nats
|
||||||
|
spec:
|
||||||
|
refreshInterval: 1h
|
||||||
|
secretStoreRef:
|
||||||
|
kind: ClusterSecretStore
|
||||||
|
name: openbao
|
||||||
|
target:
|
||||||
|
creationPolicy: Owner
|
||||||
|
name: nats-auth
|
||||||
|
dataFrom:
|
||||||
|
- extract:
|
||||||
|
key: k8s/nats
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||||
|
kind: HelmRelease
|
||||||
|
metadata:
|
||||||
|
name: nats
|
||||||
|
namespace: nats
|
||||||
|
spec:
|
||||||
|
chart:
|
||||||
|
spec:
|
||||||
|
chart: nats
|
||||||
|
interval: 1h
|
||||||
|
sourceRef:
|
||||||
|
kind: HelmRepository
|
||||||
|
name: nats
|
||||||
|
version: 2.14.2
|
||||||
|
driftDetection:
|
||||||
|
mode: enabled
|
||||||
|
install:
|
||||||
|
strategy:
|
||||||
|
name: RetryOnFailure
|
||||||
|
retryInterval: 5m
|
||||||
|
interval: 30m
|
||||||
|
releaseName: nats
|
||||||
|
targetNamespace: nats
|
||||||
|
timeout: 10m
|
||||||
|
upgrade:
|
||||||
|
strategy:
|
||||||
|
name: RetryOnFailure
|
||||||
|
retryInterval: 5m
|
||||||
|
valuesFrom:
|
||||||
|
- kind: ConfigMap
|
||||||
|
name: nats-values
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
apiVersion: source.toolkit.fluxcd.io/v1
|
||||||
|
kind: HelmRepository
|
||||||
|
metadata:
|
||||||
|
name: nats
|
||||||
|
namespace: nats
|
||||||
|
spec:
|
||||||
|
interval: 1h
|
||||||
|
url: https://nats-io.github.io/k8s/helm/charts/
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
generatorOptions:
|
||||||
|
disableNameSuffixHash: true
|
||||||
|
labels:
|
||||||
|
reconcile.fluxcd.io/watch: Enabled
|
||||||
|
configMapGenerator:
|
||||||
|
- name: nats-values
|
||||||
|
namespace: nats
|
||||||
|
files:
|
||||||
|
- values.yaml=values.yaml
|
||||||
|
resources:
|
||||||
|
- namespace.yaml
|
||||||
|
- helmrepository.yaml
|
||||||
|
- external-secret.yaml
|
||||||
|
- certificate.yaml
|
||||||
|
- helmrelease.yaml
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: nats
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
config:
|
||||||
|
jetstream:
|
||||||
|
enabled: true
|
||||||
|
fileStore:
|
||||||
|
enabled: true
|
||||||
|
maxSize: 2Gi
|
||||||
|
pvc:
|
||||||
|
enabled: true
|
||||||
|
size: 2Gi
|
||||||
|
storageClassName: localpv-zfs-ceph
|
||||||
|
memoryStore:
|
||||||
|
enabled: true
|
||||||
|
maxSize: 64Mi
|
||||||
|
nats:
|
||||||
|
tls:
|
||||||
|
enabled: true
|
||||||
|
secretName: nats-server-tls
|
||||||
|
merge:
|
||||||
|
system_account: SYS
|
||||||
|
accounts:
|
||||||
|
SYS:
|
||||||
|
users:
|
||||||
|
- user: sys
|
||||||
|
password: "<< $NATS_SYS_PASSWORD >>"
|
||||||
|
CI:
|
||||||
|
jetstream:
|
||||||
|
max_memory: 32Mi
|
||||||
|
max_file: 1Gi
|
||||||
|
max_streams: 16
|
||||||
|
max_consumers: 64
|
||||||
|
max_bytes_required: true
|
||||||
|
users:
|
||||||
|
- user: ci-producer
|
||||||
|
password: "<< $NATS_CI_PRODUCER_PASSWORD >>"
|
||||||
|
permissions:
|
||||||
|
publish:
|
||||||
|
allow: [ci.runner.>, $JS.API.>]
|
||||||
|
subscribe:
|
||||||
|
allow: [_INBOX.>]
|
||||||
|
- user: ci-worker
|
||||||
|
password: "<< $NATS_CI_WORKER_PASSWORD >>"
|
||||||
|
permissions:
|
||||||
|
publish:
|
||||||
|
allow: [$JS.API.>, $JS.ACK.>]
|
||||||
|
subscribe:
|
||||||
|
allow: [_INBOX.>]
|
||||||
|
|
||||||
|
container:
|
||||||
|
env:
|
||||||
|
NATS_SYS_PASSWORD:
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef: {name: nats-auth, key: sys_password}
|
||||||
|
NATS_CI_PRODUCER_PASSWORD:
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef: {name: nats-auth, key: ci_producer_password}
|
||||||
|
NATS_CI_WORKER_PASSWORD:
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef: {name: nats-auth, key: ci_worker_password}
|
||||||
|
resources:
|
||||||
|
requests: {cpu: 25m, memory: 64Mi}
|
||||||
|
limits: {memory: 192Mi}
|
||||||
|
|
||||||
|
natsBox:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
promExporter:
|
||||||
|
enabled: true
|
||||||
|
podMonitor:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
service:
|
||||||
|
merge:
|
||||||
|
spec:
|
||||||
|
type: LoadBalancer
|
||||||
Reference in New Issue
Block a user