Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
57b13387cb
|
||
|
|
1deb024621
|
||
|
|
311a506982
|
||
|
|
a0d38fc9ab
|
||
|
|
fa293ffc80
|
||
|
|
77df686981
|
||
|
|
9e6515406f
|
||
|
|
f6c216ce93
|
@@ -0,0 +1,63 @@
|
||||
---
|
||||
name: kind-on-kata-smoke
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- poc/kind-on-kata
|
||||
paths:
|
||||
- .gitea/workflows/kind-on-kata-smoke.yml
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
smoke:
|
||||
runs-on: kata-poc
|
||||
steps:
|
||||
- name: Create nested kind cluster
|
||||
shell: sh
|
||||
env:
|
||||
KIND_VERSION: v0.27.0
|
||||
KIND_NODE_IMAGE: kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f
|
||||
run: |
|
||||
set -eu
|
||||
apk add --no-cache ca-certificates curl docker-cli
|
||||
curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind \
|
||||
"https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64"
|
||||
curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind.sha256sum \
|
||||
"https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64.sha256sum"
|
||||
expected="$(awk '{print $1}' /tmp/kind.sha256sum)"
|
||||
printf '%s %s\n' "$expected" /tmp/kind | sha256sum -c -
|
||||
install -m 0755 /tmp/kind /usr/local/bin/kind
|
||||
docker info --format 'kernel={{.KernelVersion}} driver={{.Driver}}'
|
||||
test "$(docker info --format '{{.Driver}}')" = overlay2
|
||||
|
||||
cleanup() {
|
||||
kind delete cluster --name nested >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
cat >/tmp/kind-config.yaml <<'EOF'
|
||||
kind: Cluster
|
||||
apiVersion: kind.x-k8s.io/v1alpha4
|
||||
name: nested
|
||||
nodes:
|
||||
- role: control-plane
|
||||
extraMounts:
|
||||
- hostPath: /dev/kmsg
|
||||
containerPath: /dev/kmsg
|
||||
EOF
|
||||
kind create cluster -v 9 --retain --config /tmp/kind-config.yaml --image "$KIND_NODE_IMAGE" --wait 5m
|
||||
docker exec nested-control-plane kubectl \
|
||||
--kubeconfig=/etc/kubernetes/admin.conf wait \
|
||||
--for=condition=Ready node/nested-control-plane --timeout=2m
|
||||
docker exec nested-control-plane kubectl \
|
||||
--kubeconfig=/etc/kubernetes/admin.conf run smoke \
|
||||
--image=docker.io/library/busybox:1.37 --restart=Never \
|
||||
--command -- sh -c 'echo kind-on-kata-ok'
|
||||
docker exec nested-control-plane kubectl \
|
||||
--kubeconfig=/etc/kubernetes/admin.conf wait \
|
||||
--for=jsonpath='{.status.phase}'=Succeeded pod/smoke --timeout=2m
|
||||
test "$(docker exec nested-control-plane kubectl \
|
||||
--kubeconfig=/etc/kubernetes/admin.conf logs smoke)" = kind-on-kata-ok
|
||||
kind delete cluster --name nested
|
||||
trap - EXIT
|
||||
@@ -1,18 +0,0 @@
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: nats
|
||||
namespace: flux-system
|
||||
spec:
|
||||
dependsOn:
|
||||
- name: cert-manager
|
||||
- name: external-secrets
|
||||
- name: openebs
|
||||
interval: 10m
|
||||
path: ./platform/nats
|
||||
prune: false
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
timeout: 10m
|
||||
wait: true
|
||||
@@ -10,7 +10,6 @@ resources:
|
||||
- apps/gitea-actions.yaml
|
||||
- apps/http-echo.yaml
|
||||
- apps/openebs.yaml
|
||||
- apps/nats.yaml
|
||||
- apps/spire.yaml
|
||||
- apps/observability.yaml
|
||||
- apps/zot.yaml
|
||||
|
||||
@@ -12,7 +12,6 @@ homelab_dns:
|
||||
- { zone: ad.ddupan.top, name: pve3, type: A, values: [192.168.10.9] }
|
||||
- { zone: ad.ddupan.top, name: retrolab, type: A, values: [10.60.0.10] }
|
||||
- { zone: ad.ddupan.top, name: netbox, type: A, values: [192.168.10.127] }
|
||||
- { zone: ad.ddupan.top, name: nats, type: A, values: [192.168.10.127] }
|
||||
- { zone: ad.ddupan.top, name: s3, type: A, values: [192.168.10.127] }
|
||||
- { zone: ad.ddupan.top, name: spire-oidc, type: A, values: [192.168.10.127] }
|
||||
- { zone: ad.ddupan.top, name: zot, type: A, values: [192.168.10.127] }
|
||||
|
||||
@@ -119,10 +119,6 @@ issuerRef:
|
||||
kind: ClusterIssuer
|
||||
```
|
||||
|
||||
`values.yaml` 必须保持 `config.gatewayAPI.enabled: true`。`bao-acme` 的 HTTP-01
|
||||
solver 通过共享 Gateway 创建临时 HTTPRoute;关闭该项不会让 ClusterIssuer 变为
|
||||
NotReady,而是会让每个 Challenge 卡在 `gateway api is not enabled`。
|
||||
|
||||
Issuance is capped by `default_directory_policy = role:bao-server`
|
||||
(`../../infrastructure/openbao/terraform/pki.tf`), which permits `ad.ddupan.top` subdomains only. Clients
|
||||
need the internal CA in their trust store — already true for the PVE nodes, the DC and
|
||||
|
||||
@@ -44,13 +44,6 @@ cainjector:
|
||||
limits:
|
||||
memory: 256Mi
|
||||
|
||||
# bao-acme solves HTTP-01 through the shared Gateway. The ClusterIssuer can be
|
||||
# accepted while this is disabled, but every Challenge then stays pending with
|
||||
# "gateway api is not enabled". Gateway API CRDs are installed by Envoy Gateway.
|
||||
config:
|
||||
gatewayAPI:
|
||||
enabled: true
|
||||
|
||||
# ⚠ DNS-01 self-check: cert-manager polls authoritative NS for the _acme-challenge
|
||||
# TXT record before telling the CA to validate. By default it asks the cluster's
|
||||
# resolver, which for ad.ddupan.top is CoreDNS -> the Samba AD DC (k3s/coredns-custom.yaml).
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
# NATS
|
||||
|
||||
共享的轻量消息基础设施。首期为 Gitea microVM runner 提供 JetStream work queue,
|
||||
但 Account、subject 与部署位置均不与 CI controller 绑定,其他服务可按独立 Account
|
||||
复用。
|
||||
|
||||
## 当前拓扑
|
||||
|
||||
- 单节点 NATS;当前 homelab 没有资源运行有意义的三副本 JetStream quorum。
|
||||
- JetStream file store 使用 `localpv-zfs-ceph`,PVC 2 GiB。
|
||||
- 服务通过 k3s ServiceLB 在 `nats.ad.ddupan.top:4222` 暴露给内网;集群内客户端
|
||||
使用 `nats.nats.svc.cluster.local:4222`。访问控制由 TLS、Account 与用户权限负责,
|
||||
不额外维护易漂移的源 IP 白名单。
|
||||
- TLS 证书由 `bao-acme` 签发。PVE 节点已信任内部 CA。
|
||||
- `SYS` Account 用于管理;`CI` Account 启用 JetStream,存储上限 1 GiB。
|
||||
|
||||
首期使用静态用户,密码只存在 OpenBao `kv/k8s/nats`:
|
||||
|
||||
```text
|
||||
sys_password
|
||||
ci_producer_password
|
||||
ci_worker_password
|
||||
```
|
||||
|
||||
`ci-producer` 只能发布 `ci.runner.>` 并调用必要的 JetStream API;`ci-worker`
|
||||
只能调用 JetStream pull/ACK API。二者都不能读取另一个 Account 的 subject。
|
||||
|
||||
后续 SPIRE/Auth Callout 动态认证见 homelab-infra issue #56。该迁移只替换连接
|
||||
凭据,不改变 Account、stream、subject 或 consumer。
|
||||
|
||||
## CI stream 约定
|
||||
|
||||
controller 首次启动时幂等创建 `CI_RUNNER` stream:`ci.runner.*`、
|
||||
`WorkQueuePolicy`、file storage、24h/10000 条/256 MiB 上限。每类 runner 使用独立
|
||||
subject 和 durable pull consumer;同类型的多个 worker 共享 durable consumer。
|
||||
ACK 后消息立即删除,不保存 CI 历史。
|
||||
|
||||
## 验证
|
||||
|
||||
```bash
|
||||
kubectl -n nats get helmrelease,pod,pvc,certificate,externalsecret
|
||||
kubectl -n nats logs statefulset/nats -c nats
|
||||
```
|
||||
@@ -1,20 +0,0 @@
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: nats-ad-ddupan-top
|
||||
namespace: nats
|
||||
spec:
|
||||
secretName: nats-server-tls
|
||||
issuerRef:
|
||||
name: bao-acme
|
||||
kind: ClusterIssuer
|
||||
group: cert-manager.io
|
||||
commonName: nats.ad.ddupan.top
|
||||
dnsNames:
|
||||
- nats.ad.ddupan.top
|
||||
duration: 720h
|
||||
renewBefore: 168h
|
||||
privateKey:
|
||||
algorithm: ECDSA
|
||||
size: 256
|
||||
rotationPolicy: Always
|
||||
@@ -1,16 +0,0 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: nats-auth
|
||||
namespace: nats
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
creationPolicy: Owner
|
||||
name: nats-auth
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: k8s/nats
|
||||
@@ -1,31 +0,0 @@
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: nats
|
||||
namespace: nats
|
||||
spec:
|
||||
chart:
|
||||
spec:
|
||||
chart: nats
|
||||
interval: 1h
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: nats
|
||||
version: 2.14.2
|
||||
driftDetection:
|
||||
mode: enabled
|
||||
install:
|
||||
strategy:
|
||||
name: RetryOnFailure
|
||||
retryInterval: 5m
|
||||
interval: 30m
|
||||
releaseName: nats
|
||||
targetNamespace: nats
|
||||
timeout: 10m
|
||||
upgrade:
|
||||
strategy:
|
||||
name: RetryOnFailure
|
||||
retryInterval: 5m
|
||||
valuesFrom:
|
||||
- kind: ConfigMap
|
||||
name: nats-values
|
||||
@@ -1,8 +0,0 @@
|
||||
apiVersion: source.toolkit.fluxcd.io/v1
|
||||
kind: HelmRepository
|
||||
metadata:
|
||||
name: nats
|
||||
namespace: nats
|
||||
spec:
|
||||
interval: 1h
|
||||
url: https://nats-io.github.io/k8s/helm/charts/
|
||||
@@ -1,17 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
labels:
|
||||
reconcile.fluxcd.io/watch: Enabled
|
||||
configMapGenerator:
|
||||
- name: nats-values
|
||||
namespace: nats
|
||||
files:
|
||||
- values.yaml=values.yaml
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- helmrepository.yaml
|
||||
- external-secret.yaml
|
||||
- certificate.yaml
|
||||
- helmrelease.yaml
|
||||
@@ -1,4 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: nats
|
||||
@@ -1,74 +0,0 @@
|
||||
config:
|
||||
jetstream:
|
||||
enabled: true
|
||||
fileStore:
|
||||
enabled: true
|
||||
maxSize: 2Gi
|
||||
pvc:
|
||||
enabled: true
|
||||
size: 2Gi
|
||||
storageClassName: localpv-zfs-ceph
|
||||
memoryStore:
|
||||
enabled: true
|
||||
maxSize: 64Mi
|
||||
nats:
|
||||
tls:
|
||||
enabled: true
|
||||
secretName: nats-server-tls
|
||||
merge:
|
||||
system_account: SYS
|
||||
accounts:
|
||||
SYS:
|
||||
users:
|
||||
- user: sys
|
||||
password: "<< $NATS_SYS_PASSWORD >>"
|
||||
CI:
|
||||
jetstream:
|
||||
max_memory: 32Mi
|
||||
max_file: 1Gi
|
||||
max_streams: 16
|
||||
max_consumers: 64
|
||||
max_bytes_required: true
|
||||
users:
|
||||
- user: ci-producer
|
||||
password: "<< $NATS_CI_PRODUCER_PASSWORD >>"
|
||||
permissions:
|
||||
publish:
|
||||
allow: [ci.runner.>, $JS.API.>]
|
||||
subscribe:
|
||||
allow: [_INBOX.>]
|
||||
- user: ci-worker
|
||||
password: "<< $NATS_CI_WORKER_PASSWORD >>"
|
||||
permissions:
|
||||
publish:
|
||||
allow: [$JS.API.>, $JS.ACK.>]
|
||||
subscribe:
|
||||
allow: [_INBOX.>]
|
||||
|
||||
container:
|
||||
env:
|
||||
NATS_SYS_PASSWORD:
|
||||
valueFrom:
|
||||
secretKeyRef: {name: nats-auth, key: sys_password}
|
||||
NATS_CI_PRODUCER_PASSWORD:
|
||||
valueFrom:
|
||||
secretKeyRef: {name: nats-auth, key: ci_producer_password}
|
||||
NATS_CI_WORKER_PASSWORD:
|
||||
valueFrom:
|
||||
secretKeyRef: {name: nats-auth, key: ci_worker_password}
|
||||
resources:
|
||||
requests: {cpu: 25m, memory: 64Mi}
|
||||
limits: {memory: 192Mi}
|
||||
|
||||
natsBox:
|
||||
enabled: false
|
||||
|
||||
promExporter:
|
||||
enabled: true
|
||||
podMonitor:
|
||||
enabled: true
|
||||
|
||||
service:
|
||||
merge:
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
Reference in New Issue
Block a user