From 110dbcc2c34753c08ad0cee2943458c04aac01da Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Fri, 18 Sep 2026 19:10:44 +0000 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=20Nexus=20=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E5=88=B6=E5=93=81=E4=BB=93=E5=BA=93=20POC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/nexus/README.md | 122 +++++++++++++++++++++++ apps/nexus/deployment.yaml | 82 +++++++++++++++ apps/nexus/httproute.yaml | 16 +++ apps/nexus/kustomization.yaml | 9 ++ apps/nexus/namespace.yaml | 8 ++ apps/nexus/networkpolicy.yaml | 23 +++++ apps/nexus/pvc.yaml | 12 +++ apps/nexus/service.yaml | 14 +++ apps/nexus/terraform/.gitignore | 6 ++ apps/nexus/terraform/.terraform.lock.hcl | 24 +++++ apps/nexus/terraform/repositories.tf | 64 ++++++++++++ apps/nexus/terraform/versions.tf | 16 +++ clusters/homelab/README.md | 2 + clusters/homelab/apps/nexus.yaml | 22 ++++ clusters/homelab/kustomization.yaml | 1 + infrastructure/dns/records.yml | 1 + 16 files changed, 422 insertions(+) create mode 100644 apps/nexus/README.md create mode 100644 apps/nexus/deployment.yaml create mode 100644 apps/nexus/httproute.yaml create mode 100644 apps/nexus/kustomization.yaml create mode 100644 apps/nexus/namespace.yaml create mode 100644 apps/nexus/networkpolicy.yaml create mode 100644 apps/nexus/pvc.yaml create mode 100644 apps/nexus/service.yaml create mode 100644 apps/nexus/terraform/.gitignore create mode 100644 apps/nexus/terraform/.terraform.lock.hcl create mode 100644 apps/nexus/terraform/repositories.tf create mode 100644 apps/nexus/terraform/versions.tf create mode 100644 clusters/homelab/apps/nexus.yaml diff --git a/apps/nexus/README.md b/apps/nexus/README.md new file mode 100644 index 0000000..53575d5 --- /dev/null +++ b/apps/nexus/README.md @@ -0,0 +1,122 @@ +# Nexus Repository POC + +本目录声明一个 Nexus Repository Community Edition POC，用来验证一次性 CI runner 通过 +网络服务复用 Ansible Galaxy、Go Modules 与 OCI/BuildKit 缓存。Nexus 固定为 `3.96.1`， +镜像固定到官方 multi-arch index digest；LAN 入口为 +`https://nexus.ad.ddupan.top`。 + +## POC 边界 + +- 单副本 Deployment，`Recreate` 更新，避免一个 RWO 卷被两个 Pod 同时挂载。 +- `/nexus-data` 使用 `localpv-zfs-ceph` 上的 50 GiB RWO PVC。 +- 资源预算为 250m/2 GiB request、2 CPU/4 GiB limit；JVM heap 上限 2 GiB。 +- 使用容器默认的 embedded H2。它只用于 POC；正式接管 OCI 制品前必须迁移到外部 + PostgreSQL，并验证备份恢复。 +- 入口只在 LAN wildcard Gateway 上发布，不创建公网 DNS 或 Cloudflare route。 +- 不套 Authelia forward-auth；它会破坏 Go、Ansible 与 OCI 非浏览器客户端。 +- 现有 zot 保持不变。Nexus 完成 OCI、BuildKit cache 和恢复验收前不得迁移或删除 zot。 + +POC 先由 Terraform 创建 `ansible-public` 和 `go-public`。Nexus 3.94 才加入的原生 OCI +repository 已有 REST API，但当前锁定的 community provider 尚未暴露 OCI resource；不能 +因此在 UI 中创建一个无人管理的长期 repository。POC 上线后先根据实例 Swagger 固定 +请求 schema，再补 Terraform provider resource 或受测试的声明式 REST 适配。 + +## 部署 + +Flux 从 `clusters/homelab/apps/nexus.yaml` 协调本目录，并依赖 Envoy Gateway 与 OpenEBS。 +合并前只渲染配置，不直接 apply： + +```bash +kubectl kustomize apps/nexus +``` + +合并并由 Flux 部署后检查： + +```bash +kubectl -n flux-system get kustomization nexus +kubectl -n nexus get pod,pvc,service,httproute +kubectl -n nexus logs deployment/nexus --tail=100 +``` + +启动可能需要数分钟，startup probe 允许最多十分钟。不要因初次启动较慢反复删除 Pod； +先确认 PVC 已 Bound、Pod 没有 OOM，以及日志仍在推进。 + +DNS 期望状态已加入 `infrastructure/dns/records.yml`，需从 Samba AD Ansible root 以 +`--check --diff` 核对后再按其 README 应用 DNS tag。没有 DNS 时可先用 port-forward +验证应用，但不能据此宣称 Gateway 路径已通过。 + +## 首次初始化与 Terraform + +初始管理员密码生成在 PVC 的 `/nexus-data/admin.password`。只在交互式终端中读取并立即 +完成首次密码轮换；不得把密码复制进 shell tracing、工单、Git 或命令参数。随后将 +Terraform 管理账号的凭据存入 OpenBao，由 CI 注入以下环境变量： + +```text +NXRM_SERVER_URL=https://nexus.ad.ddupan.top +NXRM_SERVER_USERNAME= +NXRM_SERVER_PASSWORD= +``` + +`terraform/` 使用 `sonatype-nexus-community/sonatyperepo` 1.17.0，当前声明： + +- `ansible-galaxy-proxy` → `https://galaxy.ansible.com` +- `ansible-public` group +- `go-proxy` → `https://proxy.golang.org` +- `go-public` group + +provider credential 不写入 HCL 或 tfvars。正式 apply 前还必须为这个独立 Terraform root +配置远端 backend；本地 state 只允许用于可丢弃的 POC，不提交。验证命令： + +```bash +terraform -chdir=apps/nexus/terraform init -backend=false +terraform -chdir=apps/nexus/terraform validate +``` + +## 客户端验收 + +先验证冷缓存，再原样重复命令验证热缓存；记录 Nexus 请求、上游流量和耗时，不只观察 +命令成功。Ansible 配置的 URL 必须以 `/` 结尾： + +```ini +[galaxy] +server_list = nexus + +[galaxy_server.nexus] +url = https://nexus.ad.ddupan.top/repository/ansible-public/ +``` + +```bash +ansible-galaxy collection install -r collections/requirements.yml \ + -p .ansible/collections +``` + +Go POC 使用： + +```bash +GOPROXY=https://nexus.ad.ddupan.top/repository/go-public/ go mod download +``` + +私有 `git.ddupan.top/*` module 的 `GOPRIVATE`、凭据与是否允许 `direct` fallback 在实际 +workflow 中单独决定；不要让私有 module path 意外发往公共 proxy。 + +OCI 阶段至少验证普通 push/pull、multi-arch、OCI 1.1 referrers、Cosign、Helm OCI， +以及 BuildKit `registry` cache 的 `mode=max` 导入导出。Nexus 上游已有 BuildKit cache +兼容性问题记录，因此该项失败时不能用普通镜像 push 成功代替。全部通过并完成备份恢复 +演练后，才评估 zot 迁移。 + +## 数据与恢复 + +POC 的数据库、配置、blob、初始管理员状态都位于 `nexus-data` PVC。删除 Deployment +不会删除 PVC；删除 PVC 会永久删除整个 POC。当前没有独立备份，不能将它用于唯一副本的 +正式制品。 + +恢复验证至少包括：停止写入、取得一致备份、在独立 PVC/实例恢复、登录、列出 repository、 +拉取已缓存的 Ansible/Go 制品，并核对 OCI digest/referrers。正式化时再把数据库迁移至 +外部 PostgreSQL，并分别定义数据库与 blob 的备份、恢复顺序和 RPO。 + +参考： + +- [Nexus OCI repositories](https://help.sonatype.com/en/oci-repositories.html) +- [Nexus Ansible repositories](https://help.sonatype.com/en/ansible-repositories.html) +- [Nexus Go repositories](https://help.sonatype.com/en/go-repositories.html) +- [官方容器镜像](https://hub.docker.com/r/sonatype/nexus3) diff --git a/apps/nexus/deployment.yaml b/apps/nexus/deployment.yaml new file mode 100644 index 0000000..21eb655 --- /dev/null +++ b/apps/nexus/deployment.yaml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nexus + namespace: nexus + labels: + app.kubernetes.io/name: nexus +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: nexus + template: + metadata: + labels: + app.kubernetes.io/name: nexus + spec: + automountServiceAccountToken: false + securityContext: + fsGroup: 200 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 200 + runAsNonRoot: true + runAsUser: 200 + seccompProfile: + type: RuntimeDefault + terminationGracePeriodSeconds: 120 + containers: + - name: nexus + image: docker.io/sonatype/nexus3:3.96.1@sha256:56142f13432cf072e017aebb2025f201e42ae36ff40bb82618c702504c61f7dd + imagePullPolicy: IfNotPresent + env: + - name: INSTALL4J_ADD_VM_PARAMS + value: >- + -Xms1024m -Xmx2048m -XX:MaxDirectMemorySize=1024m + -Djava.util.prefs.userRoot=/nexus-data/javaprefs + ports: + - name: http + containerPort: 8081 + protocol: TCP + resources: + requests: + cpu: 250m + memory: 2Gi + limits: + cpu: "2" + memory: 4Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + startupProbe: + httpGet: + path: /service/rest/v1/status + port: http + failureThreshold: 60 + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /service/rest/v1/status + port: http + failureThreshold: 6 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + httpGet: + path: /service/rest/v1/status + port: http + failureThreshold: 6 + periodSeconds: 30 + timeoutSeconds: 5 + volumeMounts: + - name: data + mountPath: /nexus-data + volumes: + - name: data + persistentVolumeClaim: + claimName: nexus-data diff --git a/apps/nexus/httproute.yaml b/apps/nexus/httproute.yaml new file mode 100644 index 0000000..bfc47ea --- /dev/null +++ b/apps/nexus/httproute.yaml @@ -0,0 +1,16 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: nexus + namespace: nexus +spec: + parentRefs: + - name: eg + namespace: envoy-gateway-system + sectionName: https + hostnames: + - nexus.ad.ddupan.top + rules: + - backendRefs: + - name: nexus + port: 8081 diff --git a/apps/nexus/kustomization.yaml b/apps/nexus/kustomization.yaml new file mode 100644 index 0000000..8019653 --- /dev/null +++ b/apps/nexus/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - pvc.yaml + - deployment.yaml + - service.yaml + - httproute.yaml + - networkpolicy.yaml diff --git a/apps/nexus/namespace.yaml b/apps/nexus/namespace.yaml new file mode 100644 index 0000000..eb0f88a --- /dev/null +++ b/apps/nexus/namespace.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: nexus + labels: + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted diff --git a/apps/nexus/networkpolicy.yaml b/apps/nexus/networkpolicy.yaml new file mode 100644 index 0000000..dbc527f --- /dev/null +++ b/apps/nexus/networkpolicy.yaml @@ -0,0 +1,23 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: nexus-ingress + namespace: nexus +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: nexus + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: envoy-gateway-system + podSelector: + matchLabels: + gateway.envoyproxy.io/owning-gateway-name: eg + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway-system + ports: + - protocol: TCP + port: 8081 diff --git a/apps/nexus/pvc.yaml b/apps/nexus/pvc.yaml new file mode 100644 index 0000000..4222484 --- /dev/null +++ b/apps/nexus/pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nexus-data + namespace: nexus +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + storageClassName: localpv-zfs-ceph diff --git a/apps/nexus/service.yaml b/apps/nexus/service.yaml new file mode 100644 index 0000000..7fe9fe4 --- /dev/null +++ b/apps/nexus/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: nexus + namespace: nexus +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: nexus + ports: + - name: http + port: 8081 + protocol: TCP + targetPort: http diff --git a/apps/nexus/terraform/.gitignore b/apps/nexus/terraform/.gitignore new file mode 100644 index 0000000..415e7f4 --- /dev/null +++ b/apps/nexus/terraform/.gitignore @@ -0,0 +1,6 @@ +.terraform/ +*.tfstate +*.tfstate.* +*.tfplan +crash.log +crash.*.log diff --git a/apps/nexus/terraform/.terraform.lock.hcl b/apps/nexus/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..1be8199 --- /dev/null +++ b/apps/nexus/terraform/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/sonatype-nexus-community/sonatyperepo" { + version = "1.17.0" + constraints = "1.17.0" + hashes = [ + "h1:uKhvhhhI7B+HBsh0zq/ybqKt+EnOGyI6rjcRCtj79ZA=", + "zh:0dde99e7b343fa01f8eefc378171fb8621bedb20f59157d6cc8e3d46c738105f", + "zh:3315929df254a3a6ac27c8c846c2006f7d2a91fadc014351bc4d617f948e5bf9", + "zh:36be5a455af3ce4e187de26753de63e78c1ee9a32dba0135c6cf96a6c1fff25f", + "zh:3f73f7ff57b8c339a7c7ac37653e2dc0b2dd9dcc3f3a538788e7e3ac838337b2", + "zh:40286ecca4c22ab7ae90618ac6d2743f5055199dac81cf5204a4a397c784d439", + "zh:4d24e5c0195fb3155b1967583ee64cfeda402d7cc7f3c73369438f6c69f4245b", + "zh:828a9d7aceaac36af7f9c07af43ec8d20a89148780645d170ffb1c68b2da792d", + "zh:a5ab04de3fe626ec57c832618c6f990abd6610f81e132621651e0b180b970cff", + "zh:a959fa6090a8c0f53739879184e7346423494aee598003df0d1ab4a22b2eee91", + "zh:bdda26c2f03f918bbe59e75abea44868fafda019c3a543725331195df126350b", + "zh:d8048e149ee97ba62971e6a79355d59887bc6d10fcf72cc2feff3d0a2582670c", + "zh:dd36f9988af4e1ca5b1ca7b7bb6f658df9a220dfcda7fec7392fedfe9064f652", + "zh:dda2688d46c7e539fe97e8fe9d3ec81fb364170e018d9c6a681364c8955d4e9d", + "zh:e6b519afe2dea1c0434f766eb6bc9ba78cc5b6ef2c311c2ca3c65cb24744f31f", + ] +} diff --git a/apps/nexus/terraform/repositories.tf b/apps/nexus/terraform/repositories.tf new file mode 100644 index 0000000..d47c8d3 --- /dev/null +++ b/apps/nexus/terraform/repositories.tf @@ -0,0 +1,64 @@ +locals { + proxy_http_client = { + auto_block = true + blocked = false + } + + proxy_negative_cache = { + enabled = true + time_to_live = 60 + } + + repository_storage = { + blob_store_name = "default" + strict_content_type_validation = true + } +} + +resource "sonatyperepo_repository_ansiblegalaxy_proxy" "galaxy" { + name = "ansible-galaxy-proxy" + online = true + + http_client = local.proxy_http_client + negative_cache = local.proxy_negative_cache + proxy = { + remote_url = "https://galaxy.ansible.com" + content_max_age = 1440 + metadata_max_age = 60 + } + storage = local.repository_storage +} + +resource "sonatyperepo_repository_ansiblegalaxy_group" "public" { + name = "ansible-public" + online = true + + group = { + member_names = [sonatyperepo_repository_ansiblegalaxy_proxy.galaxy.name] + } + storage = local.repository_storage +} + +resource "sonatyperepo_repository_go_proxy" "public" { + name = "go-proxy" + online = true + + http_client = local.proxy_http_client + negative_cache = local.proxy_negative_cache + proxy = { + remote_url = "https://proxy.golang.org" + content_max_age = 1440 + metadata_max_age = 60 + } + storage = local.repository_storage +} + +resource "sonatyperepo_repository_go_group" "public" { + name = "go-public" + online = true + + group = { + member_names = [sonatyperepo_repository_go_proxy.public.name] + } + storage = local.repository_storage +} diff --git a/apps/nexus/terraform/versions.tf b/apps/nexus/terraform/versions.tf new file mode 100644 index 0000000..21d81a3 --- /dev/null +++ b/apps/nexus/terraform/versions.tf @@ -0,0 +1,16 @@ +terraform { + required_version = ">= 1.11.0" + + required_providers { + sonatyperepo = { + source = "sonatype-nexus-community/sonatyperepo" + version = "1.17.0" + } + } +} + +provider "sonatyperepo" { + # NXRM_SERVER_URL, NXRM_SERVER_USERNAME and NXRM_SERVER_PASSWORD are injected + # from OpenBao by CI. + # Provider credentials must never be committed or placed in tfvars/state. +} diff --git a/clusters/homelab/README.md b/clusters/homelab/README.md index 0eae482..9c2e2b8 100644 --- a/clusters/homelab/README.md +++ b/clusters/homelab/README.md @@ -53,4 +53,6 @@ sudo k3s kubectl -n flux-system get gitrepositories,kustomizations - SPIRE 已按 hardened chart 内部 fork `0.30.2-ddupan.1`（基于上游 `0.30.2`，SPIRE `1.15.3`）声明，使用共享 PostgreSQL 与独立 signing-key PVC；首次上线和 OpenBao JWT-SVID PoC 尚待合并后验证； +- Nexus Repository CE POC 已加入 GitOps 声明，计划验证 Ansible Galaxy、Go Modules 与 + OCI/BuildKit 缓存；尚未部署或完成现场验收，现有 zot 保持不变； - root Kustomization 与所有 brownfield 子 Kustomization 继续保持 `prune: false`。 diff --git a/clusters/homelab/apps/nexus.yaml b/clusters/homelab/apps/nexus.yaml new file mode 100644 index 0000000..3ea510c --- /dev/null +++ b/clusters/homelab/apps/nexus.yaml @@ -0,0 +1,22 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: nexus + namespace: flux-system +spec: + dependsOn: + - name: envoy-gateway + - name: openebs + healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: nexus + namespace: nexus + interval: 10m + path: ./apps/nexus + prune: false + sourceRef: + kind: GitRepository + name: flux-system + timeout: 15m + wait: true diff --git a/clusters/homelab/kustomization.yaml b/clusters/homelab/kustomization.yaml index 00b74f8..7d55d8d 100644 --- a/clusters/homelab/kustomization.yaml +++ b/clusters/homelab/kustomization.yaml @@ -15,3 +15,4 @@ resources: - apps/spire.yaml - apps/observability.yaml - apps/zot.yaml + - apps/nexus.yaml diff --git a/infrastructure/dns/records.yml b/infrastructure/dns/records.yml index f467def..1b71929 100644 --- a/infrastructure/dns/records.yml +++ b/infrastructure/dns/records.yml @@ -18,6 +18,7 @@ homelab_dns: - { zone: ad.ddupan.top, name: metrics-write, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: netbox, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: nats, type: A, values: [192.168.10.127] } + - { zone: ad.ddupan.top, name: nexus, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: s3, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: spire-oidc, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: spire-server, type: A, values: [192.168.10.127] } -- 2.54.0