docs: 完善开发与测试环境
This commit is contained in:
@@ -29,6 +29,10 @@ case "${MACHINE}" in
|
|||||||
esac
|
esac
|
||||||
echo "Architecture: ${ARCH}"
|
echo "Architecture: ${ARCH}"
|
||||||
|
|
||||||
|
KIND_VERSION="v0.33.0"
|
||||||
|
KUBEBUILDER_VERSION="v4.15.0"
|
||||||
|
KUBECTL_VERSION="v1.36.0"
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "------------------------------------"
|
echo "------------------------------------"
|
||||||
echo "Setting up bash completion..."
|
echo "Setting up bash completion..."
|
||||||
@@ -50,7 +54,7 @@ echo "------------------------------------"
|
|||||||
# Install kind
|
# Install kind
|
||||||
if ! command -v kind &> /dev/null; then
|
if ! command -v kind &> /dev/null; then
|
||||||
echo "Installing kind..."
|
echo "Installing kind..."
|
||||||
curl -Lo /usr/local/bin/kind "https://kind.sigs.k8s.io/dl/latest/kind-linux-${ARCH}"
|
curl -Lo /usr/local/bin/kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-${ARCH}"
|
||||||
chmod +x /usr/local/bin/kind
|
chmod +x /usr/local/bin/kind
|
||||||
echo "kind installed successfully"
|
echo "kind installed successfully"
|
||||||
fi
|
fi
|
||||||
@@ -67,7 +71,7 @@ fi
|
|||||||
# Install kubebuilder
|
# Install kubebuilder
|
||||||
if ! command -v kubebuilder &> /dev/null; then
|
if ! command -v kubebuilder &> /dev/null; then
|
||||||
echo "Installing kubebuilder..."
|
echo "Installing kubebuilder..."
|
||||||
curl -Lo /usr/local/bin/kubebuilder "https://go.kubebuilder.io/dl/latest/linux/${ARCH}"
|
curl -Lo /usr/local/bin/kubebuilder "https://github.com/kubernetes-sigs/kubebuilder/releases/download/${KUBEBUILDER_VERSION}/kubebuilder_linux_${ARCH}"
|
||||||
chmod +x /usr/local/bin/kubebuilder
|
chmod +x /usr/local/bin/kubebuilder
|
||||||
echo "kubebuilder installed successfully"
|
echo "kubebuilder installed successfully"
|
||||||
fi
|
fi
|
||||||
@@ -84,7 +88,6 @@ fi
|
|||||||
# Install kubectl
|
# Install kubectl
|
||||||
if ! command -v kubectl &> /dev/null; then
|
if ! command -v kubectl &> /dev/null; then
|
||||||
echo "Installing kubectl..."
|
echo "Installing kubectl..."
|
||||||
KUBECTL_VERSION=$(curl -Ls https://dl.k8s.io/release/stable.txt)
|
|
||||||
curl -Lo /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl"
|
curl -Lo /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl"
|
||||||
chmod +x /usr/local/bin/kubectl
|
chmod +x /usr/local/bin/kubectl
|
||||||
echo "kubectl installed successfully"
|
echo "kubectl installed successfully"
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
name: E2E Tests
|
name: E2E Tests
|
||||||
|
|
||||||
|
env:
|
||||||
|
KIND_VERSION: v0.33.0
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
@@ -26,10 +29,10 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
go-version-file: go.mod
|
go-version-file: go.mod
|
||||||
|
|
||||||
- name: Install the latest version of kind
|
- name: Install kind
|
||||||
run: |
|
run: |
|
||||||
mkdir -p ./bin
|
mkdir -p ./bin
|
||||||
curl -Lo ./bin/kind https://kind.sigs.k8s.io/dl/latest/kind-linux-$(go env GOARCH)
|
curl -Lo ./bin/kind https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-$(go env GOARCH)
|
||||||
chmod +x ./bin/kind
|
chmod +x ./bin/kind
|
||||||
|
|
||||||
- name: Verify Docker and kind
|
- name: Verify Docker and kind
|
||||||
|
|||||||
@@ -16,6 +16,9 @@ endif
|
|||||||
# tools. (i.e. podman)
|
# tools. (i.e. podman)
|
||||||
CONTAINER_TOOL ?= docker
|
CONTAINER_TOOL ?= docker
|
||||||
|
|
||||||
|
# DEV_COMPOSE manages disposable PostgreSQL and OpenBao dependencies for local work.
|
||||||
|
DEV_COMPOSE ?= docker compose -f hack/dev/compose.yaml
|
||||||
|
|
||||||
# Setting SHELL to bash allows bash commands to be executed by recipes.
|
# Setting SHELL to bash allows bash commands to be executed by recipes.
|
||||||
# Options are set to exit when a recipe line exits non-zero or a piped command fails.
|
# Options are set to exit when a recipe line exits non-zero or a piped command fails.
|
||||||
SHELL = /usr/bin/env bash -o pipefail
|
SHELL = /usr/bin/env bash -o pipefail
|
||||||
@@ -43,6 +46,19 @@ help: ## Display this help.
|
|||||||
|
|
||||||
##@ Development
|
##@ Development
|
||||||
|
|
||||||
|
.PHONY: dev-up
|
||||||
|
dev-up: ## Start disposable PostgreSQL and OpenBao development dependencies.
|
||||||
|
$(DEV_COMPOSE) up -d --wait
|
||||||
|
|
||||||
|
.PHONY: dev-smoke
|
||||||
|
dev-smoke: dev-up ## Verify PostgreSQL and OpenBao development dependencies.
|
||||||
|
$(DEV_COMPOSE) exec -T postgres psql -U postgres -d postgres -v ON_ERROR_STOP=1 -c 'SELECT 1'
|
||||||
|
$(DEV_COMPOSE) exec -T openbao sh -ec 'bao kv put secret/postgresql-admin username=postgres password=postgres-dev-only >/dev/null; test "$$(bao kv get -field=username secret/postgresql-admin)" = postgres'
|
||||||
|
|
||||||
|
.PHONY: dev-down
|
||||||
|
dev-down: ## Remove disposable development dependencies and their data.
|
||||||
|
$(DEV_COMPOSE) down --volumes --remove-orphans
|
||||||
|
|
||||||
.PHONY: manifests
|
.PHONY: manifests
|
||||||
manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
|
manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
|
||||||
"$(CONTROLLER_GEN)" rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
|
"$(CONTROLLER_GEN)" rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
|
||||||
|
|||||||
@@ -38,6 +38,8 @@ spec:
|
|||||||
[`docs/architecture.md`](docs/architecture.md)。
|
[`docs/architecture.md`](docs/architecture.md)。
|
||||||
|
|
||||||
分支、提交、PR 和 CI 约定见 [`CONTRIBUTING.md`](CONTRIBUTING.md)。
|
分支、提交、PR 和 CI 约定见 [`CONTRIBUTING.md`](CONTRIBUTING.md)。
|
||||||
|
Docker/devcontainer、PostgreSQL、OpenBao、envtest 与 Kind 的启动顺序见
|
||||||
|
[`docs/development.md`](docs/development.md)。
|
||||||
|
|
||||||
## 本地开发
|
## 本地开发
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,164 @@
|
|||||||
|
# 开发与测试环境
|
||||||
|
|
||||||
|
本项目同时依赖 Kubernetes API、PostgreSQL 和 OpenBao。日常开发不连接 homelab
|
||||||
|
中的真实服务:Kubernetes 使用 envtest 或一次性 Kind,另外两个依赖使用一次性
|
||||||
|
容器。这样既避免污染真实数据,也能把启动顺序固化为命令。
|
||||||
|
|
||||||
|
## 是否需要开发 VM
|
||||||
|
|
||||||
|
默认不需要。仓库的 devcontainer 使用独立 Docker-in-Docker daemon,Go 工具链、
|
||||||
|
Kind 节点和依赖容器都与宿主机环境隔离。宿主机只需要能够运行支持 privileged
|
||||||
|
container 的 Docker/Dev Container 环境。
|
||||||
|
|
||||||
|
只有以下情况才建议增加一台可随时重建的开发 VM:
|
||||||
|
|
||||||
|
- 宿主机不允许 privileged devcontainer;
|
||||||
|
- 无法安全使用 Docker socket 或 Docker-in-Docker;
|
||||||
|
- 本机地址段与 Kind/Docker 网络持续冲突;
|
||||||
|
- 需要长期运行、接近 homelab 网络和 TLS 配置的验收环境。
|
||||||
|
|
||||||
|
即使使用 VM,也应在 VM 内继续执行本文相同的容器化流程;不要把 VM 配置成第二套
|
||||||
|
手工维护的开发环境。
|
||||||
|
|
||||||
|
## 环境分层
|
||||||
|
|
||||||
|
| 层次 | Kubernetes | PostgreSQL / OpenBao | 用途 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| 单元测试 | fake client | fake client | SQL 计划、状态转换和错误分类 |
|
||||||
|
| controller 集成测试 | envtest | fake 或 Docker | CRD、watch、status、finalizer |
|
||||||
|
| adapter 集成测试 | 不需要 | Docker Compose | 真实协议、权限和幂等行为 |
|
||||||
|
| E2E | 一次性 Kind | Kind 内测试实例 | 验证容器化 controller 与完整网络路径 |
|
||||||
|
|
||||||
|
envtest 只启动 API server 和 etcd,没有 kubelet、scheduler 或 controller-manager,
|
||||||
|
因此不能用它验证 Deployment、Pod 调度或 Service 网络。此类行为必须留给 Kind
|
||||||
|
E2E。
|
||||||
|
|
||||||
|
## 首次准备
|
||||||
|
|
||||||
|
推荐用支持 Dev Containers 的编辑器打开仓库。devcontainer 会提供 Go、Docker、
|
||||||
|
Kubebuilder、Kind 和 kubectl。脚本固定 Kubebuilder 4.15.0、Kind 0.33.0 和
|
||||||
|
kubectl 1.36.0,与当前脚手架和 Kubernetes Go module 对齐。容器启动后先确认:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go version
|
||||||
|
docker info
|
||||||
|
kubebuilder version
|
||||||
|
kind version
|
||||||
|
kubectl version --client
|
||||||
|
```
|
||||||
|
|
||||||
|
不要在仓库中保存真实 OpenBao Token、数据库密码或 kubeconfig。Compose 中的
|
||||||
|
`postgres-dev-only` 和 `dev-only-root-token` 是仅绑定回环地址、随容器销毁的公开
|
||||||
|
测试值,不得复制到其他环境。
|
||||||
|
|
||||||
|
## 日常开发的正确顺序
|
||||||
|
|
||||||
|
### 1. 生成并验证纯 Go/Kubernetes 部分
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make manifests generate
|
||||||
|
make test
|
||||||
|
make lint
|
||||||
|
```
|
||||||
|
|
||||||
|
`make test` 会下载与 `go.mod` 中 Kubernetes minor 版本匹配的 envtest 二进制,
|
||||||
|
启动临时 API server/etcd,测试结束后自动关闭。
|
||||||
|
|
||||||
|
### 2. 启动外部依赖
|
||||||
|
|
||||||
|
只有开发 PostgreSQL/OpenBao adapter 或完整 reconcile 时才需要:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make dev-up
|
||||||
|
make dev-smoke
|
||||||
|
```
|
||||||
|
|
||||||
|
启动顺序由 Compose healthcheck 保证:
|
||||||
|
|
||||||
|
1. 创建独立 Compose 网络;
|
||||||
|
2. 启动 PostgreSQL 和 OpenBao;
|
||||||
|
3. 等待 PostgreSQL `pg_isready` 成功;
|
||||||
|
4. 等待 OpenBao `bao status` 成功;
|
||||||
|
5. smoke test 执行 `SELECT 1`;
|
||||||
|
6. smoke test 在 OpenBao dev server 默认的 `secret/` KV v2 mount 写入并读回测试管理
|
||||||
|
凭据。
|
||||||
|
|
||||||
|
本机进程使用以下端点:
|
||||||
|
|
||||||
|
```text
|
||||||
|
PostgreSQL: postgresql://postgres:[email protected]:15432/postgres
|
||||||
|
OpenBao: http://127.0.0.1:18200
|
||||||
|
Token: dev-only-root-token
|
||||||
|
```
|
||||||
|
|
||||||
|
若端口冲突,可以只对当前命令覆盖:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
POSTGRES_DEV_PORT=25432 OPENBAO_DEV_PORT=28200 make dev-up
|
||||||
|
```
|
||||||
|
|
||||||
|
后续执行 `dev-smoke` 和 controller 时必须使用相同端口变量。
|
||||||
|
|
||||||
|
### 3. 运行针对临时依赖的测试或 controller
|
||||||
|
|
||||||
|
adapter 集成测试加入后,统一通过独立 Make target 执行,不默认塞进快速单元测试。
|
||||||
|
本机运行 controller 时,先确认当前 kubeconfig 指向专用 Kind,而不是真实 homelab:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl config current-context
|
||||||
|
make setup-test-e2e
|
||||||
|
kubectl config current-context
|
||||||
|
make install
|
||||||
|
make run
|
||||||
|
```
|
||||||
|
|
||||||
|
此时 controller 运行在开发容器内,可以直接访问上面的回环端口。不要把包含
|
||||||
|
`127.0.0.1` 端点的样例部署到 Kind 内;Pod 中的回环地址只指向 Pod 自身。
|
||||||
|
|
||||||
|
### 4. 清理
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make dev-down
|
||||||
|
make cleanup-test-e2e
|
||||||
|
```
|
||||||
|
|
||||||
|
`dev-down` 会删除 Compose volume;所有数据库和 OpenBao dev 数据都应视为一次性。
|
||||||
|
|
||||||
|
## E2E 顺序
|
||||||
|
|
||||||
|
CI 的 E2E 与本机 `make run` 不同:controller 会作为 Pod 运行在 Kind 中。因此完整
|
||||||
|
E2E fixture 必须把测试 PostgreSQL 和 OpenBao也部署进 Kind,并等待两者 Ready 后
|
||||||
|
再创建 `PostgreSQLInstance` 和 `PostgreSQLTenant`:
|
||||||
|
|
||||||
|
```text
|
||||||
|
创建 Kind
|
||||||
|
-> 安装 CRD
|
||||||
|
-> 部署 PostgreSQL/OpenBao fixture
|
||||||
|
-> 等待依赖 Ready并写入测试管理凭据
|
||||||
|
-> 构建并加载 controller image
|
||||||
|
-> 部署 controller
|
||||||
|
-> 创建 Instance
|
||||||
|
-> 等待 Instance Ready
|
||||||
|
-> 创建 Tenant
|
||||||
|
-> 等待 Tenant Ready
|
||||||
|
-> 验证 PostgreSQL catalog 与 OpenBao KV
|
||||||
|
-> 删除 Tenant并验证 Retain
|
||||||
|
-> 删除 Kind
|
||||||
|
```
|
||||||
|
|
||||||
|
当前 controller 尚未实现 PostgreSQL/OpenBao adapter,脚手架 E2E 只能验证 CRD、
|
||||||
|
manager Deployment 和 metrics endpoint。上述真实依赖 fixture 应与第一个完整
|
||||||
|
reconcile 纵向切片一起实现,不能在文档中声称已经通过。
|
||||||
|
|
||||||
|
## 故障排查
|
||||||
|
|
||||||
|
查看依赖状态与日志:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose -f hack/dev/compose.yaml ps
|
||||||
|
docker compose -f hack/dev/compose.yaml logs postgres openbao
|
||||||
|
```
|
||||||
|
|
||||||
|
如果 envtest 报端口监听失败,通常是当前执行环境禁止监听回环端口,而非 controller
|
||||||
|
失败;在 devcontainer 或允许本机监听的 runner 中执行。若 Kind 无法创建,先运行
|
||||||
|
`docker info`,确认当前用户可以访问 devcontainer 内的 Docker daemon。
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
name: postgresql-tenant-operator-dev
|
||||||
|
|
||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: ${POSTGRES_IMAGE:-postgres:17-alpine}
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: postgres
|
||||||
|
POSTGRES_USER: postgres
|
||||||
|
POSTGRES_PASSWORD: postgres-dev-only
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:${POSTGRES_DEV_PORT:-15432}:5432"
|
||||||
|
tmpfs:
|
||||||
|
- /var/lib/postgresql/data
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
|
||||||
|
openbao:
|
||||||
|
image: ${OPENBAO_IMAGE:-openbao/openbao:2.6.1}
|
||||||
|
command: server -dev
|
||||||
|
environment:
|
||||||
|
BAO_ADDR: http://127.0.0.1:8200
|
||||||
|
BAO_DEV_LISTEN_ADDRESS: 0.0.0.0:8200
|
||||||
|
BAO_DEV_ROOT_TOKEN_ID: dev-only-root-token
|
||||||
|
BAO_TOKEN: dev-only-root-token
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:${OPENBAO_DEV_PORT:-18200}:8200"
|
||||||
|
cap_add:
|
||||||
|
- IPC_LOCK
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "bao", "status"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
Reference in New Issue
Block a user