diff --git a/clusters/sandbox/README.md b/clusters/sandbox/README.md index 0f2f273..8559e17 100644 --- a/clusters/sandbox/README.md +++ b/clusters/sandbox/README.md @@ -15,7 +15,7 @@ Root bootstrap 已完成。后续按依赖顺序分别引入： 1. 监控 CRD、kube-state-metrics 以及 kubelet/cAdvisor 抓取配置； 2. SPIRE Agent、SPIFFE CSI Driver 与 workload registration； 3. Kata Containers、`block-plain` RuntimeClass； -4. OpenSandbox operator/server 及 `ci-pod`、`ci-vm` Pools。 +4. OpenSandbox controller/server；CI Pool 与 runner 调度器随后独立接入。 每一阶段单独合并并等待对应 Flux Kustomization Ready，不在 bootstrap 时一次性部署。 第一阶段监控拆为 `monitoring-operator` 与依赖它的 `monitoring`，防止 VM CR 在 @@ -54,6 +54,11 @@ Kata 阶段使用官方 4.1.0 `kata-deploy` chart 的短生命周期 `job` 模 `emptyDir` 固定使用 `block-plain`，为 Docker/BuildKit overlay2 与 kind 提供 guest 内块设备文件系统。详细限制与上线验收见 `platform/sandbox-kata/README.md`。 +OpenSandbox 阶段固定官方源码 commit 与 umbrella chart `0.2.2`，只部署 controller、 +ClusterIP server 和 CRD。server 当前仅能从集群内部访问；在 CI 调度器接入并建立 +API key 的 Secret 生命周期前，显式运行于无认证 bootstrap 模式。这个临时边界和切换 +步骤记录在 `platform/sandbox-opensandbox/README.md`。 + ## 监控边界 这里只管理 sandbox LXC 内的 Kubernetes 监控，不负责 PVE 宿主监控。LXC 与宿主共享 diff --git a/clusters/sandbox/apps/opensandbox.yaml b/clusters/sandbox/apps/opensandbox.yaml new file mode 100644 index 0000000..59b26ae --- /dev/null +++ b/clusters/sandbox/apps/opensandbox.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: opensandbox + namespace: flux-system +spec: + dependsOn: + - name: kata + - name: monitoring-operator + interval: 10m + path: ./platform/sandbox-opensandbox + prune: true + sourceRef: + kind: GitRepository + name: flux-system + timeout: 15m + wait: true diff --git a/clusters/sandbox/kustomization.yaml b/clusters/sandbox/kustomization.yaml index fa2ce88..e23a3f3 100644 --- a/clusters/sandbox/kustomization.yaml +++ b/clusters/sandbox/kustomization.yaml @@ -7,3 +7,4 @@ resources: - apps/spire-bootstrap.yaml - apps/spire-agents.yaml - apps/kata.yaml + - apps/opensandbox.yaml diff --git a/infrastructure/sandbox-cluster/README.md b/infrastructure/sandbox-cluster/README.md index 640eddc..1fde90e 100644 --- a/infrastructure/sandbox-cluster/README.md +++ b/infrastructure/sandbox-cluster/README.md @@ -18,7 +18,7 @@ Flux 管理以下 Kubernetes 资源： - Kata Containers 和 CI 专用的 `block-plain` RuntimeClass； - SPIRE Agent、SPIFFE CSI Driver 与 workload identity 声明； - vmagent、kube-state-metrics、kubelet/cAdvisor scrape 配置和告警； -- OpenSandbox operator/server、`ci-pod` 与 `ci-vm` Pools。 +- OpenSandbox controller/server；CI Pool 与 runner 调度器由 runner 项目接入。 同一个对象只能有一个 owner。Ansible 不直接部署上述集群内 workload；Flux 不管理 LXC、K3s datastore 或 K3s 本身。 diff --git a/platform/sandbox-opensandbox/README.md b/platform/sandbox-opensandbox/README.md new file mode 100644 index 0000000..2aaa5fc --- /dev/null +++ b/platform/sandbox-opensandbox/README.md @@ -0,0 +1,44 @@ +# Sandbox OpenSandbox + +本目录在独立 sandbox k3s 集群部署 OpenSandbox controller、server 与 CRD。Flux 从 +上游 commit `8f01e935c2cabba778cf37a152033fae062fa0f4` 构建官方 umbrella chart +`0.2.2`；该源码渲染结果已与 release `opensandbox-0.2.2.tgz` 对比一致。不要改为跟随 +浮动 branch 或 tag。 + +server 只提供集群内 `opensandbox-server.opensandbox-system.svc:80` ClusterIP，不部署 +Gateway、Ingress 或 LoadBalancer。sandbox workload 位于 `opensandbox` namespace， +默认使用 `kata-clh-runtime-rs`；CI Pool、runner 镜像、动态 SPIFFE registration 均由 +runner 项目后续声明，本目录不预制。 + +## 临时认证边界 + +当前尚无 sandbox 集群内的 Secret 分发链路。为避免把长期凭据提交到 Git，server 暂时 +通过 `OPENSANDBOX_INSECURE_SERVER=YES` 显式确认无认证模式；其网络边界严格保持为 +ClusterIP。这不是最终认证方案。 + +runner 接入前必须先创建 `opensandbox-api-key` Secret，并把 Helm values 中的环境变量 +改为： + +```yaml +- name: OPENSANDBOX_SERVER_API_KEY + valueFrom: + secretKeyRef: + name: opensandbox-api-key + key: api-key +``` + +随后删除 `OPENSANDBOX_INSECURE_SERVER`。Secret 必须由 OpenBao/SPIFFE 派生的自动化 +链路或 Ansible 注入，不得把明文写入仓库。 + +## 验收 + +合并后等待 `flux-system/opensandbox` 与 `opensandbox-system/opensandbox` Ready，并确认： + +```bash +kubectl get crd batchsandboxes.sandbox.opensandbox.io pools.sandbox.opensandbox.io +kubectl -n opensandbox-system get deploy,pod,svc +kubectl get runtimeclass kata-clh-runtime-rs +``` + +控制面上线不创建 CI Pool，也不产生 sandbox workload。首个 runner 集成应另行提交 Pool +与完整的 Lifecycle API smoke test。 diff --git a/platform/sandbox-opensandbox/helmrelease.yaml b/platform/sandbox-opensandbox/helmrelease.yaml new file mode 100644 index 0000000..bb2773e --- /dev/null +++ b/platform/sandbox-opensandbox/helmrelease.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: opensandbox + namespace: opensandbox-system +spec: + chart: + spec: + chart: ./kubernetes/charts/opensandbox + interval: 1h + reconcileStrategy: Revision + sourceRef: + kind: GitRepository + name: opensandbox + driftDetection: + mode: enabled + install: + crds: CreateReplace + strategy: + name: RetryOnFailure + retryInterval: 5m + interval: 30m + releaseName: opensandbox + targetNamespace: opensandbox-system + timeout: 15m + upgrade: + crds: CreateReplace + strategy: + name: RetryOnFailure + retryInterval: 5m + valuesFrom: + - kind: ConfigMap + name: opensandbox-values + valuesKey: values.yaml diff --git a/platform/sandbox-opensandbox/kustomization.yaml b/platform/sandbox-opensandbox/kustomization.yaml new file mode 100644 index 0000000..886577f --- /dev/null +++ b/platform/sandbox-opensandbox/kustomization.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - repository.yaml + - template.yaml + - values.yaml + - helmrelease.yaml + - monitor-scrape.yaml diff --git a/platform/sandbox-opensandbox/monitor-scrape.yaml b/platform/sandbox-opensandbox/monitor-scrape.yaml new file mode 100644 index 0000000..40db141 --- /dev/null +++ b/platform/sandbox-opensandbox/monitor-scrape.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: opensandbox-controller + namespace: monitoring +spec: + namespaceSelector: + matchNames: + - opensandbox-system + podMetricsEndpoints: + - interval: 30s + port: metrics + selector: + matchLabels: + app.kubernetes.io/name: opensandbox diff --git a/platform/sandbox-opensandbox/namespace.yaml b/platform/sandbox-opensandbox/namespace.yaml new file mode 100644 index 0000000..b41233e --- /dev/null +++ b/platform/sandbox-opensandbox/namespace.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: opensandbox-system +--- +apiVersion: v1 +kind: Namespace +metadata: + name: opensandbox diff --git a/platform/sandbox-opensandbox/repository.yaml b/platform/sandbox-opensandbox/repository.yaml new file mode 100644 index 0000000..65af355 --- /dev/null +++ b/platform/sandbox-opensandbox/repository.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: opensandbox + namespace: opensandbox-system +spec: + interval: 1h + ref: + commit: 8f01e935c2cabba778cf37a152033fae062fa0f4 + url: https://github.com/opensandbox-group/OpenSandbox.git diff --git a/platform/sandbox-opensandbox/template.yaml b/platform/sandbox-opensandbox/template.yaml new file mode 100644 index 0000000..a4128d5 --- /dev/null +++ b/platform/sandbox-opensandbox/template.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: opensandbox-batchsandbox-template + namespace: opensandbox-system +data: + batchsandbox-template.yaml: | + metadata: + labels: + ddupan.top/workload-class: sandbox + spec: + replicas: 1 + template: + spec: + restartPolicy: Never + terminationGracePeriodSeconds: 30 diff --git a/platform/sandbox-opensandbox/values.yaml b/platform/sandbox-opensandbox/values.yaml new file mode 100644 index 0000000..ab70075 --- /dev/null +++ b/platform/sandbox-opensandbox/values.yaml @@ -0,0 +1,80 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: opensandbox-values + namespace: opensandbox-system +data: + values.yaml: | + opensandbox-controller: + controller: + metrics: + enabled: true + port: 8080 + secure: false + + opensandbox-server: + server: + replicaCount: 1 + env: + # 临时 bootstrap 边界：Service 仅为 ClusterIP。runner 接入时改为 + # secretKeyRef(OPENSANDBOX_SERVER_API_KEY) 并删除本项。 + - name: OPENSANDBOX_INSECURE_SERVER + value: "YES" + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: 100m + memory: 256Mi + volumeMounts: + - name: batchsandbox-template + mountPath: /etc/opensandbox/batchsandbox-template.yaml + subPath: batchsandbox-template.yaml + readOnly: true + volumes: + - name: batchsandbox-template + configMap: + name: opensandbox-batchsandbox-template + + configToml: | + [server] + host = "0.0.0.0" + port = 80 + api_key = "" + max_sandbox_timeout_seconds = 86400 + + [log] + level = "INFO" + + [runtime] + type = "kubernetes" + execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22" + + [storage] + allowed_host_paths = [] + volume_default_size = "1Gi" + + [kubernetes] + kubeconfig_path = "" + namespace = "opensandbox" + informer_enabled = true + informer_resync_seconds = 300 + informer_watch_timeout_seconds = 60 + snapshot_create_timeout_seconds = 900 + workload_provider = "batchsandbox" + image_pull_policy = "IfNotPresent" + batchsandbox_template_file = "/etc/opensandbox/batchsandbox-template.yaml" + + [egress] + image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.6" + mode = "dns+nft" + disable_ipv6 = true + + [secure_runtime] + type = "kata" + k8s_runtime_class = "kata-clh-runtime-rs" + + opensandbox-node-agent: + enabled: false