diff --git a/.gitea/workflows/verify.yml b/.gitea/workflows/verify.yml index ca0d426..1bd70d4 100644 --- a/.gitea/workflows/verify.yml +++ b/.gitea/workflows/verify.yml @@ -25,6 +25,11 @@ jobs: make test git diff --exit-code + - name: Build controller entrypoint + run: | + make build + ./bin/manager --help + lint: runs-on: [self-hosted, pod] steps: diff --git a/cmd/main.go b/cmd/main.go index 6215b1a..08a1f8c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,220 +1,15 @@ package main import ( - "context" - "crypto/tls" - "flag" "os" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" - - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "git.ddupan.top/panxiao81/ayatori/internal/bootstrap" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/healthz" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/metrics/filters" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "sigs.k8s.io/controller-runtime/pkg/webhook" - - databasev1alpha1 "git.ddupan.top/panxiao81/ayatori/api/database/v1alpha1" - executionv1alpha1 "git.ddupan.top/panxiao81/ayatori/api/execution/v1alpha1" - "git.ddupan.top/panxiao81/ayatori/internal/database/application" - databasecontroller "git.ddupan.top/panxiao81/ayatori/internal/database/controller" - // +kubebuilder:scaffold:imports ) -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") -) - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - - utilruntime.Must(executionv1alpha1.AddToScheme(scheme)) - utilruntime.Must(databasev1alpha1.AddToScheme(scheme)) - // +kubebuilder:scaffold:scheme -} - -// nolint:gocyclo func main() { - var openBao openBaoOptions - openBao.bindFlags(flag.CommandLine) - var databaseNamespace, databaseRootCert string - flag.StringVar(&databaseNamespace, "database-secret-namespace", os.Getenv("POD_NAMESPACE"), - "固定管理 Secret namespace；为空时不启用 Instance 观测") - flag.StringVar(&databaseRootCert, "database-root-cert", "", "PostgreSQL 管理连接信任的公开 CA bundle 路径") - var metricsAddr string - var metricsCertPath, metricsCertName, metricsCertKey string - var webhookCertPath, webhookCertName, webhookCertKey string - var webhookPort int - var enableLeaderElection bool - var probeAddr string - var secureMetrics bool - var enableHTTP2 bool - var tlsOpts []func(*tls.Config) - flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ - "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", true, - "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") - flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.") - flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.") - flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.") - flag.IntVar(&webhookPort, "webhook-port", 9443, "Port the webhook server listens on. "+ - "Defaults to 9443. Set -1 to disable the webhook server.") - flag.StringVar(&metricsCertPath, "metrics-cert-path", "", - "The directory that contains the metrics server certificate.") - flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") - flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") - flag.BoolVar(&enableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") - opts := zap.Options{ - Development: true, - } - opts.BindFlags(flag.CommandLine) - flag.Parse() - - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - - // if the enable-http2 flag is false (the default), http/2 should be disabled - // due to its vulnerabilities. More specifically, disabling http/2 will - // prevent from being vulnerable to the HTTP/2 Stream Cancellation and - // Rapid Reset CVEs. For more information see: - // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 - // - https://github.com/advisories/GHSA-4374-p667-p6c8 - disableHTTP2 := func(c *tls.Config) { - setupLog.Info("Disabling HTTP/2") - c.NextProtos = []string{"http/1.1"} - } - - if !enableHTTP2 { - tlsOpts = append(tlsOpts, disableHTTP2) - } - - // Initial webhook TLS options - webhookTLSOpts := tlsOpts - webhookServerOptions := webhook.Options{ - TLSOpts: webhookTLSOpts, - Port: webhookPort, - } - - if len(webhookCertPath) > 0 { - setupLog.Info("Initializing webhook certificate watcher using provided certificates", - "webhook-cert-path", webhookCertPath, "webhook-cert-name", webhookCertName, "webhook-cert-key", webhookCertKey) - - webhookServerOptions.CertDir = webhookCertPath - webhookServerOptions.CertName = webhookCertName - webhookServerOptions.KeyName = webhookCertKey - } - - webhookServer := webhook.NewServer(webhookServerOptions) - - // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. - // More info: - // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.25.0/pkg/metrics/server - // - https://book.kubebuilder.io/reference/metrics.html - metricsServerOptions := metricsserver.Options{ - BindAddress: metricsAddr, - SecureServing: secureMetrics, - TLSOpts: tlsOpts, - } - - if secureMetrics { - // FilterProvider is used to protect the metrics endpoint with authn/authz. - // These configurations ensure that only authorized users and service accounts - // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: - // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.25.0/pkg/metrics/filters#WithAuthenticationAndAuthorization - metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - } - - // If the certificate is not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. - // - // TODO(user): If you enable certManager, uncomment the following lines: - // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates - // managed by cert-manager for the metrics server. - // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. - if len(metricsCertPath) > 0 { - setupLog.Info("Initializing metrics certificate watcher using provided certificates", - "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) - - metricsServerOptions.CertDir = metricsCertPath - metricsServerOptions.CertName = metricsCertName - metricsServerOptions.KeyName = metricsCertKey - } - - managerOptions := ctrl.Options{ - Scheme: scheme, - Metrics: metricsServerOptions, - WebhookServer: webhookServer, - HealthProbeBindAddress: probeAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "a6325ed6.ddupan.top", - // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily - // when the Manager ends. This requires the binary to immediately end when the - // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly - // speeds up voluntary leader transitions as the new leader don't have to wait - // LeaseDuration time first. - // - // In the default scaffold provided, the program ends immediately after - // the manager stops, so would be fine to enable this option. However, - // if you are doing or is intended to do any operation such as perform cleanups - // after the manager stops then its usage might be unsafe. - // LeaderElectionReleaseOnCancel: true, - } - if databaseNamespace != "" { - managerOptions.Cache = databasecontroller.InstanceCacheOptions(databaseNamespace) - } - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), managerOptions) - if err != nil { - setupLog.Error(err, "Failed to start manager") - os.Exit(1) - } - - // +kubebuilder:scaffold:builder - if err := setupOpenBaoAuthentication(mgr, openBao); err != nil { - setupLog.Error(err, "Failed to set up OpenBao authentication") - os.Exit(1) - } - var instanceService *application.InstanceService - if databaseNamespace != "" { - instanceService, err = setupInstanceObservation(mgr, databaseNamespace, databaseRootCert) - if err != nil { - setupLog.Error(err, "Failed to set up Instance observation") - os.Exit(1) - } - } - if err := (&databasecontroller.BindingReconciler{}).SetupWithManager(context.Background(), mgr); err != nil { - setupLog.Error(err, "Failed to set up Database binding controller") - os.Exit(1) - } - - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "Failed to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "Failed to set up ready check") - os.Exit(1) - } - - setupLog.Info("Starting manager") - err = mgr.Start(ctrl.SetupSignalHandler()) - // worker 完全停止后才释放 pgxpool，避免与在途观察竞争。 - if instanceService != nil { - instanceService.Close() - } - if err != nil { - setupLog.Error(err, "Failed to run manager") + if err := bootstrap.Run(); err != nil { + ctrl.Log.WithName("setup").Error(err, "Controller manager exited") os.Exit(1) } } diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 7736cef..8857dc3 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -45,6 +45,11 @@ Proxmox 作为稀缺物理基础设施可以共享，通过 pool、tag、token ## 进程内依赖边界 +`cmd/main.go` 是唯一程序入口，只调用 `internal/bootstrap.Run` 并处理退出状态。 +命令行参数、manager 创建、infra 与领域装配集中在 `internal/bootstrap`， +不在 `cmd` 平铺组件装配文件，也不为每个领域生成独立二进制。 +Makefile 与 Dockerfile 均继续构建 `cmd/main.go`。 + 基础设施能力属于整个 controller-manager，不因首个消费者是 Database 就归入该领域。 `internal/infra/openbao` 管理官方 SDK client 的 TLS 配置、Kubernetes 认证及 token 生命周期， 不依赖 Database 或其他产品领域。Bao client 默认禁用自动重试，写入结果不确定时由用例处理； diff --git a/cmd/database.go b/internal/bootstrap/database.go similarity index 98% rename from cmd/database.go rename to internal/bootstrap/database.go index 75e87fc..9d8f6a1 100644 --- a/cmd/database.go +++ b/internal/bootstrap/database.go @@ -1,4 +1,4 @@ -package main +package bootstrap import ( "git.ddupan.top/panxiao81/ayatori/internal/database/adapter/kubernetes" diff --git a/cmd/openbao.go b/internal/bootstrap/openbao.go similarity index 99% rename from cmd/openbao.go rename to internal/bootstrap/openbao.go index 6184d24..550158b 100644 --- a/cmd/openbao.go +++ b/internal/bootstrap/openbao.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package bootstrap import ( "errors" diff --git a/internal/bootstrap/run.go b/internal/bootstrap/run.go new file mode 100644 index 0000000..bb732b7 --- /dev/null +++ b/internal/bootstrap/run.go @@ -0,0 +1,213 @@ +// Package bootstrap 负责 controller-manager 的参数解析、依赖装配与启动。 +package bootstrap + +import ( + "context" + "crypto/tls" + "flag" + "fmt" + "os" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + databasev1alpha1 "git.ddupan.top/panxiao81/ayatori/api/database/v1alpha1" + executionv1alpha1 "git.ddupan.top/panxiao81/ayatori/api/execution/v1alpha1" + "git.ddupan.top/panxiao81/ayatori/internal/database/application" + databasecontroller "git.ddupan.top/panxiao81/ayatori/internal/database/controller" + // +kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(executionv1alpha1.AddToScheme(scheme)) + utilruntime.Must(databasev1alpha1.AddToScheme(scheme)) + // +kubebuilder:scaffold:scheme +} + +// Run 启动唯一的 controller-manager 进程；命令行参数与信号处理只初始化一次。 +func Run() error { + var openBao openBaoOptions + openBao.bindFlags(flag.CommandLine) + var databaseNamespace, databaseRootCert string + flag.StringVar(&databaseNamespace, "database-secret-namespace", os.Getenv("POD_NAMESPACE"), + "固定管理 Secret namespace；为空时不启用 Instance 观测") + flag.StringVar(&databaseRootCert, "database-root-cert", "", "PostgreSQL 管理连接信任的公开 CA bundle 路径") + var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string + var webhookCertPath, webhookCertName, webhookCertKey string + var webhookPort int + var enableLeaderElection bool + var probeAddr string + var secureMetrics bool + var enableHTTP2 bool + var tlsOpts []func(*tls.Config) + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.") + flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.") + flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.") + flag.IntVar(&webhookPort, "webhook-port", 9443, "Port the webhook server listens on. "+ + "Defaults to 9443. Set -1 to disable the webhook server.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") + flag.BoolVar(&enableHTTP2, "enable-http2", false, + "If set, HTTP/2 will be enabled for the metrics and webhook servers") + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("Disabling HTTP/2") + c.NextProtos = []string{"http/1.1"} + } + + if !enableHTTP2 { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + // Initial webhook TLS options + webhookTLSOpts := tlsOpts + webhookServerOptions := webhook.Options{ + TLSOpts: webhookTLSOpts, + Port: webhookPort, + } + + if len(webhookCertPath) > 0 { + setupLog.Info("Initializing webhook certificate watcher using provided certificates", + "webhook-cert-path", webhookCertPath, "webhook-cert-name", webhookCertName, "webhook-cert-key", webhookCertKey) + + webhookServerOptions.CertDir = webhookCertPath + webhookServerOptions.CertName = webhookCertName + webhookServerOptions.KeyName = webhookCertKey + } + + webhookServer := webhook.NewServer(webhookServerOptions) + + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.25.0/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + TLSOpts: tlsOpts, + } + + if secureMetrics { + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.25.0/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) + + metricsServerOptions.CertDir = metricsCertPath + metricsServerOptions.CertName = metricsCertName + metricsServerOptions.KeyName = metricsCertKey + } + + managerOptions := ctrl.Options{ + Scheme: scheme, + Metrics: metricsServerOptions, + WebhookServer: webhookServer, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "a6325ed6.ddupan.top", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, + } + if databaseNamespace != "" { + managerOptions.Cache = databasecontroller.InstanceCacheOptions(databaseNamespace) + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), managerOptions) + if err != nil { + return fmt.Errorf("create controller manager: %w", err) + } + + // +kubebuilder:scaffold:builder + if err := setupOpenBaoAuthentication(mgr, openBao); err != nil { + return fmt.Errorf("set up OpenBao authentication: %w", err) + } + var instanceService *application.InstanceService + if databaseNamespace != "" { + instanceService, err = setupInstanceObservation(mgr, databaseNamespace, databaseRootCert) + if err != nil { + return fmt.Errorf("set up Instance observation: %w", err) + } + // Run 返回前 manager 的 worker 已停止；启动中途失败也释放已装配的连接。 + defer instanceService.Close() + } + if err := (&databasecontroller.BindingReconciler{}).SetupWithManager(context.Background(), mgr); err != nil { + return fmt.Errorf("set up Database binding controller: %w", err) + } + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return fmt.Errorf("set up health check: %w", err) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + return fmt.Errorf("set up readiness check: %w", err) + } + + setupLog.Info("Starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + return fmt.Errorf("run controller manager: %w", err) + } + return nil +}