From 534cd3731cdf44d445dcc9826eed6cf8d4ae1fc3 Mon Sep 17 00:00:00 2001 From: Manuel Dewald Date: Fri, 16 Jan 2026 16:46:59 +0100 Subject: [PATCH 1/2] feat(bootstrap): use local docker registry --- cli/cmd/bootstrap_gcp.go | 6 + internal/bootstrap/gcp.go | 157 ++++++++++++++++++------ internal/installer/files/config_yaml.go | 2 +- 3 files changed, 127 insertions(+), 38 deletions(-) diff --git a/cli/cmd/bootstrap_gcp.go b/cli/cmd/bootstrap_gcp.go index 9edf70c..acd49f2 100644 --- a/cli/cmd/bootstrap_gcp.go +++ b/cli/cmd/bootstrap_gcp.go @@ -22,6 +22,8 @@ type BootstrapGcpCmd struct { Opts *GlobalOptions Env env.Env CodesphereEnv *bootstrap.CodesphereEnvironment + + InputRegistryType string } func (c *BootstrapGcpCmd) RunE(_ *cobra.Command, args []string) error { @@ -70,6 +72,7 @@ func AddBootstrapGcpCmd(root *cobra.Command, opts *GlobalOptions) { flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.DNSProjectID, "dns-project-id", "", "GCP Project ID for Cloud DNS (optional)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.DNSZoneName, "dns-zone-name", "oms-testing", "Cloud DNS Zone Name (optional)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.InstallCodesphereVersion, "install-codesphere-version", "", "Codesphere version to install (default: none)") + flags.StringVar(&bootstrapGcpCmd.InputRegistryType, "registry-type", "local-container", "Container registry type to use (options: local-container, artifact-registry) (default: artifact-registry)") flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.WriteConfig, "write-config", true, "Write generated install config to file (default: true)") util.MarkFlagRequired(bootstrapGcpCmd.cmd, "project-name") @@ -82,11 +85,14 @@ func AddBootstrapGcpCmd(root *cobra.Command, opts *GlobalOptions) { } func (c *BootstrapGcpCmd) BootstrapGcp() error { + c.CodesphereEnv.RegistryType = bootstrap.RegistryType(c.InputRegistryType) + gcpClient := bootstrap.NewGCPClient(os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")) bootstrapper, err := bootstrap.NewGCPBootstrapper(c.Env, c.CodesphereEnv, gcpClient) if err != nil { return err } + env, err := bootstrapper.Bootstrap() envBytes, err2 := json.MarshalIndent(env, "", " ") envString := string(envBytes) diff --git a/internal/bootstrap/gcp.go b/internal/bootstrap/gcp.go index e1f4eb9..8d0f213 100644 --- a/internal/bootstrap/gcp.go +++ b/internal/bootstrap/gcp.go @@ -26,6 +26,13 @@ import ( "google.golang.org/grpc/status" ) +type RegistryType string + +const ( + RegistryTypeLocalContainer RegistryType = "local-container" + RegistryTypeArtifactRegistry RegistryType = "artifact-registry" +) + type GCPBootstrapper struct { ctx context.Context env *CodesphereEnvironment @@ -37,20 +44,21 @@ type GCPBootstrapper struct { } type CodesphereEnvironment struct { - ProjectID string `json:"project_id"` - ProjectName string `json:"project_name"` - DNSProjectID string `json:"dns_project_id"` - PostgreSQLNode node.Node `json:"postgresql_node"` - ControlPlaneNodes []node.Node `json:"control_plane_nodes"` - CephNodes []node.Node `json:"ceph_nodes"` - Jumpbox node.Node `json:"jumpbox"` - ContainerRegistryURL string `json:"container_registry_url"` - ExistingConfigUsed bool `json:"existing_config_used"` - InstallCodesphereVersion string `json:"install_codesphere_version"` - Preemptible bool `json:"preemptible"` - WriteConfig bool `json:"write_config"` - GatewayIP string `json:"gateway_ip"` - PublicGatewayIP string `json:"public_gateway_ip"` + ProjectID string `json:"project_id"` + ProjectName string `json:"project_name"` + DNSProjectID string `json:"dns_project_id"` + PostgreSQLNode node.Node `json:"postgresql_node"` + ControlPlaneNodes []node.Node `json:"control_plane_nodes"` + CephNodes []node.Node `json:"ceph_nodes"` + Jumpbox node.Node `json:"jumpbox"` + ContainerRegistryURL string `json:"container_registry_url"` + ExistingConfigUsed bool `json:"existing_config_used"` + InstallCodesphereVersion string `json:"install_codesphere_version"` + Preemptible bool `json:"preemptible"` + WriteConfig bool `json:"write_config"` + GatewayIP string `json:"gateway_ip"` + PublicGatewayIP string `json:"public_gateway_ip"` + RegistryType RegistryType `json:"registry_type"` ProjectDisplayName string BillingAccount string @@ -132,9 +140,11 @@ func (b *GCPBootstrapper) Bootstrap() (*CodesphereEnvironment, error) { return b.env, fmt.Errorf("failed to enable required APIs: %w", err) } - err = b.EnsureArtifactRegistry() - if err != nil { - return b.env, fmt.Errorf("failed to ensure artifact registry: %w", err) + if b.env.RegistryType == RegistryTypeArtifactRegistry { + err = b.EnsureArtifactRegistry() + if err != nil { + return b.env, fmt.Errorf("failed to ensure artifact registry: %w", err) + } } err = b.EnsureServiceAccounts() @@ -182,6 +192,13 @@ func (b *GCPBootstrapper) Bootstrap() (*CodesphereEnvironment, error) { return b.env, fmt.Errorf("failed to ensure hosts are configured: %w", err) } + if b.env.RegistryType == RegistryTypeLocalContainer { + err = b.EnsureLocalContainerRegistry() + if err != nil { + return b.env, fmt.Errorf("failed to ensure local container registry: %w", err) + } + } + if b.env.WriteConfig { err = b.UpdateInstallConfig() if err != nil { @@ -304,35 +321,101 @@ func (b *GCPBootstrapper) EnsureArtifactRegistry() error { return nil } +// Installs a docker registry on the postgres node to speed up image loading time +func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { + localRegistryServer := b.env.PostgreSQLNode.InternalIP + ":5000" + + // Figure out if registry is already running + checkCommand := `test "$(docker ps --filter 'name=registry' --format '{{.Names}}' | wc -l)" -eq "1"` + err := b.env.PostgreSQLNode.RunSSHCommand(&b.env.Jumpbox, b.NodeManager, "root", checkCommand) + if err == nil && b.InstallConfig.Registry != nil && b.InstallConfig.Registry.Server == localRegistryServer && + b.InstallConfig.Registry.Username != "" && b.InstallConfig.Registry.Password != "" { + log.Println("Local container registry already running on postgres node") + return nil + } + + log.Println("Installing registry") + + b.InstallConfig.Registry.Server = localRegistryServer + b.InstallConfig.Registry.Username = "custom-registry" + b.InstallConfig.Registry.Password = shortuuid.New() + commands := []string{ + "apt-get update", + "apt-get install -y docker-ce apache2-utils", + "systemctl start docker", + "systemctl enable docker", + "htpasswd -bBc /root/registry.password " + b.InstallConfig.Registry.Username + " " + b.InstallConfig.Registry.Password, + "openssl req -newkey rsa:4096 -nodes -sha256 -keyout /root/registry.key -x509 -days 365 -out /root/registry.crt -subj \"/C=DE/ST=BW/L=Karlsruhe/O=Codesphere/CN=" + b.env.PostgreSQLNode.InternalIP + "\" -addext \"subjectAltName = DNS:postgres,IP:" + b.env.PostgreSQLNode.InternalIP + "\"", + "docker rm -f registry || true", + `docker run -d -p 5000:5000 \ + --restart=always --name registry \ + --env REGISTRY_AUTH=htpasswd \ + --env REGISTRY_AUTH_HTPASSWD_REALM='Registry Realm' \ + --env REGISTRY_AUTH_HTPASSWD_PATH=/auth/registry.password \ + -v /root/registry.password:/auth/registry.password \ + -v /var/lib/registry:/var/lib/registry \ + --env REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry.crt \ + --env REGISTRY_HTTP_TLS_KEY=/certs/registry.key \ + -v /root/registry.crt:/certs/registry.crt \ + -v /root/registry.key:/certs/registry.key \ + registry:2`, + `mkdir -p /etc/docker/certs.d/` + b.InstallConfig.Registry.Server, + `cp /root/registry.crt /etc/docker/certs.d/` + b.InstallConfig.Registry.Server + `/ca.crt`, + } + for _, cmd := range commands { + err := b.env.PostgreSQLNode.RunSSHCommand(&b.env.Jumpbox, b.NodeManager, "root", cmd) + if err != nil { + return fmt.Errorf("failed to run command on postgres node: %w", err) + } + } + + allNodes := append(b.env.ControlPlaneNodes, b.env.CephNodes...) + for _, node := range allNodes { + err := b.env.PostgreSQLNode.RunSSHCommand(&b.env.Jumpbox, b.NodeManager, "root", "scp -o StrictHostKeyChecking=no /root/registry.crt root@"+node.InternalIP+":/usr/local/share/registry-ca.crt") + if err != nil { + return fmt.Errorf("failed to copy registry certificate to node %s: %w", node.InternalIP, err) + } + err = node.RunSSHCommand(&b.env.Jumpbox, b.NodeManager, "root", "update-ca-certificates") + if err != nil { + return fmt.Errorf("failed to update CA certificates on node %s: %w", node.InternalIP, err) + } + } + + return nil +} + func (b *GCPBootstrapper) EnsureServiceAccounts() error { _, _, err := b.EnsureServiceAccount("cloud-controller") if err != nil { return err } - sa, newSa, err := b.EnsureServiceAccount("artifact-registry-writer") - if err != nil { - return err - } - if !newSa && b.InstallConfig.Registry.Password != "" { - return nil - } + if b.env.RegistryType == RegistryTypeArtifactRegistry { + sa, newSa, err := b.EnsureServiceAccount("artifact-registry-writer") + if err != nil { + return err + } - for retries := range 5 { - privateKey, err := b.GCPClient.CreateServiceAccountKey(b.ctx, b.env.ProjectID, sa) + if !newSa && b.InstallConfig.Registry.Password != "" { + return nil + } + + for retries := range 5 { + privateKey, err := b.GCPClient.CreateServiceAccountKey(b.ctx, b.env.ProjectID, sa) - if err != nil && status.Code(err) != codes.AlreadyExists { - if retries > 3 { - return fmt.Errorf("failed to create service account key: %w", err) + if err != nil && status.Code(err) != codes.AlreadyExists { + if retries > 3 { + return fmt.Errorf("failed to create service account key: %w", err) + } + log.Printf("got response %d trying to create service account key for %s, retrying...", status.Code(err), sa) + time.Sleep(5 * time.Second) + continue } - log.Printf("got response %d trying to create service account key for %s, retrying...", status.Code(err), sa) - time.Sleep(5 * time.Second) - continue + fmt.Printf("Service account key for %s ensured\n", sa) + b.InstallConfig.Registry.Password = string(privateKey) + b.InstallConfig.Registry.Username = "_json_key_base64" + break } - fmt.Printf("Service account key for %s ensured\n", sa) - b.InstallConfig.Registry.Password = string(privateKey) - b.InstallConfig.Registry.Username = "_json_key_base64" - break } return nil @@ -488,7 +571,7 @@ func (b *GCPBootstrapper) EnsureComputeInstances() error { vmDefs := []VMDef{ {"jumpbox", "e2-medium", []string{"jumpbox", "ssh"}, []int64{}, true}, - {"postgres", "e2-medium", []string{"postgres"}, []int64{50}, true}, + {"postgres", "e2-standard-8", []string{"postgres"}, []int64{}, true}, {"ceph-1", "e2-standard-8", []string{"ceph"}, []int64{20, 200}, false}, {"ceph-2", "e2-standard-8", []string{"ceph"}, []int64{20, 200}, false}, {"ceph-3", "e2-standard-8", []string{"ceph"}, []int64{20, 200}, false}, diff --git a/internal/installer/files/config_yaml.go b/internal/installer/files/config_yaml.go index 7b6f1a7..0966e95 100644 --- a/internal/installer/files/config_yaml.go +++ b/internal/installer/files/config_yaml.go @@ -42,7 +42,7 @@ type SecretFields struct { type RootConfig struct { Datacenter DatacenterConfig `yaml:"dataCenter"` Secrets SecretsConfig `yaml:"secrets"` - Registry RegistryConfig `yaml:"registry,omitempty"` + Registry *RegistryConfig `yaml:"registry,omitempty"` Postgres PostgresConfig `yaml:"postgres"` Ceph CephConfig `yaml:"ceph"` Kubernetes KubernetesConfig `yaml:"kubernetes"` From 3a3bf989beae572d227d6d3b9010e43d82e23a5b Mon Sep 17 00:00:00 2001 From: NautiluX <2600004+NautiluX@users.noreply.github.com> Date: Fri, 16 Jan 2026 16:02:33 +0000 Subject: [PATCH 2/2] chore(docs): Auto-update docs and licenses Signed-off-by: NautiluX <2600004+NautiluX@users.noreply.github.com> --- docs/oms-cli_beta_bootstrap-gcp.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/oms-cli_beta_bootstrap-gcp.md b/docs/oms-cli_beta_bootstrap-gcp.md index 108cbb3..1984b14 100644 --- a/docs/oms-cli_beta_bootstrap-gcp.md +++ b/docs/oms-cli_beta_bootstrap-gcp.md @@ -32,6 +32,7 @@ oms-cli beta bootstrap-gcp [flags] --preemptible Use preemptible VMs for Codesphere infrastructure (default: false) --project-name string Unique GCP Project Name (required) --region string GCP Region (default: europe-west4) (default "europe-west4") + --registry-type string Container registry type to use (options: local-container, artifact-registry) (default: artifact-registry) (default "local-container") --secrets-dir string Directory for secrets (default: /etc/codesphere/secrets) (default "/etc/codesphere/secrets") --secrets-file string Path to secrets files (optional) (default "prod.vault.yaml") --ssh-private-key-path string SSH Private Key Path (default: ~/.ssh/id_rsa) (default "~/.ssh/id_rsa")