feat(bootstrap): use local docker registry

cli/cmd/bootstrap_gcp.go

@@ -22,6 +22,8 @@ type BootstrapGcpCmd struct {
 	Opts          *GlobalOptions
 	Env           env.Env
 	CodesphereEnv *bootstrap.CodesphereEnvironment
+
+	InputRegistryType string
 }
 
 func (c *BootstrapGcpCmd) RunE(_ *cobra.Command, args []string) error {
@@ -70,6 +72,7 @@ func AddBootstrapGcpCmd(root *cobra.Command, opts *GlobalOptions) {
 	flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.DNSProjectID, "dns-project-id", "", "GCP Project ID for Cloud DNS (optional)")
 	flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.DNSZoneName, "dns-zone-name", "oms-testing", "Cloud DNS Zone Name (optional)")
 	flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.InstallCodesphereVersion, "install-codesphere-version", "", "Codesphere version to install (default: none)")
+	flags.StringVar(&bootstrapGcpCmd.InputRegistryType, "registry-type", "local-container", "Container registry type to use (options: local-container, artifact-registry) (default: artifact-registry)")
 	flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.WriteConfig, "write-config", true, "Write generated install config to file (default: true)")
 
 	util.MarkFlagRequired(bootstrapGcpCmd.cmd, "project-name")
@@ -82,11 +85,14 @@ func AddBootstrapGcpCmd(root *cobra.Command, opts *GlobalOptions) {
 }
 
 func (c *BootstrapGcpCmd) BootstrapGcp() error {
+	c.CodesphereEnv.RegistryType = bootstrap.RegistryType(c.InputRegistryType)
+
 	gcpClient := bootstrap.NewGCPClient(os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"))
 	bootstrapper, err := bootstrap.NewGCPBootstrapper(c.Env, c.CodesphereEnv, gcpClient)
 	if err != nil {
 		return err
 	}
+
 	env, err := bootstrapper.Bootstrap()
 	envBytes, err2 := json.MarshalIndent(env, "", "  ")
 	envString := string(envBytes)

docs/oms-cli_beta_bootstrap-gcp.md

@@ -32,6 +32,7 @@ oms-cli beta bootstrap-gcp [flags]
       --preemptible                         Use preemptible VMs for Codesphere infrastructure (default: false)
       --project-name string                 Unique GCP Project Name (required)
       --region string                       GCP Region (default: europe-west4) (default "europe-west4")
+      --registry-type string                Container registry type to use (options: local-container, artifact-registry) (default: artifact-registry) (default "local-container")
       --secrets-dir string                  Directory for secrets (default: /etc/codesphere/secrets) (default "/etc/codesphere/secrets")
       --secrets-file string                 Path to secrets files (optional) (default "prod.vault.yaml")
       --ssh-private-key-path string         SSH Private Key Path (default: ~/.ssh/id_rsa) (default "~/.ssh/id_rsa")

internal/bootstrap/gcp.go

@@ -26,6 +26,13 @@ import (
 	"google.golang.org/grpc/status"
 )
 
+type RegistryType string
+
+const (
+	RegistryTypeLocalContainer   RegistryType = "local-container"
+	RegistryTypeArtifactRegistry RegistryType = "artifact-registry"
+)
+
 type GCPBootstrapper struct {
 	ctx           context.Context
 	env           *CodesphereEnvironment
@@ -37,20 +44,21 @@ type GCPBootstrapper struct {
 }
 
 type CodesphereEnvironment struct {
-	ProjectID                string      `json:"project_id"`
-	ProjectName              string      `json:"project_name"`
-	DNSProjectID             string      `json:"dns_project_id"`
-	PostgreSQLNode           node.Node   `json:"postgresql_node"`
-	ControlPlaneNodes        []node.Node `json:"control_plane_nodes"`
-	CephNodes                []node.Node `json:"ceph_nodes"`
-	Jumpbox                  node.Node   `json:"jumpbox"`
-	ContainerRegistryURL     string      `json:"container_registry_url"`
-	ExistingConfigUsed       bool        `json:"existing_config_used"`
-	InstallCodesphereVersion string      `json:"install_codesphere_version"`
-	Preemptible              bool        `json:"preemptible"`
-	WriteConfig              bool        `json:"write_config"`
-	GatewayIP                string      `json:"gateway_ip"`
-	PublicGatewayIP          string      `json:"public_gateway_ip"`
+	ProjectID                string       `json:"project_id"`
+	ProjectName              string       `json:"project_name"`
+	DNSProjectID             string       `json:"dns_project_id"`
+	PostgreSQLNode           node.Node    `json:"postgresql_node"`
+	ControlPlaneNodes        []node.Node  `json:"control_plane_nodes"`
+	CephNodes                []node.Node  `json:"ceph_nodes"`
+	Jumpbox                  node.Node    `json:"jumpbox"`
+	ContainerRegistryURL     string       `json:"container_registry_url"`
+	ExistingConfigUsed       bool         `json:"existing_config_used"`
+	InstallCodesphereVersion string       `json:"install_codesphere_version"`
+	Preemptible              bool         `json:"preemptible"`
+	WriteConfig              bool         `json:"write_config"`
+	GatewayIP                string       `json:"gateway_ip"`
+	PublicGatewayIP          string       `json:"public_gateway_ip"`
+	RegistryType             RegistryType `json:"registry_type"`
 
 	ProjectDisplayName    string
 	BillingAccount        string
@@ -132,9 +140,11 @@ func (b *GCPBootstrapper) Bootstrap() (*CodesphereEnvironment, error) {
 		return b.env, fmt.Errorf("failed to enable required APIs: %w", err)
 	}
 
-	err = b.EnsureArtifactRegistry()
-	if err != nil {
-		return b.env, fmt.Errorf("failed to ensure artifact registry: %w", err)
+	if b.env.RegistryType == RegistryTypeArtifactRegistry {
+		err = b.EnsureArtifactRegistry()
+		if err != nil {
+			return b.env, fmt.Errorf("failed to ensure artifact registry: %w", err)
+		}
 	}
 
 	err = b.EnsureServiceAccounts()
@@ -182,6 +192,13 @@ func (b *GCPBootstrapper) Bootstrap() (*CodesphereEnvironment, error) {
 		return b.env, fmt.Errorf("failed to ensure hosts are configured: %w", err)
 	}
 
+	if b.env.RegistryType == RegistryTypeLocalContainer {
+		err = b.EnsureLocalContainerRegistry()
+		if err != nil {
+			return b.env, fmt.Errorf("failed to ensure local container registry: %w", err)
+		}
+	}
+
 	if b.env.WriteConfig {
 		err = b.UpdateInstallConfig()
 		if err != nil {
@@ -304,35 +321,101 @@ func (b *GCPBootstrapper) EnsureArtifactRegistry() error {
 	return nil
 }
 
+// Installs a docker registry on the postgres node to speed up image loading time
+func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error {
+	localRegistryServer := b.env.PostgreSQLNode.InternalIP + ":5000"
+
+	// Figure out if registry is already running
+	checkCommand := `test "$(docker ps --filter 'name=registry' --format '{{.Names}}' | wc -l)" -eq "1"`
+	err := b.env.PostgreSQLNode.RunSSHCommand(&b.env.Jumpbox, b.NodeManager, "root", checkCommand)
+	if err == nil && b.InstallConfig.Registry != nil && b.InstallConfig.Registry.Server == localRegistryServer &&
+		b.InstallConfig.Registry.Username != "" && b.InstallConfig.Registry.Password != "" {
+		log.Println("Local container registry already running on postgres node")
+		return nil
+	}
+
+	log.Println("Installing registry")
+
+	b.InstallConfig.Registry.Server = localRegistryServer
+	b.InstallConfig.Registry.Username = "custom-registry"
+	b.InstallConfig.Registry.Password = shortuuid.New()
+	commands := []string{
+		"apt-get update",
+		"apt-get install -y docker-ce apache2-utils",
+		"systemctl start docker",
+		"systemctl enable docker",
+		"htpasswd -bBc /root/registry.password " + b.InstallConfig.Registry.Username + " " + b.InstallConfig.Registry.Password,
+		"openssl req -newkey rsa:4096 -nodes -sha256 -keyout /root/registry.key -x509 -days 365 -out /root/registry.crt -subj \"/C=DE/ST=BW/L=Karlsruhe/O=Codesphere/CN=" + b.env.PostgreSQLNode.InternalIP + "\" -addext \"subjectAltName = DNS:postgres,IP:" + b.env.PostgreSQLNode.InternalIP + "\"",
+		"docker rm -f registry || true",
+		`docker run -d -p 5000:5000 \
+		--restart=always --name registry \
+		--env REGISTRY_AUTH=htpasswd \
+		--env REGISTRY_AUTH_HTPASSWD_REALM='Registry Realm' \
+		--env REGISTRY_AUTH_HTPASSWD_PATH=/auth/registry.password \
+		-v /root/registry.password:/auth/registry.password \
+		-v /var/lib/registry:/var/lib/registry \
+		--env REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry.crt \
+		--env REGISTRY_HTTP_TLS_KEY=/certs/registry.key \
+		-v /root/registry.crt:/certs/registry.crt \
+		-v /root/registry.key:/certs/registry.key \
+		registry:2`,
+		`mkdir -p /etc/docker/certs.d/` + b.InstallConfig.Registry.Server,
+		`cp /root/registry.crt /etc/docker/certs.d/` + b.InstallConfig.Registry.Server + `/ca.crt`,
+	}
+	for _, cmd := range commands {
+		err := b.env.PostgreSQLNode.RunSSHCommand(&b.env.Jumpbox, b.NodeManager, "root", cmd)
+		if err != nil {
+			return fmt.Errorf("failed to run command on postgres node: %w", err)
+		}
+	}
+
+	allNodes := append(b.env.ControlPlaneNodes, b.env.CephNodes...)
+	for _, node := range allNodes {
+		err := b.env.PostgreSQLNode.RunSSHCommand(&b.env.Jumpbox, b.NodeManager, "root", "scp -o StrictHostKeyChecking=no /root/registry.crt root@"+node.InternalIP+":/usr/local/share/registry-ca.crt")
+		if err != nil {
+			return fmt.Errorf("failed to copy registry certificate to node %s: %w", node.InternalIP, err)
+		}
+		err = node.RunSSHCommand(&b.env.Jumpbox, b.NodeManager, "root", "update-ca-certificates")
+		if err != nil {
+			return fmt.Errorf("failed to update CA certificates on node %s: %w", node.InternalIP, err)
+		}
+	}
+
+	return nil
+}
+
 func (b *GCPBootstrapper) EnsureServiceAccounts() error {
 	_, _, err := b.EnsureServiceAccount("cloud-controller")
 	if err != nil {
 		return err
 	}
-	sa, newSa, err := b.EnsureServiceAccount("artifact-registry-writer")
-	if err != nil {
-		return err
-	}
 
-	if !newSa && b.InstallConfig.Registry.Password != "" {
-		return nil
-	}
+	if b.env.RegistryType == RegistryTypeArtifactRegistry {
+		sa, newSa, err := b.EnsureServiceAccount("artifact-registry-writer")
+		if err != nil {
+			return err
+		}
 
-	for retries := range 5 {
-		privateKey, err := b.GCPClient.CreateServiceAccountKey(b.ctx, b.env.ProjectID, sa)
+		if !newSa && b.InstallConfig.Registry.Password != "" {
+			return nil
+		}
+
+		for retries := range 5 {
+			privateKey, err := b.GCPClient.CreateServiceAccountKey(b.ctx, b.env.ProjectID, sa)
 
-		if err != nil && status.Code(err) != codes.AlreadyExists {
-			if retries > 3 {
-				return fmt.Errorf("failed to create service account key: %w", err)
+			if err != nil && status.Code(err) != codes.AlreadyExists {
+				if retries > 3 {
+					return fmt.Errorf("failed to create service account key: %w", err)
+				}
+				log.Printf("got response %d trying to create service account key for %s, retrying...", status.Code(err), sa)
+				time.Sleep(5 * time.Second)
+				continue
 			}
-			log.Printf("got response %d trying to create service account key for %s, retrying...", status.Code(err), sa)
-			time.Sleep(5 * time.Second)
-			continue
+			fmt.Printf("Service account key for %s ensured\n", sa)
+			b.InstallConfig.Registry.Password = string(privateKey)
+			b.InstallConfig.Registry.Username = "_json_key_base64"
+			break
 		}
-		fmt.Printf("Service account key for %s ensured\n", sa)
-		b.InstallConfig.Registry.Password = string(privateKey)
-		b.InstallConfig.Registry.Username = "_json_key_base64"
-		break
 	}
 
 	return nil
@@ -488,7 +571,7 @@ func (b *GCPBootstrapper) EnsureComputeInstances() error {
 
 	vmDefs := []VMDef{
 		{"jumpbox", "e2-medium", []string{"jumpbox", "ssh"}, []int64{}, true},
-		{"postgres", "e2-medium", []string{"postgres"}, []int64{50}, true},
+		{"postgres", "e2-standard-8", []string{"postgres"}, []int64{}, true},
 		{"ceph-1", "e2-standard-8", []string{"ceph"}, []int64{20, 200}, false},
 		{"ceph-2", "e2-standard-8", []string{"ceph"}, []int64{20, 200}, false},
 		{"ceph-3", "e2-standard-8", []string{"ceph"}, []int64{20, 200}, false},

internal/installer/files/config_yaml.go

@@ -42,7 +42,7 @@ type SecretFields struct {
 type RootConfig struct {
 	Datacenter             DatacenterConfig              `yaml:"dataCenter"`
 	Secrets                SecretsConfig                 `yaml:"secrets"`
-	Registry               RegistryConfig                `yaml:"registry,omitempty"`
+	Registry               *RegistryConfig               `yaml:"registry,omitempty"`
 	Postgres               PostgresConfig                `yaml:"postgres"`
 	Ceph                   CephConfig                    `yaml:"ceph"`
 	Kubernetes             KubernetesConfig              `yaml:"kubernetes"`