Consolidate postgres and tfstate postgres.

Author: badri

CHANGELOG.md

@@ -17,15 +17,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Install ShapeBlock operator
 - Registry credentials secret
 - App and project CRDs
+- Proper cleanup during uninstall
 
 ### Changed
 
 - Fix cron job creation
+- Unified postgres and tfstate postgres into single instance
 
 ### Removed
 
 - Removed resource constraints on backend and frontend.
 - Epinio is no longer installed.
+- Celery worker in backend
 
 ## [1.0.4] - 2024-12-19
 

assets/helm-repos/bitnami-repository.yaml

@@ -1,8 +1,10 @@
 apiVersion: source.toolkit.fluxcd.io/v1beta2
-kind: HelmRepository
+kind: OCIRepository
 metadata:
   name: bitnami
   namespace: shapeblock
 spec:
   interval: 1m
-  url: https://charts.bitnami.com/bitnami
+  url: oci://registry-1.docker.io/bitnamicharts/mysql
+  ref:
+    semver: ">12.0.0"

main.go

@@ -119,7 +119,6 @@ type BackendConfig struct {
 	TFStateUsername     string
 	TFStatePassword     string
 	TFStateDatabase     string
-	TFStateRootPW       string
 	LicenseKey          string
 }
 
@@ -687,41 +686,39 @@ func generatePassword() string {
 	return string(b)
 }
 
-func installPostgres(name, username, database string, backendConfig *BackendConfig) error {
-	printStatus(fmt.Sprintf("Installing PostgreSQL instance %s...", name))
+func installPostgres(backendConfig *BackendConfig) error {
+	printStatus("Installing PostgreSQL...")
 
-	if resourceExists("statefulset", name+"-postgresql", "shapeblock") {
-		printStatus(fmt.Sprintf("PostgreSQL instance %s already installed", name))
+	if resourceExists("statefulset", "db-postgresql", "shapeblock") {
+		printStatus("PostgreSQL already installed")
 		return nil
 	}
 
 	if err := addHelmRepo("bitnami", "https://charts.bitnami.com/bitnami"); err != nil {
 		return err
 	}
 
-	// Generate passwords
-	password := generatePassword()
+	// Generate passwords for both users and root
+	mainPassword := generatePassword()
+	tfstatePassword := generatePassword()
 	rootPW := generatePassword()
 
-	// Store credentials based on instance name
-	switch name {
-	case "db":
-		backendConfig.PostgresUsername = username
-		backendConfig.PostgresDatabase = database
-		backendConfig.PostgresPassword = password
-		backendConfig.PostgresRootPW = rootPW
-	case "tfstate":
-		backendConfig.TFStateUsername = username
-		backendConfig.TFStateDatabase = database
-		backendConfig.TFStatePassword = password
-		backendConfig.TFStateRootPW = rootPW
-	}
+	// Store credentials in backend config
+	backendConfig.PostgresUsername = "shapeblock"
+	backendConfig.PostgresDatabase = "shapeblock"
+	backendConfig.PostgresPassword = mainPassword
+	backendConfig.PostgresRootPW = rootPW
+	backendConfig.TFStateUsername = "tfstate"
+	backendConfig.TFStateDatabase = "tfstate"
+	backendConfig.TFStatePassword = tfstatePassword
 
 	// Log the credentials for debugging
-	logMessage("INFO", fmt.Sprintf("PostgreSQL %s credentials - username: %s, database: %s, password: %s",
-		name, username, database, password))
+	logMessage("INFO", fmt.Sprintf("PostgreSQL main credentials - username: %s, database: %s, password: %s",
+		backendConfig.PostgresUsername, backendConfig.PostgresDatabase, mainPassword))
+	logMessage("INFO", fmt.Sprintf("PostgreSQL tfstate credentials - username: %s, database: %s, password: %s",
+		backendConfig.TFStateUsername, backendConfig.TFStateDatabase, tfstatePassword))
 
-	// Create values.yaml
+	// Create values with initialization scripts for both databases
 	values := fmt.Sprintf(`
 auth:
   database: %s
@@ -732,13 +729,42 @@ architecture: standalone
 primary:
   persistence:
     size: 2Gi
+  initdb:
+    scripts:
+      init_db_and_users.sql: |
+        -- Create tfstate user and database
+        CREATE USER %s WITH PASSWORD '%s';
+        CREATE DATABASE %s OWNER %s;
+        GRANT ALL PRIVILEGES ON DATABASE %s TO %s;
+
+        -- Create shapeblock user and database
+        CREATE USER %s WITH PASSWORD '%s';
+        CREATE DATABASE %s OWNER %s;
+        GRANT ALL PRIVILEGES ON DATABASE %s TO %s;
 tls:
   enabled: true
-  autoGenerated: true
-`, database, username, password, rootPW)
+  autoGenerated: true`,
+		backendConfig.PostgresDatabase, // Initial database
+		backendConfig.PostgresUsername, // Initial user
+		backendConfig.PostgresPassword,
+		backendConfig.PostgresRootPW,
+		// tfstate database and user creation
+		backendConfig.TFStateUsername,
+		backendConfig.TFStatePassword,
+		backendConfig.TFStateDatabase,
+		backendConfig.TFStateUsername,
+		backendConfig.TFStateDatabase,
+		backendConfig.TFStateUsername,
+		// shapeblock database and user creation
+		backendConfig.PostgresUsername,
+		backendConfig.PostgresPassword,
+		backendConfig.PostgresDatabase,
+		backendConfig.PostgresUsername,
+		backendConfig.PostgresDatabase,
+		backendConfig.PostgresUsername)
 
 	// Write values to temporary file
-	tmpfile, err := os.CreateTemp("", fmt.Sprintf("%s-values-*.yaml", name))
+	tmpfile, err := os.CreateTemp("", "postgres-values-*.yaml")
 	if err != nil {
 		printError(fmt.Sprintf("Failed to create temp file: %v", err))
 		return err
@@ -753,16 +779,16 @@ tls:
 
 	// Install PostgreSQL
 	if err := runCommand("helm", "upgrade", "--install",
-		name, "bitnami/postgresql",
+		"db", "bitnami/postgresql",
 		"--version", "16.2.3",
 		"--namespace", "shapeblock",
 		"--values", tmpfile.Name(),
 		"--timeout", "600s"); err != nil {
-		printError(fmt.Sprintf("Failed to install PostgreSQL %s: %v", name, err))
+		printError(fmt.Sprintf("Failed to install PostgreSQL: %v", err))
 		return err
 	}
 
-	printStatus(fmt.Sprintf("PostgreSQL instance %s installed successfully", name))
+	printStatus("PostgreSQL installed successfully with both databases")
 	return nil
 }
 
@@ -775,7 +801,7 @@ func createTerraformSecret(backendConfig *BackendConfig) error {
 	}
 
 	// Create connection string
-	connStr := fmt.Sprintf("postgres://%s:%s@tfstate-postgresql/%s",
+	connStr := fmt.Sprintf("postgres://%s:%s@db-postgresql/%s",
 		backendConfig.TFStateUsername,
 		backendConfig.TFStatePassword,
 		backendConfig.TFStateDatabase)
@@ -932,14 +958,9 @@ func install(config *Config, backendConfig *BackendConfig) error {
 		return fmt.Errorf("failed to create service account: %v", err)
 	}
 
-	// Install main PostgreSQL instance
-	if err := installPostgres("db", "shapeblock", "shapeblock", backendConfig); err != nil {
-		return fmt.Errorf("failed to install main PostgreSQL: %v", err)
-	}
-
-	// Install TFState PostgreSQL instance
-	if err := installPostgres("tfstate", "tfstate", "tfstate", backendConfig); err != nil {
-		return fmt.Errorf("failed to install TFState PostgreSQL: %v", err)
+	// Install PostgreSQL instance
+	if err := installPostgres(backendConfig); err != nil {
+		return fmt.Errorf("failed to install PostgreSQL: %v", err)
 	}
 
 	// Create Terraform credentials secret
@@ -1768,19 +1789,6 @@ deployments:
       release: backend
     replicas: 1
 
-  worker:
-    containers:
-    - envConfigmaps:
-      - envs
-      envSecrets:
-      - secret-envs
-      name: worker
-      command: ['celery', '-A', 'shapeblock', 'worker', '-l', 'INFO']
-    podLabels:
-      app: shapeblock
-      release: backend
-    replicas: 1
-
 envs:
   DEBUG: "False"
   DATABASE_URL: "postgres://%s:%s@db-postgresql/%s"
@@ -2279,7 +2287,7 @@ Admin Backend Access:
 - URL: https://api.%s/admin-%s/
 - Username: admin
 - Password: <password provided during installation>
-`, domain, config.AdminEmail, domain, config.AppName)
+`, domain, config.AdminEmail, domain, slugify(config.AppName))
 
 	// Print to console
 	fmt.Println(instructions)
@@ -2411,7 +2419,7 @@ func uninstall() error {
 
 	// Uninstall PostgreSQL instances
 	printStatus("Uninstalling PostgreSQL instances...")
-	for _, instance := range []string{"db", "tfstate"} {
+	for _, instance := range []string{"db"} {
 		if err := runCommand("helm", "uninstall", instance, "-n", "shapeblock"); err != nil {
 			printStatus(fmt.Sprintf("Note: PostgreSQL instance %s was not installed or already removed", instance))
 		}
@@ -2522,6 +2530,55 @@ func uninstall() error {
 		}
 	}
 
+	// Clean up files
+	printStatus("Cleaning up installation files...")
+
+	// Get home directory
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		printStatus(fmt.Sprintf("Warning: Failed to get home directory: %v", err))
+	} else {
+		// Remove SSH keys
+		sshFiles := []string{
+			filepath.Join(homeDir, "sb"),
+			filepath.Join(homeDir, "sb.pub"),
+		}
+
+		for _, file := range sshFiles {
+			if err := os.Remove(file); err != nil {
+				if !os.IsNotExist(err) {
+					printStatus(fmt.Sprintf("Warning: Failed to remove %s: %v", file, err))
+				}
+			} else {
+				printStatus(fmt.Sprintf("Removed %s", file))
+			}
+		}
+	}
+
+	// Remove installation files from current directory
+	installFiles := []string{
+		"instructions.txt",
+		"kubeconfig",
+	}
+
+	// Add any install*.log files
+	matches, err := filepath.Glob("install*.log")
+	if err != nil {
+		printStatus(fmt.Sprintf("Warning: Failed to find log files: %v", err))
+	} else {
+		installFiles = append(installFiles, matches...)
+	}
+
+	for _, file := range installFiles {
+		if err := os.Remove(file); err != nil {
+			if !os.IsNotExist(err) {
+				printStatus(fmt.Sprintf("Warning: Failed to remove %s: %v", file, err))
+			}
+		} else {
+			printStatus(fmt.Sprintf("Removed %s", file))
+		}
+	}
+
 	printStatus("ShapeBlock has been successfully uninstalled")
 	return nil
 }
@@ -3515,7 +3572,7 @@ spec:
       serviceAccountName: shapeblock-admin
       containers:
       - name: sb-operator
-        image: shapeblock/sb-operator:20-dec-2024.1
+        image: ghcr.io/shapeblock/operator:v2
         imagePullPolicy: Always
         livenessProbe:
           failureThreshold: 3