Feature/dbt macros

databases/data-platform/multitenant-analytics-platform/.gitignore

@@ -2,3 +2,5 @@ aws-samples-zero-etl
 ./data/
 node_modules/
 bastion-redshift-connection.json
+init-db
+.env

databases/data-platform/multitenant-analytics-platform/1-etl-manager.sh

@@ -15,6 +15,7 @@ ENVIRONMENT="dev"
 CLEANUP=false
 DRY_RUN=false
 SKIP_CLONE=false
+LOCAL_EXECUTION=false
 
 # Function to print colored output
 print_info() {
@@ -48,12 +49,16 @@ OPTIONS:
     --cleanup                  Clean up resources (destroy stacks)
     --dry-run                  Show what would be deployed without actually deploying
     --skip-clone               Skip cloning AWS samples repository (use existing)
+    --local                    Set up local Docker environment instead of AWS infrastructure
     -h, --help                 Show this help message
 
 EXAMPLES:
     # Deploy Aurora PostgreSQL infrastructure
     $0 -p aurora-postgresql -c config.json
 
+    # Set up local Docker environment
+    $0 -p aurora-postgresql -c config.json --local
+
     # Dry run to see what would be deployed
     $0 -p aurora-postgresql -c config.json --dry-run
 
@@ -108,6 +113,10 @@ while [[ $# -gt 0 ]]; do
             SKIP_CLONE=true
             shift
             ;;
+        --local)
+            LOCAL_EXECUTION=true
+            shift
+            ;;
         -h|--help)
             show_usage
             exit 0
@@ -443,6 +452,90 @@ cleanup_resources() {
     fi
 }
 
+# Function to setup local Docker environment
+setup_local_environment() {
+    print_info "Setting up local Docker environment..."
+
+    # Check if Docker is installed
+    if ! command -v docker &> /dev/null; then
+        print_error "Docker is not installed. Please install Docker first."
+        exit 1
+    fi
+
+    # Check if Docker Compose is available
+    if ! docker compose version &> /dev/null; then
+        print_error "Docker Compose is not available. Please install Docker Compose."
+        exit 1
+    fi
+
+    # Check if docker-compose.yml exists
+    if [[ ! -f "$PROJECT_ROOT/docker-compose.yml" ]]; then
+        print_error "docker-compose.yml not found in project root: $PROJECT_ROOT"
+        exit 1
+    fi
+
+    print_info "Starting Docker Compose services..."
+    cd "$PROJECT_ROOT"
+
+    if [[ "$DRY_RUN" == true ]]; then
+        print_info "DRY RUN: Would execute 'docker compose up -d'"
+        return
+    fi
+
+    # Start Docker Compose services
+    docker compose up -d
+
+    # Wait a moment for services to start
+    print_info "Waiting for services to start..."
+    sleep 5
+
+    # Check service status
+    print_info "Checking service status..."
+    docker compose ps
+
+    print_success "Local Docker environment ready!"
+}
+
+# Function to show local environment info
+show_local_info() {
+    print_info "=== Local Environment Information ==="
+
+    cd "$PROJECT_ROOT"
+
+    print_info "Docker Compose Services:"
+    docker compose ps
+
+    print_info "=== Next Steps ==="
+    echo "Phase 1 (Local) completed successfully!"
+    echo ""
+    echo "Next step: Run Phase 2 for local database setup"
+    echo "  ./2-etl-manager.sh -p $PATTERN -c $CONFIG_FILE --local"
+    echo ""
+    echo "Or test local PostgreSQL connection:"
+    echo "  docker exec -it multitenant-analytics-platform-postgres-1 psql -U dbt_user -d postgres"
+}
+
+# Function to cleanup local environment
+cleanup_local_environment() {
+    print_warning "Cleaning up local Docker environment..."
+
+    cd "$PROJECT_ROOT"
+
+    if [[ "$DRY_RUN" == true ]]; then
+        print_info "DRY RUN: Would execute 'docker compose down -v'"
+        return
+    fi
+
+    print_warning "This will stop and remove all local containers and volumes. Are you sure? (y/N)"
+    read -r response
+    if [[ "$response" =~ ^[Yy]$ ]]; then
+        docker compose down -v
+        print_success "Local environment cleaned up!"
+    else
+        print_info "Cleanup cancelled."
+    fi
+}
+
 # Function to show deployment info
 show_deployment_info() {
     print_info "=== Phase 1 Deployment Information ==="
@@ -501,27 +594,45 @@ show_deployment_info() {
 
 # Main execution
 main() {
-    print_info "Starting Phase 1: Infrastructure Deployment..."
-
-    check_prerequisites
-
-    if [[ "$CLEANUP" == true ]]; then
-        cleanup_resources
-        exit 0
-    fi
-
-    clone_samples
-    generate_cdk_context
-    setup_python_env
-    bootstrap_cdk
-    deploy_infrastructure
-
-    if [[ "$DRY_RUN" != true ]]; then
-        show_deployment_info
+    if [[ "$LOCAL_EXECUTION" == true ]]; then
+        print_info "Starting Phase 1: Local Environment Setup..."
+
+        if [[ "$CLEANUP" == true ]]; then
+            cleanup_local_environment
+            exit 0
+        fi
+
+        setup_local_environment
+
+        if [[ "$DRY_RUN" != true ]]; then
+            show_local_info
+        fi
+
+        print_success "=== PHASE 1 (LOCAL) COMPLETED ==="
+        print_info "Local Docker environment ready!"
+    else
+        print_info "Starting Phase 1: Infrastructure Deployment..."
+
+        check_prerequisites
+
+        if [[ "$CLEANUP" == true ]]; then
+            cleanup_resources
+            exit 0
+        fi
+
+        clone_samples
+        generate_cdk_context
+        setup_python_env
+        bootstrap_cdk
+        deploy_infrastructure
+
+        if [[ "$DRY_RUN" != true ]]; then
+            show_deployment_info
+        fi
+
+        print_success "=== PHASE 1 COMPLETED ==="
+        print_info "Infrastructure deployed successfully!"
     fi
-
-    print_success "=== PHASE 1 COMPLETED ==="
-    print_info "Infrastructure deployed successfully!"
 }
 
 # Run main function

databases/data-platform/multitenant-analytics-platform/2-etl-manager.sh

@@ -57,10 +57,10 @@ EXAMPLES:
     $0 -p aurora-postgresql -c config.json --upload-script "test-script.sh"
 
     # Skip file transfer for faster execution (requires previous transfer)
-    $0 -p aurora-postgresql -c config.json --skip-copy --bastion-command "scripts/2-sql-execute.sh config.json sql/aurora/verification/verify-setup.sql"
+    $0 -p aurora-postgresql -c config.json --skip-copy --bastion-command "scripts/aurora-sql-execute.sh config.json sql/aurora/verification/verify-setup.sql"
 
     # Local execution with docker compose
-    $0 -p aurora-postgresql -c config.json --bastion-command "scripts/2-sql-execute.sh config.json sql/aurora/verification/verify-setup.sql" --local
+    $0 -p aurora-postgresql -c config.json --bastion-command "scripts/aurora-sql-execute.sh config.json sql/aurora/verification/verify-setup.sql" --local
 
     # Dry run mode
     $0 -p aurora-postgresql -c config.json --bastion-command "df -h" --dry-run
@@ -331,6 +331,122 @@ create_directory_archive() {
     fi
 }
 
+# Function to transfer files to Docker container based on config.json
+transfer_files_to_docker_container() {
+    local config_file="$1"
+    local container_name="multitenant-analytics-platform-dbt-local-1"
+    local container_path="/usr/app"
+
+    print_info "=== DOCKER CONTAINER FILE TRANSFER ==="
+    print_info "Config file: $config_file"
+    print_info "Container: $container_name"
+    print_info "Target path: $container_path"
+
+    # Check if jq is available
+    if ! command -v jq >/dev/null 2>&1; then
+        print_warning "jq not found, using default file transfer"
+        # Fallback: copy essential files
+        docker cp config.json "$container_name:$container_path/" 2>/dev/null || print_warning "Failed to copy config.json"
+        docker cp sql/ "$container_name:$container_path/" 2>/dev/null || print_warning "Failed to copy sql directory"
+        docker cp scripts/ "$container_name:$container_path/" 2>/dev/null || print_warning "Failed to copy scripts directory"
+        return 0
+    fi
+
+    # Check if Phase 2 auto-transfer is enabled
+    local auto_transfer_enabled=$(jq -r '.bastion.phase2.autoTransfer.enabled // false' "$config_file" 2>/dev/null)
+
+    if [[ "$auto_transfer_enabled" != "true" ]]; then
+        print_info "Auto-transfer not enabled in config, using default file transfer"
+        # Fallback: copy essential files
+        docker cp config.json "$container_name:$container_path/" 2>/dev/null || print_warning "Failed to copy config.json"
+        docker cp sql/ "$container_name:$container_path/" 2>/dev/null || print_warning "Failed to copy sql directory"
+        docker cp scripts/ "$container_name:$container_path/" 2>/dev/null || print_warning "Failed to copy scripts directory"
+        return 0
+    fi
+
+    # Get directories to transfer (Phase 2 specific)
+    local directories=$(jq -r '.bastion.phase2.autoTransfer.directories[]? // empty' "$config_file" 2>/dev/null)
+
+    # Get individual files to transfer (Phase 2 specific)
+    local files=$(jq -r '.bastion.phase2.autoTransfer.files[]? // empty' "$config_file" 2>/dev/null)
+
+    # Get exclude patterns (Phase 2 specific)
+    local exclude_patterns=$(jq -r '.bastion.phase2.autoTransfer.excludePatterns[]? // empty' "$config_file" 2>/dev/null)
+
+    print_info "Transfer configuration:"
+    print_info "  Directories: $(echo "$directories" | tr '\n' ' ')"
+    print_info "  Files: $(echo "$files" | tr '\n' ' ')"
+    print_info "  Exclude patterns: $(echo "$exclude_patterns" | tr '\n' ' ')"
+
+    # Transfer directories
+    local transfer_count=0
+    for dir in $directories; do
+        if [[ -d "$dir" ]]; then
+            local dir_size=$(du -sh "$dir" 2>/dev/null | cut -f1)
+            local file_count=$(find "$dir" -type f | wc -l)
+            print_info "Copying directory: $dir (size: $dir_size, files: $file_count)"
+
+            # Create parent directory structure in container if needed
+            local parent_dir=$(dirname "$dir")
+            if [[ "$parent_dir" != "." ]]; then
+                docker exec "$container_name" mkdir -p "$container_path/$parent_dir" 2>/dev/null || true
+            fi
+
+            if docker cp "$dir" "$container_name:$container_path/$dir"; then
+                print_success "Successfully copied directory: $dir"
+                transfer_count=$((transfer_count + 1))
+            else
+                print_warning "Failed to copy directory: $dir"
+            fi
+        else
+            print_warning "Directory not found, skipping: $dir"
+        fi
+    done
+
+    # Transfer individual files
+    for file in $files; do
+        if [[ -f "$file" ]]; then
+            local file_size=$(ls -lh "$file" 2>/dev/null | awk '{print $5}')
+            print_info "Copying file: $file (size: $file_size)"
+
+            # Create directory structure in container if needed
+            local file_dir=$(dirname "$file")
+            if [[ "$file_dir" != "." ]]; then
+                docker exec "$container_name" mkdir -p "$container_path/$file_dir" 2>/dev/null || true
+            fi
+
+            if docker cp "$file" "$container_name:$container_path/$file"; then
+                print_success "Successfully copied file: $file"
+                transfer_count=$((transfer_count + 1))
+            else
+                print_warning "Failed to copy file: $file"
+            fi
+        else
+            print_warning "File not found, skipping: $file"
+        fi
+    done
+
+    # Set execute permissions on scripts
+    print_info "Setting execute permissions on scripts..."
+    docker exec "$container_name" bash -c "find $container_path/scripts -name '*.sh' -type f -exec chmod +x {} \; 2>/dev/null || true"
+
+    # Verify transfer
+    print_info "Verifying transferred files in Docker container..."
+    local verify_output=$(docker exec "$container_name" bash -c "cd $container_path && echo 'VERIFY: Current directory:' && pwd && echo 'VERIFY: Directory contents:' && ls -la && echo 'VERIFY: Key files check:' && if [ -f config.json ]; then echo 'VERIFY: config.json exists'; else echo 'VERIFY: config.json MISSING'; fi && if [ -f scripts/aurora-sql-execute.sh ]; then echo 'VERIFY: scripts/aurora-sql-execute.sh exists'; else echo 'VERIFY: scripts/auora.sh MISSING'; fi && if [ -f sql/aurora/schema/create-tenant-schemas.sql ]; then echo 'VERIFY: sql/aurora/schema/create-tenant-schemas.sql exists'; else echo 'VERIFY: sql/aurora/schema/create-tenant-schemas.sql MISSING'; fi" 2>/dev/null)
+
+    if [[ -n "$verify_output" ]]; then
+        print_info "Docker container file verification:"
+        echo "$verify_output" | while read line; do
+            if [[ "$line" == *"VERIFY:"* ]]; then
+                print_info "  $line"
+            fi
+        done
+    fi
+
+    print_success "File transfer to Docker container completed"
+    print_info "Total items transferred: $transfer_count"
+}
+
 # Function to get Aurora connection information
 get_aurora_connection_info() {
     local is_local_execution="$1"  # true for local, false for bastion
@@ -586,7 +702,7 @@ execute_bastion_command() {
                 # Verify files were transferred successfully
                 print_info "Verifying transferred files on Bastion Host..."
                 # Use a simpler verification approach to avoid JSON escaping issues
-                local verify_command="cd $workspace_dir && echo 'VERIFY: Current directory:' && pwd && echo 'VERIFY: Directory contents:' && ls -la && echo 'VERIFY: Checking key files:' && if [ -f scripts/2-sql-execute.sh ]; then echo 'VERIFY: scripts/2-sql-execute.sh exists'; else echo 'VERIFY: scripts/2-sql-execute.sh MISSING'; fi && if [ -f config.json ]; then echo 'VERIFY: config.json exists'; else echo 'VERIFY: config.json MISSING'; fi && if [ -f sql/aurora/schema/create-tenant-schemas.sql ]; then echo 'VERIFY: sql/aurora/schema/create-tenant-schemas.sql exists'; else echo 'VERIFY: sql/aurora/schema/create-tenant-schemas.sql MISSING'; fi && echo 'VERIFY: Setting execute permissions on scripts...' && chmod +x scripts/*.sh 2>/dev/null || true && echo 'VERIFY: Verification completed'"
+                local verify_command="cd $workspace_dir && echo 'VERIFY: Current directory:' && pwd && echo 'VERIFY: Directory contents:' && ls -la && echo 'VERIFY: Checking key files:' && if [ -f scripts/aurora-sql-execute.sh ]; then echo 'VERIFY: scripts/aurora-sql-execute.sh exists'; else echo 'VERIFY: scripts/aurora-sql-execute.sh MISSING'; fi && if [ -f config.json ]; then echo 'VERIFY: config.json exists'; else echo 'VERIFY: config.json MISSING'; fi && if [ -f sql/aurora/schema/create-tenant-schemas.sql ]; then echo 'VERIFY: sql/aurora/schema/create-tenant-schemas.sql exists'; else echo 'VERIFY: sql/aurora/schema/create-tenant-schemas.sql MISSING'; fi && echo 'VERIFY: Setting execute permissions on scripts...' && chmod +x scripts/*.sh 2>/dev/null || true && echo 'VERIFY: Verification completed'"
 
                 local verify_command_id=$(aws ssm send-command \
                     --instance-ids "$bastion_instance_id" \
@@ -886,14 +1002,19 @@ main() {
                 sleep 2  # Give postgres time to start
             fi
 
-            # Set up local environment variables
-            get_aurora_connection_info "true" "$PATTERN"
+            # Handle file transfer to Docker container (similar to Bastion Host transfer)
+            if [[ "$SKIP_COPY" == true ]]; then
+                print_info "Skipping file transfer to Docker container (--skip-copy specified)"
+            else
+                # Transfer files to Docker container based on config.json
+                transfer_files_to_docker_container "$CONFIG_FILE"
+            fi
 
-            # Execute command locally
+            # Execute command locally with LOCAL_EXECUTION environment variable
             print_info "Executing command locally..."
             local start_time=$(date +%s)
 
-            if eval "$BASTION_COMMAND"; then
+            if LOCAL_EXECUTION=true eval "$BASTION_COMMAND"; then
                 local end_time=$(date +%s)
                 local duration=$((end_time - start_time))
                 print_success "Local command executed successfully!"