local deployment

mtalvi · mtalvi · commit 2ace1eab35bd · 2025-12-16T16:27:40.000+02:00
diff --git a/.env.example b/.env.example
@@ -20,6 +20,10 @@ PROD_CORS_ORIGIN=http://localhost:3000
 # RAG (Retrieval-Augmented Generation) Configuration
 # ============================================================================
 EMBEDDINGS_LLM_URL=http://localhost:8080
+# RAG Service URL (microservice endpoint)
+# Backend communicates with RAG service via HTTP
+# Default: http://alm-rag:8002 (for Kubernetes) or http://localhost:8002 (for local)
+RAG_SERVICE_URL=http://localhost:8002
 # Enable/disable RAG functionality (default: true)
 RAG_ENABLED=true
 
diff --git a/Makefile b/Makefile
@@ -1,5 +1,5 @@
 # This makefile routes targets to local or helm specific makefiles
-.PHONY: all local helm help
+.PHONY: all local helm help rag-status test-rag
 
 # ifneq (,$(wildcard .env))
 # # ifneq (,$(filter local,$(MAKECMDGOALS)))
@@ -34,3 +34,7 @@ local/%: ## Route local targets to deploy/local/Makefile
 
 cluster/%: ## Route deploy targets to deploy/helm/Makefile
 	@$(MAKE) -C deploy/helm $*
+
+# Convenience targets for common local commands
+rag-status: local/rag-status ## Check RAG service status
+test-rag: local/test-rag ## Test RAG service
diff --git a/deploy/local/Makefile b/deploy/local/Makefile
@@ -1,4 +1,4 @@
-.PHONY: deploy start stop status help postgres phoenix loki-stack aap-mock replay stop-replay backend ui annotation health logs restart clean stop-postgres stop-phoenix stop-loki-stack stop-aap-mock embedding stop-embedding wait-for-embedding test-embedding test-rag
+.PHONY: deploy start stop status help postgres phoenix loki-stack aap-mock replay stop-replay backend ui annotation health logs restart clean stop-postgres stop-phoenix stop-loki-stack stop-aap-mock embedding stop-embedding wait-for-embedding rag stop-rag test-embedding test-rag
 .DEFAULT_GOAL := help
 
 include ../../.env
@@ -36,8 +36,10 @@ help: ## Show this help message
 	@echo ""
 	@echo "📚 RAG Setup:"
 	@echo "  1. Place PDF files in data/knowledge_base/"
-	@echo "  2. Run 'make run-whole-training-pipeline' to build RAG index"
-	@echo "  3. Ensure RAG_ENABLED=true in .env file"
+	@echo "  2. Start services: 'make start' (includes RAG service)"
+	@echo "  3. Run 'make run-whole-training-pipeline' to build RAG embeddings in PostgreSQL"
+	@echo "  4. RAG service will automatically load embeddings when available"
+	@echo "  5. Ensure RAG_ENABLED=true and RAG_SERVICE_URL=http://alm-rag:8002 in .env file"
 	@echo ""
 
 
@@ -51,6 +53,7 @@ start: stop ## 🚀 Start all services locally
 	@$(MAKE) -s embedding
 	@echo "   ⚠️  Note: Embedding service may take 3-5 minutes to load the model"
 	@echo "   You can check status with: make test-embedding"
+	@$(MAKE) -s rag
 	@$(MAKE) -s aap-mock
 	@$(MAKE) -s backend
 	@$(MAKE) -s ui
@@ -79,6 +82,12 @@ embedding: ## 🤖 Start Embedding Service (TEI)
 	@echo "   Starting Embedding Service (TEI)..."
 	@$(COMPOSE_CMD) -f compose.yaml up -d alm-embedding
 
+rag: ## 🔍 Start RAG Service
+	@echo "   Starting RAG Service..."
+	@$(COMPOSE_CMD) -f compose.yaml up -d alm-rag
+	@echo "   ⚠️  Note: RAG service will poll PostgreSQL for embeddings"
+	@echo "   It will become ready once embeddings are available (after init job runs)"
+
 wait-for-embedding: ## ⏳ Wait for embedding service to be ready (optional, for manual use)
 	@echo "   Waiting for embedding service to be ready (this may take 3-5 minutes for model loading)..."
 	@timeout=300; \
@@ -167,6 +176,7 @@ stop: ## 🛑 Stop all services
 	@$(MAKE) -s stop-postgres
 	@$(MAKE) -s stop-phoenix
 	@$(MAKE) -s stop-embedding
+	@$(MAKE) -s stop-rag
 	@$(MAKE) -s kill-ports
 	@echo "👋 All services stopped"
 
@@ -190,6 +200,10 @@ stop-embedding: ## 🛑 Stop Embedding Service
 	@$(COMPOSE_CMD) -f compose.yaml down alm-embedding || true
 	@echo "   ✓ Embedding service stopped and removed"
 
+stop-rag: ## 🛑 Stop RAG Service
+	@$(COMPOSE_CMD) -f compose.yaml down alm-rag || true
+	@echo "   ✓ RAG service stopped and removed"
+
 kill-ports: ## 🔌 Kill processes using required ports
 	@if [ "$$(uname)" = "Darwin" ]; then \
 		lsof -ti :7860 | xargs kill -9 2>/dev/null || true; \
@@ -239,6 +253,12 @@ kill-ports: ## 🔌 Kill processes using required ports
 		fuser -k 8081/tcp 2>/dev/null || true; \
 	fi
 	@echo "   ✓ Loki MCP Server 8081 killed"
+	@if [ "$$(uname)" = "Darwin" ]; then \
+		lsof -ti :8002 | xargs kill -9 2>/dev/null || true; \
+	else \
+		fuser -k 8002/tcp 2>/dev/null || true; \
+	fi
+	@echo "   ✓ RAG Service 8002 killed"
 	@if [ "$$(uname)" = "Darwin" ]; then \
 		lsof -ti :3000 | xargs kill -9 2>/dev/null || true; \
 	else \
@@ -259,6 +279,7 @@ status: ## 📊 Show status of all services
 	@echo "   📊 Loki: $(shell $(COMPOSE_CMD) ps 2>/dev/null | grep 'loki' | grep -v 'loki-mcp' | grep -q 'Up' && echo 'Running' || echo 'Stopped') - http://localhost:3100"
 	@echo "   🔍 Loki MCP Server: $(shell $(COMPOSE_CMD) ps 2>/dev/null | grep 'loki-mcp-server' | grep -q 'Up' && echo 'Running' || echo 'Stopped') - http://localhost:8081"
 	@echo "   🤖 Embedding Service: $(shell $(COMPOSE_CMD) ps 2>/dev/null | grep 'alm-embedding' | grep -q 'Up' && echo 'Running' || echo 'Stopped') - http://localhost:8080"
+	@echo "   🔍 RAG Service: $(shell $(COMPOSE_CMD) ps 2>/dev/null | grep 'alm-rag' | grep -q 'Up' && echo 'Running' || echo 'Stopped') - http://localhost:8002"
 	@echo "   📈 Grafana: $(shell $(COMPOSE_CMD) ps 2>/dev/null | grep 'grafana' | grep -q 'Up' && echo 'Running' || echo 'Stopped') - http://localhost:3000"
 	@echo "   📝 Promtail: $(shell $(COMPOSE_CMD) ps 2>/dev/null | grep 'promtail' | grep -q 'Up' && echo 'Running' || echo 'Stopped')"
 	@echo "   🎭 AAP Mock: $(shell $(COMPOSE_CMD) ps 2>/dev/null | grep 'aap-mock' | grep -q 'Up' && echo 'Running' || echo 'Stopped') - http://localhost:8082"
@@ -293,14 +314,20 @@ run-whole-training-pipeline: ## 🔍 Run whole training pipeline (builds RAG ind
 	@( cd ../.. && uv run init_pipeline.py )
 
 # Add new target to check RAG status
-rag-status: ## 📊 Check RAG index status
-	@echo "📊 RAG Index Status:"
-	@if [ -f "../../data/ansible_errors.index" ] && [ -f "../../data/error_metadata.pkl" ]; then \
-		echo "   ✅ RAG index exists"; \
-		ls -lh ../../data/ansible_errors.index ../../data/error_metadata.pkl; \
+rag-status: ## 📊 Check RAG service and index status
+	@echo "📊 RAG Service Status:"
+	@if curl -s -f http://localhost:8002/health >/dev/null 2>&1; then \
+		echo "   ✅ RAG service is running"; \
+		if curl -s -f http://localhost:8002/ready >/dev/null 2>&1; then \
+			index_size=$$(curl -s http://localhost:8002/ready | grep -o '"index_size":[0-9]*' | grep -o '[0-9]*' || echo "unknown"); \
+			echo "   ✅ RAG index is loaded ($$index_size embeddings)"; \
+		else \
+			echo "   ⚠️  RAG index not loaded yet (waiting for embeddings)"; \
+			echo "   Run 'make run-whole-training-pipeline' to build embeddings"; \
+		fi; \
 	else \
-		echo "   ❌ RAG index not found"; \
-		echo "   Run 'make run-whole-training-pipeline' to build it"; \
+		echo "   ❌ RAG service is not running"; \
+		echo "   Start it with: make rag"; \
 	fi
 	@if [ -d "../../data/knowledge_base" ]; then \
 		pdf_count=$$(ls -1 ../../data/knowledge_base/*.pdf 2>/dev/null | wc -l); \
@@ -335,14 +362,40 @@ test-embedding: ## 🧪 Test embedding service
 	@echo ""
 	@echo "✅ Embedding service test passed!"
 
-test-rag: ## 🧪 Test RAG system (requires RAG index)
-	@echo "🧪 Testing RAG System..."
-	@if [ ! -f "../../data/ansible_errors.index" ]; then \
-		echo "❌ RAG index not found. Run 'make run-whole-training-pipeline' first."; \
+test-rag: ## 🧪 Test RAG service (requires RAG service running and embeddings in PostgreSQL)
+	@echo "🧪 Testing RAG Service..."
+	@echo ""
+	@echo "1. Health Check:"
+	@if curl -s -f http://localhost:8002/health >/dev/null 2>&1; then \
+		echo "   ✅ Service is healthy"; \
+	else \
+		echo "   ❌ Service is not responding"; \
+		exit 1; \
+	fi
+	@echo ""
+	@echo "2. Readiness Check:"
+	@if curl -s -f http://localhost:8002/ready >/dev/null 2>&1; then \
+		echo "   ✅ Service is ready (index loaded)"; \
+	else \
+		echo "   ⚠️  Service is not ready (index not loaded yet)"; \
+		echo "   Run 'make run-whole-training-pipeline' to build embeddings"; \
+		exit 1; \
+	fi
+	@echo ""
+	@echo "3. Test Query:"
+	@response=$$(curl -s -X POST http://localhost:8002/rag/query \
+		-H "Content-Type: application/json" \
+		-d '{"query": "ansible playbook execution failed", "top_k": 5, "top_n": 3, "similarity_threshold": 0.6}'); \
+	if echo "$$response" | grep -q '"results"'; then \
+		echo "   ✅ Query successful"; \
+		echo "   Response preview: $$(echo $$response | head -c 200)..."; \
+	else \
+		echo "   ❌ Query failed"; \
+		echo "   Response: $$response"; \
 		exit 1; \
 	fi
-	@echo "Running RAG test script..."
-	@(cd ../.. && uv run python tests/rag/test_embeddings.py)
+	@echo ""
+	@echo "✅ RAG service test passed!"
 
 health: ## 🔍 Check health of running services
 	@echo "🔍 Health Checks:"
@@ -381,6 +434,16 @@ health: ## 🔍 Check health of running services
 	else \
 		echo "   🤖  Embedding Service http://localhost:8080: Unhealthy (may still be loading model)"; \
 	fi
+	@if curl -s http://localhost:8002/health >/dev/null 2>&1; then \
+		echo "   🔍  RAG Service http://localhost:8002: Healthy"; \
+		if curl -s http://localhost:8002/ready >/dev/null 2>&1; then \
+			echo "   🔍  RAG Service http://localhost:8002: Ready (index loaded)"; \
+		else \
+			echo "   🔍  RAG Service http://localhost:8002: Not ready (index not loaded yet)"; \
+		fi; \
+	else \
+		echo "   🔍  RAG Service http://localhost:8002: Unhealthy"; \
+	fi
 	@if curl -s http://localhost:3000 >/dev/null 2>&1; then \
 		echo "   📈  Grafana http://localhost:3000: Healthy"; \
 	else \
diff --git a/deploy/local/compose.yaml b/deploy/local/compose.yaml
@@ -125,7 +125,7 @@ services:
     restart: unless-stopped
 
   postgres:
-    image: postgres:15
+    image: pgvector/pgvector:pg15  # PostgreSQL with pgvector extension
     environment:
       - POSTGRES_USER=user
       - POSTGRES_PASSWORD=password
@@ -173,6 +173,7 @@ services:
       - PROD_CORS_ORIGIN=${PROD_CORS_ORIGIN}
       # RAG Configuration
       - RAG_ENABLED=${RAG_ENABLED}
+      - RAG_SERVICE_URL=${RAG_SERVICE_URL:-http://alm-rag:8002}
       - DATA_DIR=${DATA_DIR}
       - KNOWLEDGE_BASE_DIR=${KNOWLEDGE_BASE_DIR}
       - RAG_TOP_K=${RAG_TOP_K}
@@ -194,6 +195,8 @@ services:
         condition: service_healthy
       alm-embedding:
         condition: service_healthy
+      alm-rag:
+        condition: service_started  # RAG service can start before embeddings are ready
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
@@ -244,6 +247,35 @@ services:
     # Note: This service requires significant memory (8Gi recommended)
     # Adjust resources based on your system capabilities
 
+  alm-rag:
+    build:
+      context: ../..
+      dockerfile: services/rag/Containerfile
+    container_name: alm-rag
+    ports:
+      - "8002:8002"
+    environment:
+      # Use container network name for PostgreSQL (not localhost)
+      # Override any local DATABASE_URL to use container network
+      - DATABASE_URL=postgresql+asyncpg://user:password@postgres:5432/logsdb
+      - EMBEDDINGS_LLM_URL=http://alm-embedding:8080
+      - RAG_MODEL_NAME=nomic-ai/nomic-embed-text-v1.5
+      - PORT=8002
+    networks:
+      - alm
+    depends_on:
+      postgres:
+        condition: service_healthy
+      alm-embedding:
+        condition: service_healthy
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8002/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s  # Service starts immediately, but index may take time to load
+
 volumes:
   postgres_data:
   aap_mock_data:
diff --git a/init_pipeline.py b/init_pipeline.py
@@ -29,21 +29,21 @@ def setup_data_directories():
     print(f"  ✓ {data_dir}")
     print(f"  ✓ {logs_dir}")
 
-    # Check for knowledge base PDFs in image
-    image_kb_dir = Path("/app/data/knowledge_base")
-    if image_kb_dir.exists():
-        image_pdfs = list(image_kb_dir.glob("*.pdf"))
-        if image_pdfs:
-            print(f"\n✓ Found {len(image_pdfs)} PDF file(s) in container image:")
-            for pdf in image_pdfs:
+    # Check for knowledge base PDFs
+    # Use config path (works for both local and container)
+    kb_dir = Path(config.storage.knowledge_base_dir)
+    if kb_dir.exists():
+        pdfs = list(kb_dir.glob("*.pdf"))
+        if pdfs:
+            print(f"\n✓ Found {len(pdfs)} PDF file(s) in knowledge base ({kb_dir}):")
+            for pdf in pdfs:
                 print(f"  - {pdf.name}")
         else:
-            print(f"\n⚠ No PDF files found in image at {image_kb_dir}")
+            print(f"\n⚠ No PDF files found in {kb_dir}")
+            print("  Add PDF files to the knowledge base directory to enable RAG")
     else:
-        print(f"\n⚠ Knowledge base directory not found in image at {image_kb_dir}")
-        print(
-            "  PDFs should be baked into the container image at /app/data/knowledge_base"
-        )
+        print(f"\n⚠ Knowledge base directory not found at {kb_dir}")
+        print("  Create the directory and add PDF files to enable RAG")
 
     print("=" * 70)
 
@@ -108,13 +108,13 @@ async def build_rag_index():
         parser = AnsibleErrorParser()
         embedder = AnsibleErrorEmbedder()
 
-        # Find PDFs in knowledge base (from container image)
-        # PDFs should be baked into the image at /app/data/knowledge_base
-        image_kb_dir = Path("/app/data/knowledge_base")
-        pdf_files = sorted(glob.glob(str(image_kb_dir / "*.pdf")))
+        # Find PDFs in knowledge base
+        # Use config path (works for both local and container)
+        kb_dir = Path(config.storage.knowledge_base_dir)
+        pdf_files = sorted(glob.glob(str(kb_dir / "*.pdf")))
 
         if not pdf_files:
-            print(f"⚠ WARNING: No PDF files found in {image_kb_dir}")
+            print(f"⚠ WARNING: No PDF files found in {kb_dir}")
             print("  RAG index will not be created")
             return
 
diff --git a/src/alm/database.py b/src/alm/database.py
@@ -24,6 +24,19 @@
 
 # Create tables
 async def init_tables(delete_tables=False):
+    # First, try to enable pgvector extension in a separate transaction
+    # This prevents transaction abort errors if extension creation fails
+    try:
+        async with engine.begin() as ext_conn:
+            await ext_conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
+            logger.info("pgvector extension enabled")
+    except Exception as e:
+        logger.warning(f"Could not enable pgvector extension: {e}")
+        logger.warning("This is OK if extension is already enabled or not available")
+        # For local dev without pgvector, we'll continue but RAG won't work
+        # In production, this should fail
+
+    # Now create tables in a separate transaction
     async with engine.begin() as conn:
         if delete_tables:
             logger.info("Starting to delete tables")
@@ -32,16 +45,6 @@ async def init_tables(delete_tables=False):
             await conn.run_sync(GrafanaAlert.metadata.drop_all)
             # RAGEmbedding table is NOT deleted - it persists across runs
 
-        # Ensure pgvector extension is enabled (must be done before creating tables)
-        try:
-            await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
-            logger.info("pgvector extension enabled")
-        except Exception as e:
-            logger.warning(f"Could not enable pgvector extension: {e}")
-            logger.warning(
-                "This is OK if extension is already enabled or not available"
-            )
-
         # Create all tables
         await conn.run_sync(GrafanaAlert.metadata.create_all)
         await conn.run_sync(RAGEmbedding.metadata.create_all)
