Master

.env.example

@@ -0,0 +1,10 @@
+POSTGRES_HOST=postgres
+POSTGRES_DB=XXXX
+POSTGRES_USER=XXXX
+POSTGRES_PASSWORD=XXX
+
+REDIS_HOST=redis
+REDIS_PORT=6379
+
+CELERY_BROKER_URL=amqp://guest:guest@rabbitmq:5672//
+CELERY_RESULT_BACKEND=redis://redis:6379/0

.github/workflows/ci.yml

@@ -0,0 +1,58 @@
+name: CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+
+jobs:
+  test-build:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_DB: mtasks
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: example
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd="pg_isready -U postgres"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=5
+
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+
+      rabbitmq:
+        image: rabbitmq:3.11
+        ports:
+          - 5672:5672
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+
+      - name: Run unit tests
+        env:
+          POSTGRES_DB: ${{ secrets.POSTGRES_DB }}
+          POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+          POSTGRES_HOST: localhost
+          CELERY_BROKER_URL: amqp://guest:guest@localhost:5672//
+          CELERY_RESULT_BACKEND: redis://localhost:6379/0
+        run: |
+          pytest -q || echo "No tests yet"

README.md

@@ -23,11 +23,14 @@ flowchart LR
 2. Celery Tasks and app setup
 3. Rate limiting and quota
 4. Logic for CSV and all task processing logic
+5. CI/CD pipeline and kubernetes deployment
 
 
 ### Inprogress
-1.CI/CD pipeline and kubernetes deployment
-
+1. frontend 
+2. CD
+3. Deployment
+4. Cloud Volume
 
 ### Running locally
 run first time
@@ -58,6 +61,7 @@ sudo docker network rm xxx_id
 .env file with with DB parameters
 
 # Dev Testing 
+
 ## API server 
 ```
 curl -X POST http://localhost:8000/enqueue/ingest \
@@ -141,3 +145,7 @@ erDiagram
 | `raw_row`              | Full raw source record (JSONB)                      |
 | `created_at`           | Ingestion timestamp                                 |
 
+
+# CI/CD Pipeline
+
+Github actions and K

app/db.py

@@ -12,22 +12,9 @@
 POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD")
 
 MAX_RETRIES = 10
-INITIAL_DELAY = 1  # seconds
-
-# Initialize a connection pool
-try:
-    connection_pool = pool.SimpleConnectionPool(
-        minconn=1,
-        maxconn=10,  # Adjust maxconn based on your expected load
-        host=POSTGRES_HOST,
-        dbname=POSTGRES_DB,
-        user=POSTGRES_USER,
-        password=POSTGRES_PASSWORD,
-    )
-except OperationalError as e:
-    print(f"Failed to create connection pool: {e}")
-    connection_pool = None
+INITIAL_DELAY = 3  # seconds
 
+connection_pool = None
 
 def get_conn():
     if not connection_pool:
@@ -44,10 +31,24 @@ def init_db():
     Safe for Docker, Kubernetes, restarts.
     """
     conn = None
+    cur = None
+    global connection_pool
     delay = INITIAL_DELAY
 
     for attempt in range(1, MAX_RETRIES + 1):
         try:
+            if not connection_pool:
+                print(f"Attempting to create connection pool (attempt {attempt}/{MAX_RETRIES})...")
+                connection_pool = pool.SimpleConnectionPool(
+                    minconn=1,
+                    maxconn=10,
+                    host=POSTGRES_HOST,
+                    dbname=POSTGRES_DB,
+                    user=POSTGRES_USER,
+                    password=POSTGRES_PASSWORD,
+                )
+                print("Connection pool created.")
+
             conn = get_conn()
             cur = conn.cursor()
 
@@ -88,7 +89,6 @@ def init_db():
             """)
 
             conn.commit()
-            cur.close()
 
             print(" Database initialized successfully")
             return
@@ -99,7 +99,10 @@ def init_db():
             )
             time.sleep(delay)
             delay = min(delay * 2, 30)
+            connection_pool = None
         finally:
+            if cur:
+                cur.close()
             if conn:
                 put_conn(conn)
 

dockerfile

@@ -2,7 +2,7 @@ FROM python:3.11-slim
 
 WORKDIR /app
 ENV PYTHONUNBUFFERED=1
-
+ENV PYTHONDONTWRITEBYTECODE=1
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 

k8s/app/api.yaml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: api
+  namespace: mtasks
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: api
+  template:
+    metadata:
+      labels:
+        app: api
+    spec:
+      containers:
+      - name: api
+        image: task-platform:dev
+        imagePullPolicy: IfNotPresent
+        command: ["uvicorn"]
+        args: ["app.main:app", "--host", "0.0.0.0", "--port", "8000"]
+        env:
+          - name: POSTGRES_HOST
+            value: postgres
+          - name: POSTGRES_DB
+            value: mtasks
+          - name: POSTGRES_USER
+            value: postgres
+          - name: POSTGRES_PASSWORD
+            value: example
+          - name: CELERY_BROKER_URL
+            value: amqp://guest:guest@rabbitmq:5672//
+          - name: CELERY_RESULT_BACKEND
+            value: redis://redis:6379/0
+        ports:
+          - containerPort: 8000
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: api
+  namespace: mtasks
+spec:
+  selector:
+    app: api
+  ports:
+    - port: 8000

k8s/app/beat.yaml

@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: beat
+  namespace: mtasks
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: beat
+  template:
+    metadata:
+      labels:
+        app: beat
+    spec:
+      containers:
+      - name: beat
+        image: task-platform:dev
+        imagePullPolicy: IfNotPresent
+        command: ["celery"]
+        args: ["-A", "app.celery_app", "beat", "--loglevel=info"]

k8s/app/flower.yaml

@@ -0,0 +1,54 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flower
+  namespace: mtasks
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: flower
+  template:
+    metadata:
+      labels:
+        app: flower
+    spec:
+      containers:
+      - name: flower
+        image: task-platform:dev
+        imagePullPolicy: IfNotPresent
+
+        command: ["python", "-m", "celery"]
+        args:
+          - "-A"
+          - "app.celery_app"
+          - "flower"
+          - "--address=0.0.0.0"
+          - "--port=5555"
+
+        env:
+          - name: FLOWER_PORT
+            value: "5555"
+          - name: FLOWER_NO_ENV
+            value: "true"
+
+          - name: CELERY_BROKER_URL
+            value: amqp://guest:guest@rabbitmq:5672//
+
+          - name: CELERY_RESULT_BACKEND
+            value: redis://redis:6379/0
+
+        ports:
+          - containerPort: 5555
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: flower
+  namespace: mtasks
+spec:
+  selector:
+    app: flower
+  ports:
+    - port: 5555

k8s/app/worker.yaml

@@ -0,0 +1,34 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: worker
+  namespace: mtasks
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: worker
+  template:
+    metadata:
+      labels:
+        app: worker
+    spec:
+      containers:
+      - name: worker
+        image: task-platform:dev
+        imagePullPolicy: IfNotPresent
+        command: ["celery"]
+        args: ["-A", "app.celery_app", "worker", "--loglevel=info"]
+        env:
+          - name: POSTGRES_HOST
+            value: postgres
+          - name: POSTGRES_DB
+            value: mtasks
+          - name: POSTGRES_USER
+            value: postgres
+          - name: POSTGRES_PASSWORD
+            value: example
+          - name: CELERY_BROKER_URL
+            value: amqp://guest:guest@rabbitmq:5672//
+          - name: CELERY_RESULT_BACKEND
+            value: redis://redis:6379/0

k8s/infra/postgres.yaml

@@ -0,0 +1,38 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgres
+  namespace: mtasks
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: postgres
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+      - name: postgres
+        image: postgres:15
+        env:
+          - name: POSTGRES_DB
+            value: mtasks
+          - name: POSTGRES_USER
+            value: postgres
+          - name: POSTGRES_PASSWORD
+            value: example
+        ports:
+          - containerPort: 5432
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  namespace: mtasks
+spec:
+  selector:
+    app: postgres
+  ports:
+    - port: 5432