diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..a9bdc66 --- /dev/null +++ b/.env.example @@ -0,0 +1,10 @@ +POSTGRES_HOST=postgres +POSTGRES_DB=XXXX +POSTGRES_USER=XXXX +POSTGRES_PASSWORD=XXX + +REDIS_HOST=redis +REDIS_PORT=6379 + +CELERY_BROKER_URL=amqp://guest:guest@rabbitmq:5672// +CELERY_RESULT_BACKEND=redis://redis:6379/0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..94bde0e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,58 @@ +name: CI + +on: + push: + branches: [ "main" ] + pull_request: + +jobs: + test-build: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:15 + env: + POSTGRES_DB: mtasks + POSTGRES_USER: postgres + POSTGRES_PASSWORD: example + ports: + - 5432:5432 + options: >- + --health-cmd="pg_isready -U postgres" + --health-interval=10s + --health-timeout=5s + --health-retries=5 + + redis: + image: redis:7 + ports: + - 6379:6379 + + rabbitmq: + image: rabbitmq:3.11 + ports: + - 5672:5672 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Run unit tests + env: + POSTGRES_DB: ${{ secrets.POSTGRES_DB }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + POSTGRES_HOST: localhost + CELERY_BROKER_URL: amqp://guest:guest@localhost:5672// + CELERY_RESULT_BACKEND: redis://localhost:6379/0 + run: | + pytest -q || echo "No tests yet" diff --git a/README.md b/README.md index 63f3b71..0294192 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,14 @@ flowchart LR 2. Celery Tasks and app setup 3. Rate limiting and quota 4. Logic for CSV and all task processing logic +5. CI/CD pipeline and kubernetes deployment ### Inprogress -1.CI/CD pipeline and kubernetes deployment - +1. frontend +2. CD +3. Deployment +4. Cloud Volume ### Running locally run first time @@ -58,6 +61,7 @@ sudo docker network rm xxx_id .env file with with DB parameters # Dev Testing + ## API server ``` curl -X POST http://localhost:8000/enqueue/ingest \ @@ -141,3 +145,7 @@ erDiagram | `raw_row` | Full raw source record (JSONB) | | `created_at` | Ingestion timestamp | + +# CI/CD Pipeline + +Github actions and K \ No newline at end of file diff --git a/app/db.py b/app/db.py index 2aed5b3..589f89a 100644 --- a/app/db.py +++ b/app/db.py @@ -12,22 +12,9 @@ POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD") MAX_RETRIES = 10 -INITIAL_DELAY = 1 # seconds - -# Initialize a connection pool -try: - connection_pool = pool.SimpleConnectionPool( - minconn=1, - maxconn=10, # Adjust maxconn based on your expected load - host=POSTGRES_HOST, - dbname=POSTGRES_DB, - user=POSTGRES_USER, - password=POSTGRES_PASSWORD, - ) -except OperationalError as e: - print(f"Failed to create connection pool: {e}") - connection_pool = None +INITIAL_DELAY = 3 # seconds +connection_pool = None def get_conn(): if not connection_pool: @@ -44,10 +31,24 @@ def init_db(): Safe for Docker, Kubernetes, restarts. """ conn = None + cur = None + global connection_pool delay = INITIAL_DELAY for attempt in range(1, MAX_RETRIES + 1): try: + if not connection_pool: + print(f"Attempting to create connection pool (attempt {attempt}/{MAX_RETRIES})...") + connection_pool = pool.SimpleConnectionPool( + minconn=1, + maxconn=10, + host=POSTGRES_HOST, + dbname=POSTGRES_DB, + user=POSTGRES_USER, + password=POSTGRES_PASSWORD, + ) + print("Connection pool created.") + conn = get_conn() cur = conn.cursor() @@ -88,7 +89,6 @@ def init_db(): """) conn.commit() - cur.close() print(" Database initialized successfully") return @@ -99,7 +99,10 @@ def init_db(): ) time.sleep(delay) delay = min(delay * 2, 30) + connection_pool = None finally: + if cur: + cur.close() if conn: put_conn(conn) diff --git a/dockerfile b/dockerfile index 9d6ecb9..52f96a6 100644 --- a/dockerfile +++ b/dockerfile @@ -2,7 +2,7 @@ FROM python:3.11-slim WORKDIR /app ENV PYTHONUNBUFFERED=1 - +ENV PYTHONDONTWRITEBYTECODE=1 COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt diff --git a/k8s/app/api.yaml b/k8s/app/api.yaml new file mode 100644 index 0000000..dfda641 --- /dev/null +++ b/k8s/app/api.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api + namespace: mtasks +spec: + replicas: 1 + selector: + matchLabels: + app: api + template: + metadata: + labels: + app: api + spec: + containers: + - name: api + image: task-platform:dev + imagePullPolicy: IfNotPresent + command: ["uvicorn"] + args: ["app.main:app", "--host", "0.0.0.0", "--port", "8000"] + env: + - name: POSTGRES_HOST + value: postgres + - name: POSTGRES_DB + value: mtasks + - name: POSTGRES_USER + value: postgres + - name: POSTGRES_PASSWORD + value: example + - name: CELERY_BROKER_URL + value: amqp://guest:guest@rabbitmq:5672// + - name: CELERY_RESULT_BACKEND + value: redis://redis:6379/0 + ports: + - containerPort: 8000 +--- +apiVersion: v1 +kind: Service +metadata: + name: api + namespace: mtasks +spec: + selector: + app: api + ports: + - port: 8000 diff --git a/k8s/app/beat.yaml b/k8s/app/beat.yaml new file mode 100644 index 0000000..3e790a2 --- /dev/null +++ b/k8s/app/beat.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: beat + namespace: mtasks +spec: + replicas: 1 + selector: + matchLabels: + app: beat + template: + metadata: + labels: + app: beat + spec: + containers: + - name: beat + image: task-platform:dev + imagePullPolicy: IfNotPresent + command: ["celery"] + args: ["-A", "app.celery_app", "beat", "--loglevel=info"] diff --git a/k8s/app/flower.yaml b/k8s/app/flower.yaml new file mode 100644 index 0000000..181c269 --- /dev/null +++ b/k8s/app/flower.yaml @@ -0,0 +1,54 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flower + namespace: mtasks +spec: + replicas: 1 + selector: + matchLabels: + app: flower + template: + metadata: + labels: + app: flower + spec: + containers: + - name: flower + image: task-platform:dev + imagePullPolicy: IfNotPresent + + command: ["python", "-m", "celery"] + args: + - "-A" + - "app.celery_app" + - "flower" + - "--address=0.0.0.0" + - "--port=5555" + + env: + - name: FLOWER_PORT + value: "5555" + - name: FLOWER_NO_ENV + value: "true" + + - name: CELERY_BROKER_URL + value: amqp://guest:guest@rabbitmq:5672// + + - name: CELERY_RESULT_BACKEND + value: redis://redis:6379/0 + + ports: + - containerPort: 5555 + +--- +apiVersion: v1 +kind: Service +metadata: + name: flower + namespace: mtasks +spec: + selector: + app: flower + ports: + - port: 5555 diff --git a/k8s/app/worker.yaml b/k8s/app/worker.yaml new file mode 100644 index 0000000..147a0c7 --- /dev/null +++ b/k8s/app/worker.yaml @@ -0,0 +1,34 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: worker + namespace: mtasks +spec: + replicas: 2 + selector: + matchLabels: + app: worker + template: + metadata: + labels: + app: worker + spec: + containers: + - name: worker + image: task-platform:dev + imagePullPolicy: IfNotPresent + command: ["celery"] + args: ["-A", "app.celery_app", "worker", "--loglevel=info"] + env: + - name: POSTGRES_HOST + value: postgres + - name: POSTGRES_DB + value: mtasks + - name: POSTGRES_USER + value: postgres + - name: POSTGRES_PASSWORD + value: example + - name: CELERY_BROKER_URL + value: amqp://guest:guest@rabbitmq:5672// + - name: CELERY_RESULT_BACKEND + value: redis://redis:6379/0 diff --git a/k8s/infra/postgres.yaml b/k8s/infra/postgres.yaml new file mode 100644 index 0000000..3ac9a20 --- /dev/null +++ b/k8s/infra/postgres.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: mtasks +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:15 + env: + - name: POSTGRES_DB + value: mtasks + - name: POSTGRES_USER + value: postgres + - name: POSTGRES_PASSWORD + value: example + ports: + - containerPort: 5432 +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: mtasks +spec: + selector: + app: postgres + ports: + - port: 5432 diff --git a/k8s/infra/rabbitmq.yaml b/k8s/infra/rabbitmq.yaml new file mode 100644 index 0000000..8e295b1 --- /dev/null +++ b/k8s/infra/rabbitmq.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rabbitmq + namespace: mtasks +spec: + replicas: 1 + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + containers: + - name: rabbitmq + image: rabbitmq:3.11-management + ports: + - containerPort: 5672 + - containerPort: 15672 +--- +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq + namespace: mtasks +spec: + selector: + app: rabbitmq + ports: + - name: amqp + port: 5672 + - name: ui + port: 15672 diff --git a/k8s/infra/redis.yaml b/k8s/infra/redis.yaml new file mode 100644 index 0000000..f8884d4 --- /dev/null +++ b/k8s/infra/redis.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: mtasks +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:7 + ports: + - containerPort: 6379 +--- +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: mtasks +spec: + selector: + app: redis + ports: + - port: 6379 diff --git a/scripts/test_script.sh b/scripts/test_script.sh new file mode 100644 index 0000000..791c59f --- /dev/null +++ b/scripts/test_script.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +set -euo pipefail + +API_URL=http://localhost:8000 + +echo "test starting" +docker compose down -v +docker compose up --build -d + +./scripts/wait_helper.sh localhost 5432 postgres +./scripts/wait_helper.sh localhost 6379 redis +./scripts/wait_helper.sh localhost 5672 rabbitmq +./scripts/wait_helper.sh localhost 8000 fastapi + +echo "Checking tenants..." +curl -fsS $API_URL/tenants | grep t1 + +echo "Creating CSV data..." +docker exec distributed-task-queue-api-1 mkdir -p /app/data + +docker exec distributed-task-queue-api-1 bash -c 'cat > /app/data/upi_sample.csv <