S009 Celery, results backend and task scheduler installation

.dockerignore

@@ -0,0 +1,39 @@
+# Django project
+/media/
+/static/
+*.sqlite3
+
+# Python and others
+__pycache__
+*.pyc
+.DS_Store
+*.swp
+/venv/
+/tmp/
+/.vagrant/
+/Vagrantfile.local
+node_modules/
+/npm-debug.log
+/.idea/
+.vscode
+coverage
+.python-version
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg

.gitignore

@@ -109,6 +109,9 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+.env_prod
+env_dev
+env.env
 
 # Spyder project settings
 .spyderproject

Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3
+
+WORKDIR /
+
+# environment variables
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+# apt-get update
+RUN apt-get update
+
+# psycog2 dependencies
+RUN apt-get -y install libpq-dev python-dev
+RUN pip install psycopg2
+
+# dependencies
+COPY requirements.txt ./
+RUN pip install --upgrade pip
+RUN pip install --no-cache-dir -r requirements.txt
+RUN rm requirements.txt
+
+# copy project and set workdir
+COPY . /
+WORKDIR /hyde

docker-compose.yml

@@ -0,0 +1,96 @@
+version: '3.6'
+
+services:
+  app:
+    container_name: app
+    build: ./
+    image: &hyde hyde
+    restart: always
+    command: sh -c "python manage.py makemigrations && python manage.py migrate && python manage.py runserver 0.0.0.0:8000"
+    env_file:
+      &envfile
+      - env_dev
+    ports:
+      - "8000:8000"
+    volumes:
+      - ./hyde:/usr/src/hyde
+    depends_on:
+      - rabbit
+    networks:
+      - app_network
+
+  app_db:
+    container_name: app_db
+    image: postgres:latest
+    env_file: *envfile
+    volumes:
+      - postgres_data:/var/lib/postgresql/data/
+    ports:
+      - "5432:5432"
+    expose:
+      - "5432"
+    networks:
+      - app_network
+
+  rabbit:
+    container_name: rabbit
+    image: rabbitmq:management
+    # env_file: *envfile
+    env_file: *envfile
+    ports:
+      - "5672:5672"
+      - 15672:15672
+    networks:
+      - app_network
+    restart: on-failure
+
+  worker:
+    build: .
+    image: *hyde
+    restart: on-failure
+    env_file: *envfile
+    container_name: worker
+    command: sh -c "celery -A engine.celery.app worker -l info"
+    volumes:
+      - ./hyde:/hyde
+    depends_on:
+      - app
+      - rabbit
+      - app_db
+    networks:
+      - app_network
+
+  beat:
+    build: .
+    image: *hyde
+    restart: on-failure
+    env_file: *envfile
+    container_name: beat
+    command: sh -c "celery -A engine.celery.app beat -l info"
+    volumes:
+      - ./hyde:/hyde
+    depends_on:
+      - app
+      - rabbit
+      - app_db
+      - worker
+    networks:
+      - app_network
+
+  flower:
+    image: mher/flower
+    restart: on-failure
+    env_file: *envfile
+    ports:
+      - "5555:5555"
+    depends_on:
+      - rabbit
+    networks:
+      - app_network
+
+networks:
+  app_network:
+
+
+volumes:
+  postgres_data:

hyde/engine/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import absolute_import, unicode_literals
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+__all__ = ('celery_app',)

hyde/engine/celery.py

@@ -0,0 +1,8 @@
+import os
+from celery import Celery
+from django.conf import settings
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "hyde.settings")
+app = Celery("hyde")
+app.conf.update(settings.CELERY)
+app.autodiscover_tasks()

hyde/engine/tasks.py

@@ -0,0 +1,20 @@
+from __future__ import absolute_import, unicode_literals
+from celery import shared_task
+from hyde.celery import app
+
+# Create your tasks here
+
+@shared_task
+def add(x, y):
+    return x + y
+
+
+@shared_task
+def mul(x, y):
+    return x * y
+
+
+@shared_task
+def xsum(numbers):
+    return sum(numbers)
+

hyde/engine/urls.py

@@ -0,0 +1,4 @@
+from django.urls import path
+from engine import views
+app_name = 'engine'
+urlpatterns = [path('publish', views.my_pub_view, name='publish'),]

hyde/engine/views.py

@@ -1,3 +1,8 @@
 from django.shortcuts import render
 
 # Create your views here.
+from django.http import HttpResponse
+from engine import tasks
+def my_pub_view(request):
+    tasks.publish_message({'hello': 'world'})
+    return HttpResponse(status=201)

hyde/hyde/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import absolute_import, unicode_literals
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+__all__ = ('celery_app',)

hyde/hyde/celery.py

@@ -0,0 +1,7 @@
+import os
+from celery import Celery
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "hyde.settings")
+app = Celery("hyde")
+app.config_from_object("django.conf:settings", namespace="CELERY")
+app.autodiscover_tasks()

hyde/hyde/settings.py

@@ -20,7 +20,7 @@
 env = environ.Env()
 
 # Get environment variables from .env file
-env.read_env(str(BASE_DIR / ".env"))
+env.read_env(str(BASE_DIR.parent / "env_dev"))
 
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/
@@ -43,8 +43,10 @@
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    'django_celery_beat',
+    'django_celery_results',
     'django_extensions',
-    'engine'
+    'engine',
 ]
 
 MIDDLEWARE = [
@@ -82,8 +84,17 @@
 # https://docs.djangoproject.com/en/4.1/ref/settings/#databases
 
 DATABASES = {
-    "default": {'ENGINE':'django.db.backends.sqlite3', 'NAME':BASE_DIR / 'db.sqlite3'},
+    # "default": {'ENGINE':'django.db.backends.sqlite3', 'NAME':BASE_DIR / 'db.sqlite3'},
+    # "default": env.db("DEFAULT_URL"),
     "hypance_db": env.db("ENGINE_URL"),
+    "default": {
+        "ENGINE": env("SQL_ENGINE"),
+        "NAME": env("SQL_DATABASE"),
+        "USER": env("SQL_USER"),
+        "PASSWORD": env("SQL_PASSWORD"),
+        "HOST": env("SQL_HOST"),
+        "PORT": env("SQL_PORT"),
+    }
         }
 
 DATABASE_ROUTERS = [
@@ -131,3 +142,14 @@
 # https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field
 
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+
+# Celery settings
+CELERY = {
+    'imports': ('engine.tasks', ),
+    'task_serializer': 'json',
+    'result_serializer': 'json',
+    'accept_content': ['json'],
+    'result_backend': env("CELERY_RESULTS_BACKEND_URL"),
+    'beat_scheduler': 'django_celery_beat.schedulers:DatabaseScheduler',
+    # 'cache_backend': env("CELERY_CACHE_BACKEND_URL"),
+}

hyde/routers/db_routers.py

@@ -1,5 +1,5 @@
 class DefaultRouter:
-    route_app_labels = {'auth', 'contenttypes', 'sessions', 'admin'}
+    route_app_labels = {'auth', 'contenttypes', 'sessions', 'admin', 'django_celery_beat', 'django_celery_results'}
 
     def db_for_read(self, model, **hints):
         if model._meta.app_label in self.route_app_labels:

requirements.txt

@@ -1,5 +1,10 @@
 git+https://github.com/Hypance/HypanceDataAnalysis.git
+numpy==1.23.3
 Django==4.1.2
+django-celery-beat==2.4.0
 django-environ==0.9.0
 django-extensions==3.2.1
-psycopg2==2.9.4
+django-celery-results==2.4.0
+psycopg2==2.9.4
+celery==5.2.7
+docker