From b7c8f68bf2165a33415ed4df12248357d4babc98 Mon Sep 17 00:00:00 2001 From: Toygar Aksoy Date: Sun, 6 Nov 2022 16:55:26 +0300 Subject: [PATCH 1/4] celery install initial --- Dockerfile | 20 +++++++++++ docker-compose.yml | 51 +++++++++++++++++++++++++++ hyde/engine/tasks.py | 11 ++++++ hyde/engine/urls.py | 4 +++ hyde/engine/views.py | 5 +++ hyde/hyde/__init__.py | 5 +++ hyde/hyde/celery.py | 7 ++++ hyde/hyde/settings.py | 8 ++++- requirements.txt | 4 ++- wait-for | 81 +++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 hyde/engine/tasks.py create mode 100644 hyde/engine/urls.py create mode 100644 hyde/hyde/celery.py create mode 100644 wait-for diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..83ce98f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.9-alpine3.16 + +ENV PYTHONUNBUFFERED 1 + +COPY ./requirements.txt /requirements.txt + +# Install python and postgres dependencies under a virtual package +RUN apk update +RUN apk add git +RUN apk add --update --no-cache postgresql-client +RUN apk add --update --no-cache --virtual .tmp-build-deps gcc libc-dev linux-headers postgresql-dev musl-dev +RUN pip install --upgrade pip -r requirements.txt + +# Delete virtual packages as we installed our dependencies +RUN apk del .tmp-build-deps + +# Copy and set our project folder from local to container +RUN mkdir /hyde_project +WORKDIR /hyde_project +COPY ./ /hyde_project diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d421087 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,51 @@ +version: "3" + +services: + backend: + build: + context: . + image: backend-image + container_name: backend + hostname: backend + restart: on-failure + ports: + - "8080:8080" + volumes: + - .:/hyde_project + command: > + sh -c "python manage.py migrate && python manage.py runserver 0.0.0.0:8080" + stdin_open: true + tty: true + depends_on: + - rabbit + networks: + - shared_network + + rabbit: + image: rabbitmq:3.11-management + container_name: rabbit + hostname: rabbit + environment: + - RABBITMQ_DEFAULT_USER=admin + - RABBITMQ_DEFAULT_PASS=admin + - RABBITMQ_DEFAULT_VHOST=/ + ports: + - "5682:5672" + - "15682:15672" + networks: + - shared_network + + worker: + image: backend-image + restart: on-failure + command: > + sh -c "celery worker -B -l info -A mypubsub.celery -s /tmp/celerybeat-schedule" + depends_on: + - backend + - rabbit + networks: + - shared_network + +networks: + shared_network: + driver: bridge \ No newline at end of file diff --git a/hyde/engine/tasks.py b/hyde/engine/tasks.py new file mode 100644 index 0000000..593a9b8 --- /dev/null +++ b/hyde/engine/tasks.py @@ -0,0 +1,11 @@ +from __future__ import absolute_import, unicode_literals +from celery import shared_task +from hyde.celery import app + +def publish_message(message): + with app.producer_pool.acquire(block=True) as producer: + producer.publish( + message, + exchange='myexchange', + routing_key='mykey', + ) \ No newline at end of file diff --git a/hyde/engine/urls.py b/hyde/engine/urls.py new file mode 100644 index 0000000..ebf1b05 --- /dev/null +++ b/hyde/engine/urls.py @@ -0,0 +1,4 @@ +from django.urls import path +from engine import views +app_name = 'engine' +urlpatterns = [path('publish', views.my_pub_view, name='publish'),] \ No newline at end of file diff --git a/hyde/engine/views.py b/hyde/engine/views.py index 91ea44a..b1da70a 100644 --- a/hyde/engine/views.py +++ b/hyde/engine/views.py @@ -1,3 +1,8 @@ from django.shortcuts import render # Create your views here. +from django.http import HttpResponse +from engine import tasks +def my_pub_view(request): + tasks.publish_message({'hello': 'world'}) + return HttpResponse(status=201) \ No newline at end of file diff --git a/hyde/hyde/__init__.py b/hyde/hyde/__init__.py index e69de29..68d1082 100644 --- a/hyde/hyde/__init__.py +++ b/hyde/hyde/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import, unicode_literals +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app +__all__ = ('celery_app',) \ No newline at end of file diff --git a/hyde/hyde/celery.py b/hyde/hyde/celery.py new file mode 100644 index 0000000..e70e4a0 --- /dev/null +++ b/hyde/hyde/celery.py @@ -0,0 +1,7 @@ +import os +from celery import Celery + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "hyde.settings") +app = Celery("hyde") +app.config_from_object("django.conf:settings", namespace="CELERY") +app.autodiscover_tasks() \ No newline at end of file diff --git a/hyde/hyde/settings.py b/hyde/hyde/settings.py index eaae60e..e8a3aa0 100644 --- a/hyde/hyde/settings.py +++ b/hyde/hyde/settings.py @@ -44,7 +44,8 @@ 'django.contrib.messages', 'django.contrib.staticfiles', 'django_extensions', - 'engine' + 'engine', + 'django_celery_results', ] MIDDLEWARE = [ @@ -131,3 +132,8 @@ # https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +# Celery settings +CELERY_BROKER_URL = "amqp://admin:admin@rabbit:5672//" +CELERY_RESULT_BACKEND = "django-db" +CELERY_CACHE_BACKEND = "django-cache" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2f1e436..a73556d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ git+https://github.com/Hypance/HypanceDataAnalysis.git +numpy==1.23.3 Django==4.1.2 django-environ==0.9.0 django-extensions==3.2.1 -psycopg2==2.9.4 \ No newline at end of file +psycopg2==2.9.4 +celery==5.2.7 \ No newline at end of file diff --git a/wait-for b/wait-for new file mode 100644 index 0000000..3edda5d --- /dev/null +++ b/wait-for @@ -0,0 +1,81 @@ +#!/bin/sh + +# Obtained on 19-11-2019 from https://github.com/eficode/wait-for + +TIMEOUT=15 +QUIET=0 + +echoerr() { + if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi +} + +usage() { + exitcode="$1" + cat << USAGE >&2 +Usage: + $cmdname host:port [-t timeout] [-- command args] + -q | --quiet Do not output any status messages + -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit "$exitcode" +} + +wait_for() { + for i in `seq $TIMEOUT` ; do + nc -z "$HOST" "$PORT" > /dev/null 2>&1 + + result=$? + if [ $result -eq 0 ] ; then + if [ $# -gt 0 ] ; then + exec "$@" + fi + exit 0 + fi + sleep 1 + done + echo "Operation timed out" >&2 + exit 1 +} + +while [ $# -gt 0 ] +do + case "$1" in + *:* ) + HOST=$(printf "%s\n" "$1"| cut -d : -f 1) + PORT=$(printf "%s\n" "$1"| cut -d : -f 2) + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -t) + TIMEOUT="$2" + if [ "$TIMEOUT" = "" ]; then break; fi + shift 2 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + break + ;; + --help) + usage 0 + ;; + *) + echoerr "Unknown argument: $1" + usage 1 + ;; + esac +done + +if [ "$HOST" = "" -o "$PORT" = "" ]; then + echoerr "Error: you need to provide a host and port to test." + usage 2 +fi + +wait_for "$@" \ No newline at end of file From 0786c956d561c7cb45e41a6784a422e9a8a01481 Mon Sep 17 00:00:00 2001 From: Toygar Aksoy Date: Sun, 5 Feb 2023 02:56:15 +0300 Subject: [PATCH 2/4] Async dockerized --- Dockerfile | 25 +++----- docker-compose.yml | 134 ++++++++++++++++++++++++++++++---------- env.env | 8 +++ hyde/engine/__init__.py | 5 ++ hyde/engine/celery.py | 8 +++ hyde/engine/tasks.py | 23 ++++--- hyde/hyde/settings.py | 16 +++-- requirements.txt | 5 +- wait-for | 81 ------------------------ 9 files changed, 162 insertions(+), 143 deletions(-) create mode 100644 env.env create mode 100644 hyde/engine/celery.py delete mode 100644 wait-for diff --git a/Dockerfile b/Dockerfile index 83ce98f..6a9d503 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,11 @@ -FROM python:3.9-alpine3.16 - +FROM python:3 +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV PYTHONUNBUFFERED 1 -COPY ./requirements.txt /requirements.txt - -# Install python and postgres dependencies under a virtual package -RUN apk update -RUN apk add git -RUN apk add --update --no-cache postgresql-client -RUN apk add --update --no-cache --virtual .tmp-build-deps gcc libc-dev linux-headers postgresql-dev musl-dev -RUN pip install --upgrade pip -r requirements.txt - -# Delete virtual packages as we installed our dependencies -RUN apk del .tmp-build-deps +WORKDIR / +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +RUN rm requirements.txt -# Copy and set our project folder from local to container -RUN mkdir /hyde_project -WORKDIR /hyde_project -COPY ./ /hyde_project +COPY . / +WORKDIR /hyde \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d421087..e0deafb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,51 +1,119 @@ -version: "3" +version: '3.2' services: - backend: - build: - context: . - image: backend-image - container_name: backend - hostname: backend + app: + container_name: app + build: ./ + image: &hyde hyde restart: on-failure + command: sh -c "python manage.py makemigrations && python manage.py migrate && python manage.py runserver 0.0.0.0:8000" + env_file: + &envfile + - env.env ports: - - "8080:8080" + - "8000:8000" volumes: - - .:/hyde_project - command: > - sh -c "python manage.py migrate && python manage.py runserver 0.0.0.0:8080" - stdin_open: true - tty: true + - ./hyde:/hyde depends_on: - rabbit networks: - - shared_network + - app_network + + worker: + build: . + image: *hyde + restart: on-failure + env_file: *envfile + container_name: worker + command: sh -c "celery -A engine.celery.app worker -l info" + volumes: + - ./hyde:/hyde + depends_on: + - app + - rabbit + - app_db + networks: + - app_network rabbit: - image: rabbitmq:3.11-management container_name: rabbit - hostname: rabbit + image: rabbitmq:management + # env_file: *envfile environment: - - RABBITMQ_DEFAULT_USER=admin - - RABBITMQ_DEFAULT_PASS=admin - - RABBITMQ_DEFAULT_VHOST=/ + - RABBITMQ_PASS=guest ports: - - "5682:5672" - - "15682:15672" + - "5672:5672" + - 15672:15672 networks: - - shared_network - - worker: - image: backend-image + - app_network restart: on-failure - command: > - sh -c "celery worker -B -l info -A mypubsub.celery -s /tmp/celerybeat-schedule" - depends_on: - - backend - - rabbit + + # app: + # container_name: app + # build: ./ + # image: &hyde hyde + # restart: on-failure + # command: [ "python", "manage.py", "runserver", "0.0.0.0:8000" ] + # env_file: + # &envfile + # - env.env + # ports: + # - "8000:8000" + # volumes: + # - ./hyde:/hyde + # depends_on: + # - app_rabbit_broker + # networks: + # - app_network + + # worker_engine: + # build: . + # image: *hyde + # restart: on-failure + # env_file: *envfile + # container_name: worker_engine + # command: sh -c "celery -A engine.celery.app worker -l info" + # volumes: + # - ./hyde:/hyde + # depends_on: + # - app + # - rabbit + # - app_db + # networks: + # - app_network + + + # flower: + # image: zoomeranalytics/flower:0.9.1-4.0.2 + # restart: on-failure + # env_file: *envfile + # ports: + # - "5555:5555" + # depends_on: + # - app_rabbit_broker + # networks: + # - app_network + + # app_rabbit_broker: + # container_name: app_rabbitmq + # image: rabbitmq:management + # env_file: *envfile + # ports: + # - "5672:5672" + # - 15672:15672 + # networks: + # - app_network + # restart: on-failure + + app_db: + container_name: app_db + image: postgres:13.1 + environment: + - POSTGRES_USER=app_user + - POSTGRES_PASSWORD=changeme + - POSTGRES_DB=app_db networks: - - shared_network + - app_network networks: - shared_network: - driver: bridge \ No newline at end of file + app_network: diff --git a/env.env b/env.env new file mode 100644 index 0000000..798ca26 --- /dev/null +++ b/env.env @@ -0,0 +1,8 @@ +RABBITMQ_URL=pyamqp://guest:guest@rabbit:5672// +RABBITMQ_DEFAULT_USER=guest +RABBITMQ_DEFAULT_PASS=guest +POSTGRES_USER=app_user +POSTGRES_PASSWORD=changeme +POSTGRES_DB=app_db +CELERY_BROKER_URL=pyamqp://guest@rabbit// +FLOWER_PORT=5555 \ No newline at end of file diff --git a/hyde/engine/__init__.py b/hyde/engine/__init__.py index e69de29..68d1082 100644 --- a/hyde/engine/__init__.py +++ b/hyde/engine/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import, unicode_literals +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app +__all__ = ('celery_app',) \ No newline at end of file diff --git a/hyde/engine/celery.py b/hyde/engine/celery.py new file mode 100644 index 0000000..ebdcc1f --- /dev/null +++ b/hyde/engine/celery.py @@ -0,0 +1,8 @@ +import os +from celery import Celery +from django.conf import settings + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "hyde.settings") +app = Celery("hyde") +app.conf.update(settings.CELERY) +app.autodiscover_tasks() \ No newline at end of file diff --git a/hyde/engine/tasks.py b/hyde/engine/tasks.py index 593a9b8..c72b76d 100644 --- a/hyde/engine/tasks.py +++ b/hyde/engine/tasks.py @@ -2,10 +2,19 @@ from celery import shared_task from hyde.celery import app -def publish_message(message): - with app.producer_pool.acquire(block=True) as producer: - producer.publish( - message, - exchange='myexchange', - routing_key='mykey', - ) \ No newline at end of file +# Create your tasks here + +@app.task +def add(x, y): + return x + y + + +@shared_task +def mul(x, y): + return x * y + + +@shared_task +def xsum(numbers): + return sum(numbers) + diff --git a/hyde/hyde/settings.py b/hyde/hyde/settings.py index ad2cbbb..ab07264 100644 --- a/hyde/hyde/settings.py +++ b/hyde/hyde/settings.py @@ -43,9 +43,10 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + # 'django-celery-beat', + # 'django_celery_results', 'django_extensions', 'engine', - 'django_celery_results', ] MIDDLEWARE = [ @@ -134,6 +135,13 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' # Celery settings -CELERY_BROKER_URL = "amqp://admin:admin@rabbit:5672//" -CELERY_RESULT_BACKEND = "django-db" -CELERY_CACHE_BACKEND = "django-cache" \ No newline at end of file +CELERY = { + # 'broker_url': f'{env.db("RABBITMQ_URL")}', + 'imports': ('engine.tasks', ), + 'task_serializer': 'json', + 'result_serializer': 'json', + 'accept_content': ['json'], +} +# CELERY_BROKER_URL = env.db("RABBITMQ_URL") +# CELERY_RESULT_BACKEND = "django-db" +# CELERY_CACHE_BACKEND = "django-cache" diff --git a/requirements.txt b/requirements.txt index a73556d..ed3a282 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,10 @@ git+https://github.com/Hypance/HypanceDataAnalysis.git numpy==1.23.3 Django==4.1.2 +# django-celery-beat==2.4.0 django-environ==0.9.0 django-extensions==3.2.1 +# django-celery-results==2.4.0 psycopg2==2.9.4 -celery==5.2.7 \ No newline at end of file +celery==5.2.7 +docker \ No newline at end of file diff --git a/wait-for b/wait-for deleted file mode 100644 index 3edda5d..0000000 --- a/wait-for +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/sh - -# Obtained on 19-11-2019 from https://github.com/eficode/wait-for - -TIMEOUT=15 -QUIET=0 - -echoerr() { - if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi -} - -usage() { - exitcode="$1" - cat << USAGE >&2 -Usage: - $cmdname host:port [-t timeout] [-- command args] - -q | --quiet Do not output any status messages - -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit "$exitcode" -} - -wait_for() { - for i in `seq $TIMEOUT` ; do - nc -z "$HOST" "$PORT" > /dev/null 2>&1 - - result=$? - if [ $result -eq 0 ] ; then - if [ $# -gt 0 ] ; then - exec "$@" - fi - exit 0 - fi - sleep 1 - done - echo "Operation timed out" >&2 - exit 1 -} - -while [ $# -gt 0 ] -do - case "$1" in - *:* ) - HOST=$(printf "%s\n" "$1"| cut -d : -f 1) - PORT=$(printf "%s\n" "$1"| cut -d : -f 2) - shift 1 - ;; - -q | --quiet) - QUIET=1 - shift 1 - ;; - -t) - TIMEOUT="$2" - if [ "$TIMEOUT" = "" ]; then break; fi - shift 2 - ;; - --timeout=*) - TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - break - ;; - --help) - usage 0 - ;; - *) - echoerr "Unknown argument: $1" - usage 1 - ;; - esac -done - -if [ "$HOST" = "" -o "$PORT" = "" ]; then - echoerr "Error: you need to provide a host and port to test." - usage 2 -fi - -wait_for "$@" \ No newline at end of file From 05035bfa707b01703b08061418ad26fd5c31c3a1 Mon Sep 17 00:00:00 2001 From: Toygar Aksoy Date: Sun, 5 Feb 2023 03:05:54 +0300 Subject: [PATCH 3/4] Flower is added --- docker-compose.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e0deafb..b42c891 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,16 +83,16 @@ services: # - app_network - # flower: - # image: zoomeranalytics/flower:0.9.1-4.0.2 - # restart: on-failure - # env_file: *envfile - # ports: - # - "5555:5555" - # depends_on: - # - app_rabbit_broker - # networks: - # - app_network + flower: + image: mher/flower + restart: on-failure + env_file: *envfile + ports: + - "5555:5555" + depends_on: + - rabbit + networks: + - app_network # app_rabbit_broker: # container_name: app_rabbitmq From ae1c8c715180de5b1785d5f94be3fa6faccceee9 Mon Sep 17 00:00:00 2001 From: Toygar Aksoy Date: Wed, 8 Feb 2023 02:33:14 +0300 Subject: [PATCH 4/4] Celery backends and scheduler are installed --- .dockerignore | 39 +++++++++++++ .gitignore | 3 + Dockerfile | 16 ++++- docker-compose.yml | 117 +++++++++++++++---------------------- env.env | 8 --- hyde/engine/tasks.py | 2 +- hyde/hyde/settings.py | 24 +++++--- hyde/routers/db_routers.py | 2 +- requirements.txt | 4 +- 9 files changed, 124 insertions(+), 91 deletions(-) create mode 100644 .dockerignore delete mode 100644 env.env diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b9bf744 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,39 @@ +# Django project +/media/ +/static/ +*.sqlite3 + +# Python and others +__pycache__ +*.pyc +.DS_Store +*.swp +/venv/ +/tmp/ +/.vagrant/ +/Vagrantfile.local +node_modules/ +/npm-debug.log +/.idea/ +.vscode +coverage +.python-version + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg \ No newline at end of file diff --git a/.gitignore b/.gitignore index bb9522c..f3ffebc 100644 --- a/.gitignore +++ b/.gitignore @@ -109,6 +109,9 @@ venv/ ENV/ env.bak/ venv.bak/ +.env_prod +env_dev +env.env # Spyder project settings .spyderproject diff --git a/Dockerfile b/Dockerfile index 6a9d503..5d384b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,25 @@ FROM python:3 + +WORKDIR / + +# environment variables ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 -WORKDIR / +# apt-get update +RUN apt-get update + +# psycog2 dependencies +RUN apt-get -y install libpq-dev python-dev +RUN pip install psycopg2 + +# dependencies COPY requirements.txt ./ +RUN pip install --upgrade pip RUN pip install --no-cache-dir -r requirements.txt RUN rm requirements.txt +# copy project and set workdir COPY . / WORKDIR /hyde \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index b42c891..621d267 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,24 +1,49 @@ -version: '3.2' +version: '3.6' services: app: container_name: app build: ./ image: &hyde hyde - restart: on-failure + restart: always command: sh -c "python manage.py makemigrations && python manage.py migrate && python manage.py runserver 0.0.0.0:8000" env_file: &envfile - - env.env + - env_dev ports: - "8000:8000" volumes: - - ./hyde:/hyde + - ./hyde:/usr/src/hyde depends_on: - rabbit networks: - app_network + app_db: + container_name: app_db + image: postgres:latest + env_file: *envfile + volumes: + - postgres_data:/var/lib/postgresql/data/ + ports: + - "5432:5432" + expose: + - "5432" + networks: + - app_network + + rabbit: + container_name: rabbit + image: rabbitmq:management + # env_file: *envfile + env_file: *envfile + ports: + - "5672:5672" + - 15672:15672 + networks: + - app_network + restart: on-failure + worker: build: . image: *hyde @@ -35,53 +60,22 @@ services: networks: - app_network - rabbit: - container_name: rabbit - image: rabbitmq:management - # env_file: *envfile - environment: - - RABBITMQ_PASS=guest - ports: - - "5672:5672" - - 15672:15672 + beat: + build: . + image: *hyde + restart: on-failure + env_file: *envfile + container_name: beat + command: sh -c "celery -A engine.celery.app beat -l info" + volumes: + - ./hyde:/hyde + depends_on: + - app + - rabbit + - app_db + - worker networks: - app_network - restart: on-failure - - # app: - # container_name: app - # build: ./ - # image: &hyde hyde - # restart: on-failure - # command: [ "python", "manage.py", "runserver", "0.0.0.0:8000" ] - # env_file: - # &envfile - # - env.env - # ports: - # - "8000:8000" - # volumes: - # - ./hyde:/hyde - # depends_on: - # - app_rabbit_broker - # networks: - # - app_network - - # worker_engine: - # build: . - # image: *hyde - # restart: on-failure - # env_file: *envfile - # container_name: worker_engine - # command: sh -c "celery -A engine.celery.app worker -l info" - # volumes: - # - ./hyde:/hyde - # depends_on: - # - app - # - rabbit - # - app_db - # networks: - # - app_network - flower: image: mher/flower @@ -94,26 +88,9 @@ services: networks: - app_network - # app_rabbit_broker: - # container_name: app_rabbitmq - # image: rabbitmq:management - # env_file: *envfile - # ports: - # - "5672:5672" - # - 15672:15672 - # networks: - # - app_network - # restart: on-failure - - app_db: - container_name: app_db - image: postgres:13.1 - environment: - - POSTGRES_USER=app_user - - POSTGRES_PASSWORD=changeme - - POSTGRES_DB=app_db - networks: - - app_network - networks: app_network: + + +volumes: + postgres_data: diff --git a/env.env b/env.env deleted file mode 100644 index 798ca26..0000000 --- a/env.env +++ /dev/null @@ -1,8 +0,0 @@ -RABBITMQ_URL=pyamqp://guest:guest@rabbit:5672// -RABBITMQ_DEFAULT_USER=guest -RABBITMQ_DEFAULT_PASS=guest -POSTGRES_USER=app_user -POSTGRES_PASSWORD=changeme -POSTGRES_DB=app_db -CELERY_BROKER_URL=pyamqp://guest@rabbit// -FLOWER_PORT=5555 \ No newline at end of file diff --git a/hyde/engine/tasks.py b/hyde/engine/tasks.py index c72b76d..c270333 100644 --- a/hyde/engine/tasks.py +++ b/hyde/engine/tasks.py @@ -4,7 +4,7 @@ # Create your tasks here -@app.task +@shared_task def add(x, y): return x + y diff --git a/hyde/hyde/settings.py b/hyde/hyde/settings.py index ab07264..c161bf2 100644 --- a/hyde/hyde/settings.py +++ b/hyde/hyde/settings.py @@ -20,7 +20,7 @@ env = environ.Env() # Get environment variables from .env file -env.read_env(str(BASE_DIR / ".env")) +env.read_env(str(BASE_DIR.parent / "env_dev")) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/ @@ -43,8 +43,8 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', - # 'django-celery-beat', - # 'django_celery_results', + 'django_celery_beat', + 'django_celery_results', 'django_extensions', 'engine', ] @@ -84,8 +84,17 @@ # https://docs.djangoproject.com/en/4.1/ref/settings/#databases DATABASES = { - "default": {'ENGINE':'django.db.backends.sqlite3', 'NAME':BASE_DIR / 'db.sqlite3'}, + # "default": {'ENGINE':'django.db.backends.sqlite3', 'NAME':BASE_DIR / 'db.sqlite3'}, + # "default": env.db("DEFAULT_URL"), "hypance_db": env.db("ENGINE_URL"), + "default": { + "ENGINE": env("SQL_ENGINE"), + "NAME": env("SQL_DATABASE"), + "USER": env("SQL_USER"), + "PASSWORD": env("SQL_PASSWORD"), + "HOST": env("SQL_HOST"), + "PORT": env("SQL_PORT"), + } } DATABASE_ROUTERS = [ @@ -136,12 +145,11 @@ # Celery settings CELERY = { - # 'broker_url': f'{env.db("RABBITMQ_URL")}', 'imports': ('engine.tasks', ), 'task_serializer': 'json', 'result_serializer': 'json', 'accept_content': ['json'], + 'result_backend': env("CELERY_RESULTS_BACKEND_URL"), + 'beat_scheduler': 'django_celery_beat.schedulers:DatabaseScheduler', + # 'cache_backend': env("CELERY_CACHE_BACKEND_URL"), } -# CELERY_BROKER_URL = env.db("RABBITMQ_URL") -# CELERY_RESULT_BACKEND = "django-db" -# CELERY_CACHE_BACKEND = "django-cache" diff --git a/hyde/routers/db_routers.py b/hyde/routers/db_routers.py index 014367e..82b5b87 100644 --- a/hyde/routers/db_routers.py +++ b/hyde/routers/db_routers.py @@ -1,5 +1,5 @@ class DefaultRouter: - route_app_labels = {'auth', 'contenttypes', 'sessions', 'admin'} + route_app_labels = {'auth', 'contenttypes', 'sessions', 'admin', 'django_celery_beat', 'django_celery_results'} def db_for_read(self, model, **hints): if model._meta.app_label in self.route_app_labels: diff --git a/requirements.txt b/requirements.txt index ed3a282..8e9aa96 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ git+https://github.com/Hypance/HypanceDataAnalysis.git numpy==1.23.3 Django==4.1.2 -# django-celery-beat==2.4.0 +django-celery-beat==2.4.0 django-environ==0.9.0 django-extensions==3.2.1 -# django-celery-results==2.4.0 +django-celery-results==2.4.0 psycopg2==2.9.4 celery==5.2.7 docker \ No newline at end of file