From a1cd2651277ba7881ee6865d9494215213c24edd Mon Sep 17 00:00:00 2001 From: Ori Pekelman Date: Thu, 14 Nov 2024 15:24:49 +0100 Subject: [PATCH 1/4] freeze correct libs --- coptic/requirements.txt | 16 ++++++++-------- coptic/requirements_django_2.txt | 9 --------- 2 files changed, 8 insertions(+), 17 deletions(-) delete mode 100644 coptic/requirements_django_2.txt diff --git a/coptic/requirements.txt b/coptic/requirements.txt index 751c34ca..1de7b697 100644 --- a/coptic/requirements.txt +++ b/coptic/requirements.txt @@ -1,10 +1,10 @@ requests -django -django-grappelli -beautifulsoup4 -selenium==2.45.0 -xvfbwrapper==0.2.4 -mod-wsgi==4.4.11 -celery==3.1.18 +django==2.2.13 +mysqlclient +django-grappelli==2.13.1 +beautifulsoup4==4.8.0 +selenium==3.141.0 +xvfbwrapper==0.2.9 +mod-wsgi +celery github3.py==1.3.0 -tqdm diff --git a/coptic/requirements_django_2.txt b/coptic/requirements_django_2.txt deleted file mode 100644 index 4768448e..00000000 --- a/coptic/requirements_django_2.txt +++ /dev/null @@ -1,9 +0,0 @@ -requests -django==2.2.13 -django-grappelli==2.13.1 -beautifulsoup4==4.8.0 -selenium==3.141.0 -xvfbwrapper==0.2.9 -mod-wsgi -celery==4.3.0 -github3.py==1.3.0 From 6f3a713750fd07994396cf25ebdd2e05927e61a7 Mon Sep 17 00:00:00 2001 From: Ori Pekelman Date: Thu, 14 Nov 2024 15:26:57 +0100 Subject: [PATCH 2/4] Add docker configuration --- .vscode/launch.json | 21 +++++++++ .vscode/tasks.json | 17 +++++++ coptic/.dockerignore | 27 +++++++++++ coptic/Dockerfile | 42 +++++++++++++++++ coptic/README_Docker.md | 9 ++++ coptic/coptic/settings/docker.py | 78 ++++++++++++++++++++++++++++++++ docker-compose.yml | 29 ++++++++++++ 7 files changed, 223 insertions(+) create mode 100644 .vscode/launch.json create mode 100644 .vscode/tasks.json create mode 100644 coptic/.dockerignore create mode 100644 coptic/Dockerfile create mode 100644 coptic/README_Docker.md create mode 100644 coptic/coptic/settings/docker.py create mode 100644 docker-compose.yml diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..201598f0 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,21 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Remote Attach", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "/var/www/cts" + } + ], + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 00000000..13759f92 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,17 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Docker Compose Up", + "type": "shell", + "command": "docker-compose up --build", + "problemMatcher": [] + }, + { + "label": "Docker Compose Down", + "type": "shell", + "command": "docker-compose down", + "problemMatcher": [] + } + ] +} \ No newline at end of file diff --git a/coptic/.dockerignore b/coptic/.dockerignore new file mode 100644 index 00000000..0b1e1e7e --- /dev/null +++ b/coptic/.dockerignore @@ -0,0 +1,27 @@ +**/__pycache__ +**/.venv +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/bin +**/charts +**/docker-compose* +**/compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +LICENSE +README.md diff --git a/coptic/Dockerfile b/coptic/Dockerfile new file mode 100644 index 00000000..0345c4f0 --- /dev/null +++ b/coptic/Dockerfile @@ -0,0 +1,42 @@ +# Base image +FROM ubuntu:20.04 + +# Environment settings +ENV DEBIAN_FRONTEND=noninteractive + +# Install key packages +RUN apt-get update && apt-get install -y \ + apache2 \ + apache2-dev \ + chromium-browser \ + git \ + libapache2-mod-wsgi-py3 \ + default-libmysqlclient-dev \ + build-essential \ + pkg-config \ + python3-pip \ + redis-server \ + redis-tools \ + unzip \ + xvfb \ + && apt-get clean + +# Install Python dependencies +COPY coptic/requirements.txt /var/www/cts/coptic/requirements.txt +RUN pip3 install -r /var/www/cts/coptic/requirements.txt +RUN pip3 install debugpy + +# Copy application files +WORKDIR /var/www/cts/coptic +COPY ./coptic /var/www/cts/coptic +RUN mkdir -p /var/www/cts/coptic/scripts +COPY ./scripts/ /var/www/cts/coptic/scripts + +# Apache configuration +COPY ./ansible/roles/scriptorium/files/scriptorium.conf /etc/apache2/sites-available/000-default.conf + +EXPOSE 80 +# Environment variables for Django +ENV DJANGO_SETTINGS_MODULE="coptic.settings.docker" +# Command to start Apache +CMD ["python3", "-m", "debugpy", "--listen", "0.0.0.0:5678", "manage.py", "runserver", "0.0.0.0:8000"] \ No newline at end of file diff --git a/coptic/README_Docker.md b/coptic/README_Docker.md new file mode 100644 index 00000000..3975c4e5 --- /dev/null +++ b/coptic/README_Docker.md @@ -0,0 +1,9 @@ +# Developing with visual studio code + +``` +export SECRET_KEY="what you want" +export GITHUB_TOKEN="what you need" +docker compose build +docker compose up +code . +``` \ No newline at end of file diff --git a/coptic/coptic/settings/docker.py b/coptic/coptic/settings/docker.py new file mode 100644 index 00000000..3cb61a35 --- /dev/null +++ b/coptic/coptic/settings/docker.py @@ -0,0 +1,78 @@ +import os +from pathlib import Path +SECRET_KEY="what you want" # this is local development only + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent.parent + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ + + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = ['*'] + +# Application definition +INSTALLED_APPS = ( + 'grappelli', + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'texts', + #'annis', + #'ingest', + 'api', + 'mod_wsgi.server' +) + +MIDDLEWARE = ( + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +) + +ROOT_URLCONF = 'coptic.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [os.path.join(BASE_DIR, 'templates')], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'coptic.wsgi.application' + +# Database +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': os.environ.get('MYSQL_DB', 'coptic'), + 'USER': os.environ.get('MYSQL_USER', 'coptic'), + 'PASSWORD': os.environ.get('MYSQL_PASSWORD', ''), + 'HOST': os.environ.get('MYSQL_HOST', 'db'), + 'PORT': os.environ.get('MYSQL_PORT', '3306'), + } +} + +# Static files (CSS, JavaScript, Images) +STATIC_URL = '/static/' +STATIC_ROOT = "/var/www/cts/coptic/static/" + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..1c0e99a8 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,29 @@ +services: + web: + build: + context: . + dockerfile: coptic/Dockerfile + environment: + - DJANGO_SETTINGS_MODULE=coptic.settings.docker + - MYSQL_HOST=db + - MYSQL_USER=coptic + - MYSQL_PASSWORD=your_mysql_password + - MYSQL_DB=coptic + command: ["python3", "-m", "debugpy", "--listen", "0.0.0.0:5678", "manage.py", "runserver", "0.0.0.0:8000"] + volumes: + - .:/var/www/cts + ports: + - "8000:8000" + - "5678:5678" + depends_on: + - db + db: + image: mariadb:10.2 + restart: always + environment: + MYSQL_ROOT_PASSWORD: root_password + MYSQL_DATABASE: coptic + MYSQL_USER: coptic + MYSQL_PASSWORD: your_mysql_password + ports: + - "3306:3306" \ No newline at end of file From d152f6a1e8d31ffbdb2cc6b9ae268642b6fd139e Mon Sep 17 00:00:00 2001 From: Ori Pekelman Date: Thu, 14 Nov 2024 16:19:53 +0100 Subject: [PATCH 3/4] implement minimal caching --- coptic/coptic/settings/docker.py | 14 ++++++++++--- coptic/coptic/views.py | 34 ++++++++++++++++---------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/coptic/coptic/settings/docker.py b/coptic/coptic/settings/docker.py index 3cb61a35..46a75228 100644 --- a/coptic/coptic/settings/docker.py +++ b/coptic/coptic/settings/docker.py @@ -30,7 +30,7 @@ 'mod_wsgi.server' ) -MIDDLEWARE = ( +MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -38,7 +38,9 @@ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', -) + 'django.middleware.cache.UpdateCacheMiddleware', + 'django.middleware.cache.FetchFromCacheMiddleware', +] ROOT_URLCONF = 'coptic.urls' @@ -71,7 +73,13 @@ 'PORT': os.environ.get('MYSQL_PORT', '3306'), } } - +# Cache configuration +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + 'LOCATION': os.path.join(BASE_DIR, 'cache'), + } +} # Static files (CSS, JavaScript, Images) STATIC_URL = '/static/' STATIC_ROOT = "/var/www/cts/coptic/static/" diff --git a/coptic/coptic/views.py b/coptic/coptic/views.py index f2778e45..1375bda4 100644 --- a/coptic/coptic/views.py +++ b/coptic/coptic/views.py @@ -5,6 +5,8 @@ from django.db.models import Q, Case, When, IntegerField, F from django.shortcuts import render, get_object_or_404, redirect from django.db.models.functions import Lower +from django.views.decorators.cache import cache_page +from django.core.cache import cache from texts.search_fields import get_search_fields from coptic.settings.base import DEPRECATED_URNS from collections import OrderedDict @@ -19,12 +21,13 @@ def keyvalue(dict, key): return dict.get(key) +@cache_page(60 * 15) def home_view(request): 'Home' context = _base_context() return render(request, 'home.html', context) - +@cache_page(60 * 15) def corpus_view(request, corpus=None): corpus_object = get_object_or_404(models.Corpus, slug=corpus) @@ -64,7 +67,7 @@ def corpus_view(request, corpus=None): }) return render(request, 'corpus.html', context) - +@cache_page(60 * 15) def text_view(request, corpus=None, text=None, format=None): text_object = get_object_or_404(models.Text, slug=text) if not format: @@ -107,11 +110,10 @@ def text_view(request, corpus=None, text=None, format=None): }) return render(request, 'text.html', context) - +@cache_page(60 * 15) def not_found(request): return render(request, '404.html', {}) - def _resolve_urn(urn): try: text = models.Text.objects.get(text_meta__name="document_cts_urn", text_meta__value=urn) @@ -123,7 +125,7 @@ def _resolve_urn(urn): except models.Corpus.DoesNotExist: return None - +@cache_page(60 * 15) def urn(request, urn=None): # https://github.com/CopticScriptorium/cts/issues/112 if re.match(r'urn:cts:copticLit:ot.*.crosswire', urn): @@ -139,7 +141,6 @@ def urn(request, urn=None): return redirect('corpus', corpus=obj.slug) return redirect(reverse('search') + f"?text={urn}") - def get_meta_values(meta): unsplit_values = map(lambda x: x['value'], models.TextMeta.objects.filter(name__iexact=meta.name).values("value").distinct()) if not meta.splittable: @@ -157,7 +158,7 @@ def get_meta_values(meta): meta_values = [re.sub(HTML_TAG_REGEX, '', meta_value) for meta_value in meta_values] return meta_values - +@cache_page(60 * 15) def index_view(request, special_meta=None): context = _base_context() @@ -299,7 +300,6 @@ def _fetch_and_filter_texts_for_special_metadata_query(queries): add_author_and_urn(texts) return texts - def _build_explanation(params): meta_explanations = [] for meta_name, meta_values in params.items(): @@ -326,7 +326,6 @@ def _build_explanation(params): meta_explanations.append("(" + " OR ".join(meta_name_explanations) + ")") return " AND ".join(meta_explanations) - def _build_result_for_query_text(params, texts, explanation): query_text = params["text"] results = [] @@ -351,16 +350,18 @@ def _build_result_for_query_text(params, texts, explanation): all_empty_explanation += explanation return results, all_empty_explanation - def _base_context(): - search_fields = get_search_fields() - context = { - 'search_fields': search_fields[:5], - 'secondary_search_fields': search_fields[5:] - } + context = cache.get('base_context') + if not context: + search_fields = get_search_fields() + context = { + 'search_fields': search_fields[:5], + 'secondary_search_fields': search_fields[5:] + } + cache.set('base_context', context, 60 * 15) # Cache for 15 minutes return context - +@cache_page(60 * 15) def search(request): context = _base_context() @@ -418,7 +419,6 @@ def search(request): return render(request, 'search.html', context) - def add_author_and_urn(texts): for text in texts: try: From 9bec32716abc77a0e2143235fcdbc9bda65f5293 Mon Sep 17 00:00:00 2001 From: Ori Pekelman Date: Tue, 19 Nov 2024 15:01:42 +0100 Subject: [PATCH 4/4] Add cach dir to gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3328fb7d..5a552b71 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,7 @@ __pycache__ */django_logger.log **/secrets.py **.pyc - +coptic/cache/ # -------------------- # Env Files # --------------------