Major Codebase Refactoring and Search Enhancement

.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: Django CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements_django_5.txt
+
+    - name: Run addcorpora.sh
+      run: |
+        chmod +x ./addcorpora.sh
+        ./addcorpora.sh
+
+    - name: Run tests
+      run: python manage.py test -t .

.upsun/config.yaml

@@ -0,0 +1,205 @@
+# Complete list of all available properties: https://docs.upsun.com/create-apps/app-reference.html
+applications:
+  search:
+    source:
+      root: "/meili_empty_root"
+    container_profile: HIGH_MEMORY
+    stack: [ "meilisearch" ]
+    hooks:
+      build: |
+        wget https://github.com/meilisearch/meilisearch/releases/download/v1.12.8/meilisearch-linux-amd64
+        mv meilisearch-linux-amd64 meilisearch
+        chmod +x meilisearch 
+    web:
+      # Commands are run once after deployment to start the application process.
+      # More information: https://docs.upsun.com/create-apps/app-reference.html#web-commands
+      commands:
+        # The command to launch your app. If it terminates, it’s restarted immediately.
+        # You can use the $PORT or the $SOCKET environment variable depending on the socket family of your upstream
+        start: "./meilisearch --master-key=$PLATFORM_PROJECT_ENTROPY --http-addr=localhost:$PORT"
+      # You can listen to a UNIX socket (unix) or a TCP port (tcp, default).
+      # Whether your app should speak to the webserver via TCP or Unix socket. Defaults to tcp
+      # More information: https://docs.upsun.com/create-apps/app-reference.html#where-to-listen
+      upstream:
+        # Whether your app should speak to the webserver via TCP or Unix socket. Defaults to tcp
+        # More information: https://docs.upsun.com/create-apps/app-reference.html#where-to-listen
+        socket_family: tcp
+      # Each key in locations is a path on your site with a leading /.
+      # More information: https://docs.upsun.com/create-apps/app-reference.html#locations
+      locations:
+        "/":
+            passthru: true
+    mounts:
+      "/data.ms":
+        source: "storage"
+        source_path: "data.ms"
+      "/dumps":
+        source: "storage"
+        source_path: "dumps"
+
+  cts:
+    # Application source code directory
+    source:
+      root: "/coptic"
+    container_profile: HIGHER_MEMORY
+    # The runtime the application uses.
+    # Complete list of available runtimes: https://docs.upsun.com/create-apps/app-reference.html#types
+    type: "python:3.12"
+
+    # Choose which container profile (ratio CPU+RAM) your app will use. Default value comes from the image itself.
+    # More information: https://docs.upsun.com/manage-resources/adjust-resources.html#adjust-a-container-profile
+    # container_profile:
+
+    # The relationships of the application with services or other applications.
+    # The left-hand side is the name of the relationship as it will be exposed
+    # to the application in the PLATFORM_RELATIONSHIPS variable. The right-hand
+    # side is in the form `<service name>:<endpoint name>`.
+    # More information: https://docs.upsun.com/create-apps/app-reference.html#relationships
+
+    relationships:
+       search: "search:http"
+
+    # Mounts define directories that are writable after the build is complete.
+    # More information: https://docs.upsun.com/create-apps/app-reference.html#mounts
+    mounts:
+       "/db": # Represents the path in the app.
+         source: "storage" # "storage" sources are unique to the app, but shared among instances of the app. "service" sources can be shared among apps.
+         source_path: "db" # The subdirectory within the mounted disk (the source) where the mount should point.
+       "/corpora": # Represents the path in the app.
+         source: "storage" # "storage" sources are unique to the app, but shared among instances of the app. "service" sources can be shared among apps.
+         source_path: "corpora" # The subdirectory within the mounted disk (the source) where the mount should point.
+    # The web key configures the web server running in front of your app.
+    # More information: https://docs.upsun.com/create-apps/app-reference.html#web
+    web:
+      # Commands are run once after deployment to start the application process.
+      # More information: https://docs.upsun.com/create-apps/app-reference.html#web-commands
+      commands:
+        # The command to launch your app. If it terminates, it’s restarted immediately.
+        # You can use the $PORT or the $SOCKET environment variable depending on the socket family of your upstream
+        start: ". $HOME/.local/bin/env;uv run gunicorn --workers 3 --bind unix:$SOCKET coptic.wsgi:application"
+      # You can listen to a UNIX socket (unix) or a TCP port (tcp, default).
+      # Whether your app should speak to the webserver via TCP or Unix socket. Defaults to tcp
+      # More information: https://docs.upsun.com/create-apps/app-reference.html#where-to-listen
+      upstream:
+        # Whether your app should speak to the webserver via TCP or Unix socket. Defaults to tcp
+        # More information: https://docs.upsun.com/create-apps/app-reference.html#where-to-listen
+        socket_family: unix
+      # Each key in locations is a path on your site with a leading /.
+      # More information: https://docs.upsun.com/create-apps/app-reference.html#locations
+      locations:
+        "/":
+            passthru: true
+        "/static":
+            "allow": true
+            "expires": "1h"
+            "root": "static"
+
+
+    # Alternate copies of the application to run as background processes.
+    # More information: https://docs.upsun.com/create-apps/app-reference.html#workers
+    # workers:
+
+    # The timezone for crons to run. Format: a TZ database name. Defaults to UTC, which is the timezone used for all logs
+    # no matter the value here. More information: https://docs.upsun.com/create-apps/timezone.html
+    # timezone: <time-zone>
+
+    # Access control for roles accessing app environments.
+    # More information: https://docs.upsun.com/create-apps/app-reference.html#access
+    # access:
+
+    # Variables to control the environment. More information: https://docs.upsun.com/create-apps/app-reference.html#variables
+    # variables:
+    #   env:
+    #     # Add environment variables here that are static.
+    #     PYTHONUNBUFFERED: "1"
+
+    # Outbound firewall rules for the application. More information: https://docs.upsun.com/create-apps/app-reference.html#firewall
+    # firewall:
+
+    # Specifies a default set of build tasks to run. Flavors are language-specific.
+    # More information: https://docs.upsun.com/create-apps/app-reference.html#build
+    build:
+      flavor: none
+
+    # Installs global dependencies as part of the build process. They’re independent of your app’s dependencies and
+    # are available in the PATH during the build process and in the runtime environment. They’re installed before
+    # the build hook runs using a package manager for the language.
+    # More information: https://docs.upsun.com/create-apps/app-reference.html#dependencies
+    # dependencies:
+    #   python3: # Specify one Python 3 package per line.
+    #     numpy: '*'
+
+    # Hooks allow you to customize your code/environment as the project moves through the build and deploy stages
+    # More information: https://docs.upsun.com/create-apps/app-reference.html#hooks
+    hooks:
+      # The build hook is run after any build flavor.
+      # More information: https://docs.upsun.com/create-apps/hooks/hooks-comparison.html#build-hook
+      build: |
+        set -eux
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        . $HOME/.local/bin/env  
+        uv sync
+        #./manage.py collectstatic --noinput
+
+      # The deploy hook is run after the app container has been started, but before it has started accepting requests.
+      # More information: https://docs.upsun.com/create-apps/hooks/hooks-comparison.html#deploy-hook
+      deploy: |
+        set -eux
+        # echo 'Put your deploy command here'
+        uvm migrate
+
+      # The post_deploy hook is run after the app container has been started and after it has started accepting requests.
+      # More information: https://docs.upsun.com/create-apps/hooks/hooks-comparison.html#deploy-hook
+      post_deploy: |
+        uvm clearcache
+    operations:
+      import:
+        role: admin
+        commands:
+          start: uv run ./addcorpora.sh corpora
+      index:
+        role: admin
+        commands:
+          start: uvm index_corpora
+      clearcache:
+        role: admin
+        commands:
+          start: uvm clearcache
+    # Scheduled tasks for the app.
+    # More information: https://docs.upsun.com/create-apps/app-reference.html#crons
+    # crons:
+
+    # Customizations to your PHP or Lisp runtime. More information: https://docs.upsun.com/create-apps/app-reference.html#runtime
+    # runtime:
+
+    # More information: https://docs.upsun.com/create-apps/app-reference.html#additional-hosts
+    # additional_hosts:
+
+# The services of the project.
+#
+# Each service listed will be deployed
+# to power your Upsun project.
+# More information: https://docs.upsun.com/add-services.html
+# Full list of available services: https://docs.upsun.com/add-services.html#available-services
+# services:
+#   db:
+#     type: postgresql:14
+
+
+# The routes of the project.
+#
+# Each route describes how an incoming URL is going
+# to be processed by Upsun.
+# More information: https://docs.upsun.com/define-routes.html
+routes:
+  "https://{default}/":
+    type: upstream
+    upstream: "cts:http"
+  "https://search.{default}/":
+    type: upstream
+    upstream: "search:http"
+  # A basic redirect definition
+  # More information: https://docs.upsun.com/define-routes.html#basic-redirect-definition
+  "https://www.{default}":
+    type: redirect
+    to: "https://{default}/"