Use TimescaleDB as database for optimized time series queries (#39)

* Use timescaledb to query mystrom results

* Fix insert in minutely update request, fix calculation of total kwh

by taking the fractions of the hour of first/last date in query

* Use timescaledb features in shelly results

* Format code with ruff

* Mention that project only supports timescaledb as database

* Add database in github actions service to test

* Fix tests

* Revert development changes

* Increase max requests

* Remove default logging level

* Remove unneeded package from builder stage

* Fix total power calculation query on partial hours

* Use logger instead of print

* use django 5.2 composite key

* add ruff check workflow

* remove mariadb connector since only timescaledb is supported

Author: maexled

.dockerignore

@@ -0,0 +1,6 @@
+db-data/
+docker-compose.yml
+Dockerfile
+.env
+.git/
+.gitignore

.env

@@ -1,12 +1,11 @@
-ENGINE_TYPE=postgresql
-DB_HOST=db
+DB_HOST=timescaledb
 DB_PORT=5432
 DB_NAME=postgres
 DB_USER=postgres
 DB_PASSWORD=postgres
 
 SECRET_KEY=very-secret-key
-ALLOWED_HOSTS=localhost,example.com,interface
+ALLOWED_HOSTS=localhost,example.com,interface,192.168.0.19
 CORS_ORIGIN_ALLOW_ALL=False
 CORS_ORIGIN_WHITELIST=http://example.com,http://localhost,http://interface:8000
 CHART_TYPE=uplot

.github/workflows/ruff-format-check.yml

@@ -0,0 +1,31 @@
+name: Ruff Format Check
+
+on:
+  push:
+    paths:
+      - '**.py'
+  pull_request:
+    paths:
+      - '**.py'
+
+jobs:
+  ruff-format:
+    name: Check code formatting with Ruff
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: Install Ruff
+        run: |
+          pip install ruff
+
+      - name: Run ruff format check
+        run: |
+          ruff format --check .

.github/workflows/test.yml

@@ -16,6 +16,21 @@ jobs:
       matrix:
         python-version: ['3.10', '3.11', '3.12']
 
+    services:
+      timescaledb:
+        image: timescale/timescaledb:latest-pg16
+        env:
+          POSTGRES_PASSWORD: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
@@ -30,3 +45,9 @@ jobs:
       run: |
         python manage.py collectstatic --noinput
         python manage.py test
+      env:
+        DB_NAME: postgres
+        DB_USER: postgres
+        DB_PASSWORD: postgres
+        DB_HOST: localhost
+        DB_PORT: 5432

Dockerfile

@@ -5,7 +5,7 @@ WORKDIR /app
 
 COPY requirements.txt .
 
-RUN apk add --no-cache --virtual .build-deps gcc musl-dev mariadb-connector-c-dev libpq-dev
+RUN apk add --no-cache --virtual .build-deps gcc musl-dev libpq-dev
 
 RUN pip install --no-cache-dir --upgrade pip && \
     pip install --no-cache-dir gunicorn && \
@@ -20,10 +20,11 @@ COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/pytho
 COPY --from=builder /usr/local/bin/gunicorn /usr/local/bin/gunicorn
 COPY --from=builder /app /app
 
-RUN apk add --no-cache mariadb-connector-c libpq
+RUN apk add --no-cache libpq
 
 COPY . .
 
-CMD ["sh", "-c", "python manage.py makemigrations && python manage.py migrate && python manage.py collectstatic --noinput && gunicorn --bind=0.0.0.0:8000 --timeout 300 --workers=3 --threads=3 --max-requests 5 --max-requests-jitter 2 pim.wsgi:application"]
+
+CMD ["sh", "-c", "python manage.py migrate && python manage.py collectstatic --noinput && gunicorn --bind=0.0.0.0:8000 --timeout 300 --workers=3 --threads=3 --max-requests 20 --max-requests-jitter 5 pim.wsgi:application"]
 
 EXPOSE 8000/tcp

README.md

@@ -2,6 +2,9 @@
 
 The Django application is designed to allow users to view power usage charts for MyStrom and Shelly3EM devices. The application collects the current power usage data every 60 seconds, which is then used to calculate and present power usage charts to the user.
 
+This project uses [TimeScaleDB](https://www.timescale.com/) as database. 
+The database is optimized for time-series data and this project uses queries which are only compatible with TimeScaleDB.
+
 There are two main views: the Device view and the Result view. Here's what each view looks like:
 
 #### Device View
@@ -37,7 +40,6 @@ pip install -r requirements.txt
 
 Here's a breakdown of all the environment variables that are being used in the Django application:
 
-- `ENGINE_TYPE`: Specifies the type of database engine to use, either `mysql` or `postgresql`.
 - `DB_NAME`: Specifies the name of the database to use.
 - `DB_USER`: Specifies the username to use when connecting to the database.
 - `DB_PASSWORD`: Specifies the password to use when connecting to the database.
@@ -87,7 +89,6 @@ Static files are served with WhiteNoise.
 docker run \
    --name mystrom-interface \
    -p 8000:8000 \
-   -e ENGINE_TYPE={mysql/postgresql}
    -e DB_NAME=db-name \
    -e DB_USER=username \
    -e DB_PASSWORD=password \
@@ -107,7 +108,3 @@ Make sure to replace the content in `{...}` with your variable of your choice.
 ```sh
 docker compose up
 ```
-
-## Configurations
-### Important things to know
-Database configuraton in `pim/settings.py` under `DATABASES`

docker-compose.yml

@@ -1,19 +1,42 @@
+
 services:
   interface:
     container_name: interface
     image: ghcr.io/maexled/mystrom-django-interface:master
-#   build: .
+    # build: .
     privileged: true
     restart: always
     ports:
       - "80:8000"
     env_file:
       - .env
+    depends_on:
+      - timescaledb
+
+  timescaledb:
+    image: timescale/timescaledb:latest-pg16
+    environment:
+      POSTGRES_DB: postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+    volumes:
+      - ./db-data:/var/lib/postgresql/data
+
+  pgadmin:
+    image: dpage/pgadmin4
+    environment:
+      PGADMIN_DEFAULT_EMAIL: admin@admin.com
+      PGADMIN_DEFAULT_PASSWORD: admin
+    ports:
+      - "8080:80"
+    depends_on:
+      - timescaledb
 
   requester:
     container_name: interface-requester
     image: curlimages/curl:7.80.0
-    command: ["sh", "-c", "while true; do sleep 60; curl -X POST http://interface:8000/shelly-api/devices/request-and-save-results >/dev/null 2>&1 & curl -X POST http://interface:8000/api/devices/request-and-save-results >/dev/null 2>&1; done"]    
+    command: ["sh", "-c", "while true; do sleep 60; curl -fsS -X POST http://interface:8000/shelly-api/devices/request-and-save-results && curl -fsS -X POST http://interface:8000/api/devices/request-and-save-results; done"]
     depends_on:
       - interface
+    
 

interface/admin.py

@@ -1,3 +1 @@
-from django.contrib import admin
-
 # Register your models here.

interface/apps.py

@@ -2,5 +2,5 @@
 
 
 class MystromConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'interface'
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "interface"

interface/forms.py

@@ -1,32 +1,24 @@
 from django import forms
 from mystrom_rest.models import MystromDevice
 from shelly3em_rest.models import Shelly3EMDevice
- 
+
+
 # creating a form
 class MystromDeviceForm(forms.ModelForm):
- 
     # create meta class
     class Meta:
         # specify model to be used
         model = MystromDevice
- 
+
         # specify fields to be used
-        fields = [
-            "name",
-            "ip",
-            "active"
-        ]
+        fields = ["name", "ip", "active"]
+
 
 class Shelly3EMDeviceForm(forms.ModelForm):
- 
     # create meta class
     class Meta:
         # specify model to be used
         model = Shelly3EMDevice
- 
+
         # specify fields to be used
-        fields = [
-            "name",
-            "ip",
-            "active"
-        ]
+        fields = ["name", "ip", "active"]