Add backend support for PostgreSQL

amotl · amotl · commit bc0e8566089a · 2024-01-28T19:30:14.000+01:00
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -1,10 +1,17 @@
-name: Run CI
+name: Tests
 
 on:
+  pull_request: ~
   push:
-  pull_request:
+    branches: [ main ]
+
+  # Allow job to be triggered manually.
   workflow_dispatch:
 
+  # Run job each night.
+  schedule:
+    - cron: '0 3 * * *'
+
 # Cancel in-progress jobs when pushing to the same branch.
 concurrency:
   cancel-in-progress: true
@@ -30,6 +37,13 @@ jobs:
       max-parallel: 4
       matrix:
         python-version: [3.7, 3.8, 3.9, '3.10', '3.11']
+    services:
+      postgresql:
+        image: postgres:16
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
@@ -59,6 +73,13 @@ jobs:
           ls -l dist
   documentation:
     runs-on: ubuntu-latest
+    services:
+      postgresql:
+        image: postgres:16
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
     steps:
       - uses: actions/checkout@v3
       - name: Setup python
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,4 +1,4 @@
-name: Release new version
+name: Release
 
 on:
   push:
diff --git a/.gitignore b/.gitignore
@@ -113,6 +113,9 @@ ENV/
 env.bak/
 venv.bak/
 
+# PyCharm project settings
+.idea
+
 # Spyder project settings
 .spyderproject
 .spyproject
diff --git a/README.rst b/README.rst
@@ -79,6 +79,23 @@ Usage
   Please access http://localhost:8000/search.html
 
 
+Development
+===========
+
+Install package in development mode::
+
+    pip install --editable='.[cli,docs,test]' --prefer-binary
+
+Start PostgreSQL server::
+
+    docker run --rm -it --publish=5432:5432 --env "POSTGRES_HOST_AUTH_METHOD=trust" postgres:16 postgres -c log_statement=all
+
+Invoke software tests::
+
+    export POSTGRES_LOG_STATEMENT=all
+    pytest -vvv
+
+
 .. _atsphinx-sqlite3fts: https://pypi.org/project/atsphinx-sqlite3fts/
 .. _Kazuya Takei: https://github.com/attakei
 .. _readthedocs-sphinx-search: https://github.com/readthedocs/readthedocs-sphinx-search
diff --git a/docs/_static/.gitkeep b/docs/_static/.gitkeep
diff --git a/docs/conf.py b/docs/conf.py
@@ -32,6 +32,7 @@
 }
 # atsphinx-sqlite3fts
 sqlite3fts_use_search_html = True
+sqlite3fts_database_url = "postgresql://postgres@localhost:5432"
 
 
 def setup(app):  # noqa: D103
diff --git a/docs/getting-started.rst b/docs/getting-started.rst
@@ -47,8 +47,11 @@ You can build database by ``sqlite`` builder.
 
 .. code-block:: console
 
-   make sqlite
-   sqlite3 _build/sqlite/db.sqlite
+   make fts-index
+
+.. code-block:: console
+
+   psql postgresql://postgres@localhost:5432/ --command 'SELECT * FROM document;'
 
 .. code-block:: sqlite3
 
diff --git a/pyproject.toml b/pyproject.toml
@@ -35,6 +35,7 @@ dynamic = ["version", "description"]
 dependencies = [
   "docutils",
   "peewee",
+  "psycopg2[binary]",
   "Sphinx",
 ]
 
diff --git a/src/atsphinx/sqlite3fts/__init__.py b/src/atsphinx/sqlite3fts/__init__.py
@@ -1,4 +1,4 @@
-"""Sphinx document searcher using SQLite3."""
+"""Sphinx document searcher using SQL database."""
 from sphinx.application import Sphinx
 
 from . import builders, events
@@ -10,9 +10,10 @@ def setup(app: Sphinx):
     """Entrypoint as Sphinx extension."""
     app.add_config_value("sqlite3fts_exclude_pages", [], "env")
     app.add_config_value("sqlite3fts_use_search_html", False, "env")
-    app.add_builder(builders.SqliteBuilder)
+    app.add_config_value("sqlite3fts_database_url", None, "env")
+    app.add_builder(builders.FtsIndexer)
     app.connect("config-inited", events.setup_search_html)
-    app.connect("builder-inited", events.configure_database)
+    app.connect("config-inited", events.configure_database)
     app.connect("html-page-context", events.register_document)
     app.connect("build-finished", events.save_database)
     return {
diff --git a/src/atsphinx/sqlite3fts/builders.py b/src/atsphinx/sqlite3fts/builders.py
@@ -7,13 +7,13 @@
 from . import models, services
 
 
-class SqliteBuilder(Builder):
+class FtsIndexer(Builder):
     """Single database generation builder.
 
     This is custom builder to generate only SQLite database file
     """
 
-    name = "sqlite"
+    name = "fts-index"
     allow_parallel = True
 
     def get_target_uri(self, docname: str, typ: str = None) -> str:  # noqa: D102
@@ -23,7 +23,11 @@ def get_outdated_docs(self) -> str:  # noqa: D102
         return "db.sqlite"
 
     def prepare_writing(self, docnames: Set[str]) -> None:  # noqa: D102
-        pass
+        from atsphinx.sqlite3fts.models import Content, Document, Section
+
+        Document.truncate_table(cascade=True)
+        Section.truncate_table(cascade=True)
+        Content.truncate_table(cascade=True)
 
     def write_doc(self, docname: str, doctree: nodes.document) -> None:
         """Register content of document into database.
diff --git a/src/atsphinx/sqlite3fts/events.py b/src/atsphinx/sqlite3fts/events.py
@@ -33,12 +33,23 @@ def _generate_search_html(app: Sphinx):
     app.connect("html-collect-pages", _generate_search_html)
 
 
-def configure_database(app: Sphinx):
-    """Connect database for project output."""
+def configure_database(app: Sphinx, config: Config):
+    """
+    Connect database for project output.
+
+    TODO: Add support for multiple database backends?
+    """
+    # SQLite
+    """
     db_path = Path(app.outdir) / "db.sqlite"
     if db_path.exists():
         db_path.unlink()
-    models.initialize(db_path)
+    models.initialize("sqlite", db_path)
+    """
+    # PostgreSQL
+    if not app.config.sqlite3fts_database_url:
+        raise ValueError("Configuring database failed")
+    models.initialize("postgresql", app.config.sqlite3fts_database_url)
 
 
 def register_document(
diff --git a/src/atsphinx/sqlite3fts/models.py b/src/atsphinx/sqlite3fts/models.py
@@ -6,47 +6,49 @@
 
 TODO: Add support for multiple database backends?
 """
+import os
 from pathlib import Path
 from typing import Iterable
 
-from playhouse import sqlite_ext
+from playhouse import postgres_ext as ext
 
-db_proxy = sqlite_ext.DatabaseProxy()
+db_proxy = ext.DatabaseProxy()
 
 
-class Document(sqlite_ext.Model):
+class Document(ext.Model):
     """Document main model."""
 
-    page = sqlite_ext.TextField(null=False, unique=True)
-    title = sqlite_ext.TextField(null=False)
+    page = ext.TextField(null=False, unique=True)
+    title = ext.TextField(null=False)
 
     class Meta:  # noqa: D106
         database = db_proxy
 
 
-class Section(sqlite_ext.Model):
+class Section(ext.Model):
     """Section unit of document."""
 
-    document = sqlite_ext.ForeignKeyField(Document)
-    root = sqlite_ext.BooleanField(default=False, null=False)
-    ref = sqlite_ext.TextField(null=False)
-    title = sqlite_ext.TextField(null=False)
-    body = sqlite_ext.TextField(null=False)
+    document = ext.ForeignKeyField(Document)
+    root = ext.BooleanField(default=False, null=False)
+    ref = ext.TextField(null=False)
+    title = ext.TextField(null=False)
+    body = ext.TextField(null=False)
 
     class Meta:  # noqa: D106
         database = db_proxy
 
 
-class Content(sqlite_ext.FTS5Model):
+class Content(ext.Model):
     """Searching model."""
 
-    rowid = sqlite_ext.RowIDField()
-    title = sqlite_ext.SearchField()
-    body = sqlite_ext.SearchField()
+    rowid = ext.IntegerField()
+    title = ext.TextField()
+    body = ext.TextField()
 
     class Meta:  # noqa: D106
         database = db_proxy
-        options = {"tokenize": "trigram"}
+        # TODO: This is an option from SQLite, it does not work on other DBMS.
+        # options = {"tokenize": "trigram"}
 
 
 def store_document(document: Document, sections: Iterable[Section]):
@@ -74,16 +76,25 @@ def search_documents(keyword: str) -> Iterable[Section]:
     )
 
 
-def bind(db_path: Path):
+def bind(db_type: str, db_path: Path):
     """Bind connection.
 
     This works only set db into proxy, not included creating tables.
     """
-    db = sqlite_ext.SqliteExtDatabase(db_path)
+    if db_type == "sqlite":
+        db = ext.SqliteExtDatabase(db_path)
+    elif db_type == "postgresql":
+        db = ext.PostgresqlExtDatabase(db_path)
+        if "POSTGRES_LOG_STATEMENT" in os.environ:
+            db.execute_sql(
+                f"SET log_statement='{os.environ['POSTGRES_LOG_STATEMENT']}';"
+            )
+    else:
+        raise ValueError(f"Unknown database type: {db_type}")
     db_proxy.initialize(db)
 
 
-def initialize(db_path: Path):
+def initialize(db_type: str, db_path: Path):
     """Bind connection and create tables."""
-    bind(db_path)
+    bind(db_type, db_path)
     db_proxy.create_tables([Document, Section, Content])
diff --git a/tests/__init__.py b/tests/__init__.py
@@ -0,0 +1 @@
+"""Test package."""
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,11 +1,40 @@
 """Configuration for pytest."""
+import os
+
 import pytest
 from sphinx.testing.path import path
 
+from tests.util import Database
+
 pytest_plugins = "sphinx.testing.fixtures"
 collect_ignore = ["roots"]
 
 
+@pytest.fixture(scope="session")
+def database_dsn():
+    """Pytest fixture providing the database connection string for software tests."""
+    return "postgresql://postgres@localhost:5432"
+
+
+@pytest.fixture(scope="session")
+def database(database_dsn):
+    """Pytest fixture returning a database wrapper object."""
+    return Database(database_dsn)
+
+
+@pytest.fixture
+def conn(database):
+    """
+    Pytest fixture returning a database wrapper object, with content cleared.
+
+    This is intended to provide each test case with a blank slate.
+    """
+    if "POSTGRES_LOG_STATEMENT" in os.environ:
+        database.execute(f"SET log_statement='{os.environ['POSTGRES_LOG_STATEMENT']}';")
+    database.reset()
+    return database
+
+
 @pytest.fixture(scope="session")
 def rootdir():
     """Set root directory to use testing sphinx project."""
diff --git a/tests/roots/test-default/conf.py b/tests/roots/test-default/conf.py
@@ -3,6 +3,8 @@
     "atsphinx.sqlite3fts",
 ]
 
+sqlite3fts_database_url = "postgresql://postgres@localhost:5432"
+
 # To skip toctree
 rst_prolog = """
 :orphan:
diff --git a/tests/test_builders.py b/tests/test_builders.py
@@ -1,17 +1,11 @@
 """Test cases for custom builders."""
-import sqlite3
-from pathlib import Path
-
 import pytest
 from sphinx.testing.util import SphinxTestApp
 
 
 @pytest.mark.sphinx("sqlite", testroot="default")
-def test___work_builder(app: SphinxTestApp, status, warning):  # noqa
+def test___work_builder(app: SphinxTestApp, status, warning, conn):  # noqa
     app.build()
-    db_path = Path(app.outdir) / "db.sqlite"
-    assert db_path.exists()
-    conn = sqlite3.connect(db_path)
     assert len(conn.execute("SELECT * FROM document").fetchall()) > 0
     assert len(conn.execute("SELECT * FROM section").fetchall()) > 0
     assert len(conn.execute("SELECT * FROM content").fetchall()) > 0
diff --git a/tests/test_events.py b/tests/test_events.py
@@ -1,15 +1,9 @@
 """Test cases for result of events."""
-import sqlite3
-from pathlib import Path
-
 import pytest
 from sphinx.testing.util import SphinxTestApp
 
 
 @pytest.mark.sphinx("html", testroot="default")
-def test___work_builder(app: SphinxTestApp, status, warning):  # noqa
+def test___work_builder(app: SphinxTestApp, status, warning, conn):  # noqa
     app.build()
-    db_path = Path(app.outdir) / "db.sqlite"
-    assert db_path.exists()
-    conn = sqlite3.connect(db_path)
     assert len(conn.execute("SELECT * FROM document").fetchall()) > 0
diff --git a/tests/util.py b/tests/util.py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-name: Release new version`
	`1`	`+name: Release`
`2`	`2`
`3`	`3`	`on:`
`4`	`4`	`push:`
Original file line number	Diff line number	Diff line change
`@@ -32,6 +32,7 @@`
`32`	`32`	`}`
`33`	`33`	`# atsphinx-sqlite3fts`
`34`	`34`	`sqlite3fts_use_search_html = True`
	`35`	`+sqlite3fts_database_url = "postgresql://postgres@localhost:5432"`
`35`	`36`
`36`	`37`
`37`	`38`	`def setup(app): # noqa: D103`
Original file line number	Diff line number	Diff line change
`@@ -35,6 +35,7 @@ dynamic = ["version", "description"]`
`35`	`35`	`dependencies = [`
`36`	`36`	`"docutils",`
`37`	`37`	`"peewee",`
	`38`	`+ "psycopg2[binary]",`
`38`	`39`	`"Sphinx",`
`39`	`40`	`]`
`40`	`41`