Implement SQLAlchemy

Author: bitdruid

README.md

@@ -326,6 +326,11 @@ Exceptions will be written into `waybackup_error.log` (each run overwrites the f
 <br>
 <br>
 
+## Future ideas (long run)
+
+- More module functionality
+- Docker UI
+
 ## Contributing
 
 I'm always happy for some feature requests to improve the usability of this tool.

pyproject.toml

@@ -7,7 +7,7 @@ packages = ["pywaybackup"]
 
 [project]
 name = "pywaybackup"
-version = "4.0.0"
+version = "4.1.0"
 description = "Query and download archive.org as simple as possible."
 authors = [
     { name = "bitdruid", email = "bitdruid@outlook.com" }
@@ -16,8 +16,7 @@ license = { file = "LICENSE" }
 readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [
-    "pysqlite3-binary==0.5.4; sys_platform == 'linux'",
-    "pysqlite-binary; sys_platform == 'win32'",
+    "SQLAlchemy==2.0.43",
     "requests==2.32.3",
     "tqdm==4.67.1",
     "python-magic==0.4.27; sys_platform == 'linux'",

pywaybackup/PyWayBackup.py

@@ -1,18 +1,18 @@
-import sys
+import multiprocessing
 import os
-import time
 import signal
-import multiprocessing
-from pywaybackup.helper import url_split, sanitize_filename
+import sys
+import time
 from importlib.metadata import version
 
 import pywaybackup.archive_save as archive_save
+from pywaybackup.archive_download import DownloadArchive
 from pywaybackup.db import Database as db
-from pywaybackup.Verbosity import Verbosity as vb
 from pywaybackup.Exception import Exception as ex
+from pywaybackup.files import CDXfile, CDXquery, CSVfile, File
+from pywaybackup.helper import sanitize_filename, url_split
 from pywaybackup.SnapshotCollection import SnapshotCollection
-from pywaybackup.archive_download import DownloadArchive
-from pywaybackup.files import CDXquery, CDXfile, CSVfile, File
+from pywaybackup.Verbosity import Verbosity as vb
 
 
 class _Status:
@@ -401,9 +401,9 @@ def paths(self, rel: bool = False) -> dict:
         """
         files = {
             "snapshots": os.path.join(self._output, self._domain),
-            "cdxfile": self._cdxfile,
+            "cdxfile": self._cdxfile.filepath,
             "dbfile": self._dbfile,
-            "csvfile": self._csvfile,
+            "csvfile": self._csvfile.filepath,
             "log": self._log,
             "debug": self._debug,
         }
@@ -475,10 +475,10 @@ def stop(self) -> bool:
         return False
 
     def _startup(self):
-        if db.QUERY_EXIST:
+        if db.query_exist:
             self._status.task = "resuming"
             vb.write(
-                content=f"\nDOWNLOAD job exist - processed: {db.QUERY_PROGRESS}\nResuming download... (to reset the job use '--reset')"
+                content=f"\nDOWNLOAD job exist - processed: {db.query_progress}\nResuming download... (to reset the job use '--reset')"
             )
 
             if not self._silent:

pywaybackup/Snapshot.py

@@ -1,20 +1,35 @@
 import os
+import threading
 
-from pywaybackup.db import Database
+from pywaybackup.db import Database, select, update, waybackup_snapshots
 from pywaybackup.helper import url_split
 
 
 class Snapshot:
     """
-    If a relevant property of the snapshot is modified, the change will be pushed to the database.
+    Represents a single snapshot entry and manages its state and persistence.
 
-    - _redirect_url
-    - _redirect_timestamp
-    - _response
-    - _file
+    When a relevant property of the snapshot is modified, the change is automatically
+    pushed to the database:
+        - redirect_url
+        - redirect_timestamp
+        - response_status
+        - file
+
+    Thread-safe for SQLite operations using a lock.
     """
 
+    __sqlite_lock = threading.Lock()
+
     def __init__(self, db: Database, output: str, mode: str):
+        """
+        Initialize a Snapshot instance and fetch its database row if available.
+
+        Args:
+            db (Database): Database connection/session manager.
+            output (str): Output directory for downloaded files.
+            mode (str): Download mode ('first', 'last', or default).
+        """
         self._db = db
         self.output = output
         self.mode = mode
@@ -26,52 +41,78 @@ def __init__(self, db: Database, output: str, mode: str):
 
         self._row = self.fetch()
         if self._row:
-            self.counter = self._row["counter"]
-            self.timestamp = self._row["timestamp"]
-            self.url_archive = self._row["url_archive"]
-            self.url_origin = self._row["url_origin"]
-            self.redirect_url = self._row["redirect_url"]
-            self.redirect_timestamp = self._row["redirect_timestamp"]
-            self.response_status = self._row["response"]
-            self.file = self._row["file"]
+            self.scid = self._row.scid
+            self.counter = self._row.counter
+            self.timestamp = self._row.timestamp
+            self.url_archive = self._row.url_archive
+            self.url_origin = self._row.url_origin
+            self.redirect_url = self._row.redirect_url
+            self.redirect_timestamp = self._row.redirect_timestamp
+            self.response_status = self._row.response
+            self.file = self._row.file
         else:
             self.counter = False
 
     def fetch(self):
         """
-        Get a snapshot-row from the snapshot table with response NULL. (not processed)
+        Fetch a snapshot row from the database with response=NULL (not processed).
+        Uses row locking to prevent concurrent workers from processing the same row.
+
+        Returns:
+            waybackup_snapshots or None: The next unprocessed snapshot row, or None if none available.
         """
         # mark as locked for other workers // only visual because get_snapshot fetches by NULL
-        self._db.cursor.execute(
-            """
-            UPDATE snapshot_tbl
-            SET response = 'LOCK'
-            WHERE rowid = (
-                SELECT rowid FROM snapshot_tbl 
-                WHERE response IS NULL
-                LIMIT 1
-            )
-            RETURNING rowid, *;
-            """
-        )
-        row = self._db.cursor.fetchone()
-        self._db.conn.commit()
-        return row
+        # prevent another worker from fetching between LOCK-update (for sqlite by threading.Lock, else lock row)
+
+        def __on_sqlite():
+            if self._db.session.bind.dialect.name == "sqlite":
+                return True
+            return False
+
+        def __get_row():
+            with self._db.session.begin():
+                row = self._db.session.execute(
+                    select(waybackup_snapshots)
+                    .where(waybackup_snapshots.response.is_(None))
+                    .order_by(waybackup_snapshots.scid)
+                    .limit(1)
+                    .with_for_update(skip_locked=True)
+                ).scalar_one_or_none()
+
+                if row is None:
+                    return None
+
+                row.response = "LOCK"
+
+            return row
+
+        if __on_sqlite():
+            with self.__sqlite_lock:
+                return __get_row()
+        else:
+            return __get_row()
 
     def modify(self, column, value):
         """
-        Modify the snapshot in the database.
+        Update a column value for this snapshot in the database.
+
+        Args:
+            column (str): Name of the column to update.
+            value: New value to set for the column.
         """
-        query = f"UPDATE snapshot_tbl SET {column} = ? WHERE counter = ?"
-        self._db.cursor.execute(query, (value, self.counter))
-        self._db.conn.commit()
+        column = getattr(waybackup_snapshots, column)
+        self._db.session.execute(update(waybackup_snapshots).where(waybackup_snapshots.scid == self.scid).values({column: value}))
+        self._db.session.commit()
 
     def create_output(self):
         """
-        Create a file path for the snapshot.
+        Generate the file path for the snapshot download.
 
-        - If MODE_LAST or MODE_FIRST is enabled, the path does not include the timestamp.
-        - Otherwise, include the timestamp in the path.
+        If mode is 'first' or 'last', the path does not include the timestamp.
+        Otherwise, the timestamp is included in the path.
+
+        Returns:
+            str: Absolute path to the output file for the snapshot.
         """
         domain, subdir, filename = url_split(self.url_archive.split("id_/")[1], index=True)
 
@@ -86,43 +127,79 @@ def create_output(self):
 
     @property
     def redirect_url(self):
+        """
+        str: The redirect URL for this snapshot, if any.
+        """
         return self._redirect_url
 
     @redirect_url.setter
     def redirect_url(self, value):
+        """
+        Set the redirect URL and update the database.
+
+        Args:
+            value (str): The new redirect URL.
+        """
         if self.redirect_timestamp is None and value is None:
             return
         self._redirect_url = value
         self.modify(column="redirect_url", value=value)
 
     @property
     def redirect_timestamp(self):
+        """
+        str: The timestamp of the redirect, if any.
+        """
         return self._redirect_timestamp
 
     @redirect_timestamp.setter
     def redirect_timestamp(self, value):
+        """
+        Set the redirect timestamp and update the database.
+
+        Args:
+            value (str): The new redirect timestamp.
+        """
         if self.redirect_url is None and value is None:
             return
         self._redirect_timestamp = value
         self.modify(column="redirect_timestamp", value=value)
 
     @property
     def response_status(self):
+        """
+        str: The HTTP response/status for this snapshot.
+        """
         return self._response_status
 
     @response_status.setter
     def response_status(self, value):
+        """
+        Set the response status and update the database.
+
+        Args:
+            value (str): The new response status.
+        """
         if self.response_status is None and value is None:
             return
         self._response_status = value
         self.modify(column="response", value=value)
 
     @property
     def file(self):
+        """
+        str: The file path for the downloaded snapshot.
+        """
         return self._file
 
     @file.setter
     def file(self, value):
+        """
+        Set the file path and update the database.
+
+        Args:
+            value (str): The new file path.
+        """
         if self.file is None and value is None:
             return
         self._file = value