feat(series): offload series log data to external store

Author: quasiTriestino

PyReconstruct/modules/constants/__init__.py

@@ -43,5 +43,7 @@
 )
 
 from .getdatetime import(
-    getDateTime
+    getDateTime,
+    get_now,
+    remove_days_from_today
 )

PyReconstruct/modules/constants/getdatetime.py

@@ -1,11 +1,34 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from PySide6.QtCore import QSettings
 
-def getDateTime(date_str="%y-%m-%d", time_str="%H:%M"):
+
+def utc_p() -> bool:
+    """Determine if user using UTC."""
+
     settings = QSettings("KHLab", "PyReconstruct")
     utc = settings.value("utc", False)
-    dt = datetime.utcnow() if utc else datetime.now()
+
+    return False if utc == "false" else True
+
+
+def get_now() -> datetime:
+    """Return now's datetime object."""
+
+    return datetime.utcnow() if utc_p() else datetime.now()
+
+
+def remove_days_from_today(delta_days: int):
+    """Remove days from now."""
+
+    return get_now().date() - timedelta(days=delta_days)
+
+
+def getDateTime(date_str="%y-%m-%d", time_str="%H:%M"):
+    
+    dt = get_now()
+    
     d = dt.strftime(date_str)
     t = dt.strftime(time_str)
-    return d, t
+    
+    return d, t

PyReconstruct/modules/datatypes/log.py

@@ -1,7 +1,9 @@
 import re
+from pathlib import Path
 from datetime import datetime
 
-from PyReconstruct.modules.constants import getDateTime
+from PyReconstruct.modules.constants import getDateTime, get_now, remove_days_from_today
+
 
 class Log():
 
@@ -234,14 +236,6 @@ def addLog(self, user : str, obj_name : str, snum : int, event : str):
             else:
                 self.all_logs.append(log)
 
-        # # for debugging
-        # # Clear console
-        # if os.name == 'nt':  # Windows
-        #     _ = os.system('cls')
-        # else:  # Mac and Linux
-        #     _ = os.system('clear')
-        # print(str(self).replace(", ", "\t"))
-    
     def addExistingLog(self, log : Log, track_dyn=False):
         """Add an existing log object to the set.
         
@@ -334,6 +328,42 @@ def getLastIndex(self, snum : int, cname : str):
             i -= 1
         return i
 
+    @staticmethod
+    def exportLogHistory(hidden_dir: str, output_fp: str, older_than: int) -> None:
+        """Export log history as CSV for external storage."""
+
+        existing_log = Path(hidden_dir) / "existing_log.csv"
+        storage_log = Path(output_fp)
+        new_log = existing_log.with_name("new_log.csv")
+
+        if storage_log.exists(): storage_log.unlink()  # remove if already exists
+        
+        older_than = remove_days_from_today(older_than)
+
+        with storage_log.open("a") as external_store, new_log.open("a") as new:
+            
+            with existing_log.open("r") as log:
+                
+                for line in log.readlines():
+                    
+                    if "Date" in line:
+
+                        external_store.write(line)
+                        new.write(line)
+                        
+                    else:
+
+                        log_date = line.split(",")[0].strip()
+                        log_date = datetime.strptime(log_date, "%y-%m-%d").date()
+
+                        if log_date <= older_than:
+                            external_store.write(line)
+                        else:
+                            new.write(line)
+
+        new_log.replace(existing_log)  # overwrite old log
+                        
+
 class LogSetPair():
 
     def __init__(self, logset0 : LogSet, logset1 : LogSet):

PyReconstruct/modules/gui/main/main_imports.py

@@ -127,7 +127,8 @@
     Series,
     Trace,
     Transform,
-    Flag
+    Flag,
+    LogSet
 )
 
 from PyReconstruct.modules.constants import (

PyReconstruct/modules/gui/main/main_window.py

@@ -2278,6 +2278,73 @@ def hideSeriesTraces(self, hidden=True):
         self.series.hideAllTraces(hidden)
         self.field.reload()
 
+    def exportLog(self):
+        """Export complete log history"""
+
+        pass
+
+    def offloadLog(self):
+        """Offload log history to external file for storage."""
+
+        ## Notify user
+        note_offloading = (
+            f"Log creation in PyReconstruct is still a work in progress. Series with extensive "
+            f"log history may become very large. You can reduce series size in some cases by "
+            f"offloading and saving log history externally as a CSV file. \n\nClick OK to select "
+            f"where to save the log CSV."
+        )
+
+        confirm = notifyConfirm(note_offloading)
+
+        if not confirm:
+            return
+
+        ## Get filepath
+
+        path_check_ok = False
+
+        while not path_check_ok:
+
+            output_fp = FileDialog.get("save", self, "Save data as CSV file", "*.csv", "log_export.csv")
+            if not output_fp:
+
+                return
+
+            elif Path(output_fp).exists():
+
+                notify(
+                    f"To prevent accidnetally overwriting previously offloaded logs, "
+                    f"please provide a filename that does not yet exist."
+                )
+
+            else:
+
+                path_check_ok = True
+
+        ## Query for age of exported existing data
+
+        structure = [
+            ["Export history older than x days: ", ("int", 10)],
+        ]
+        response, confirmed = QuickDialog.get(self, structure, "Export log history")
+        
+        if not confirmed:
+            return
+
+        days = response[0]
+
+        ## Export for external storage
+
+        LogSet.exportLogHistory(self.series.hidden_dir, output_fp, days)
+        self.series.addLog(None, None, f"Offloaded log to {output_fp}")
+
+        notify(
+            f"Log offloaded to external CSV file.\n\n"
+            f"Be sure to save offloaded log data in a safe place."
+        )
+
+        return None
+
     def setFindZoom(self):
         """Set the magnification for find contour."""
         z, confirmed = QInputDialog.getInt(

PyReconstruct/modules/gui/main/menubar.py

@@ -145,6 +145,14 @@ def return_series_menu(self):
                     ("unhidealltraces_act", "Unhide all traces", "", lambda : self.hideSeriesTraces(hidden=False))
                 ]
             },
+            {
+                "attr_name": "serieslogmenu",
+                "text": "Log",
+                "opts":
+                [
+                    ("offloadlog_act", "Offload log history...", "", self.offloadLog),
+                ]
+            },
             {
                 "attr_name": "threedeemenu",
                 "text": "3D",