From 12029a1c6ea9d755b6836f103f69de42224d4054 Mon Sep 17 00:00:00 2001
From: Marc Mengel <mengel@fnal.gov>
Date: Wed, 12 Nov 2025 16:37:46 -0600
Subject: [PATCH] make daemon more reliable by hiding exceptions from TaskQueue

---
 daemon/daemon.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/daemon/daemon.py b/daemon/daemon.py
index 1a023a2..7aeece2 100644
--- a/daemon/daemon.py
+++ b/daemon/daemon.py
@@ -15,6 +15,19 @@
 import urllib3
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
+import functools, traceback
+
+def log_exceptions(f):
+    ''' decorator logging exceptions and not letting them past '''
+    @functools.wraps(f)
+    def wrapper( self,  *args, **kwargs ):
+        try:
+           return f( self, *args, **kwargs )
+        except Exception:
+           self.log(traceback.format_exc())
+           return None
+    return wrapper
+
 def to_did(namespace, name):
     return f"{namespace}:{name}"
 
@@ -296,6 +309,7 @@ def tape_replicas_by_rse(self, active_handles):
                     tape_replicas_by_rse.setdefault(rse, {})[replica.did()] = replica
         return tape_replicas_by_rse
         
+    @log_exceptions
     @synchronized
     def check_project_state(self):
         if self.Removed:
@@ -311,6 +325,7 @@ def check_project_state(self):
             self.log("removing project:", reason)
             self.remove_me(reason)          # this will cancel this and other repeating task
 
+    @log_exceptions
     @synchronized
     def sync_replicas(self):
         if self.Removed:
@@ -382,6 +397,7 @@ def sync_replicas(self):
             self.error(textwrap.indent(traceback.format_exc(), "  "))
         self.debug("sync_replicas done")
 
+    @log_exceptions
     @synchronized
     def update_replicas_availability(self):
         if self.Removed:
@@ -448,6 +464,7 @@ def __init__(self, db, rse_config, rucio_client, url_schemes):
         self.RucioClient = rucio_client
         self.URLSchemes = url_schemes or None
 
+    @log_exceptions
     def clean(self):
         self.debug("cleaner...")
         try: