Merge pull request #5 from objectrocket/SRE-2645-added-more-loging

Sre 2645 added more loging

Author: ikapriz-objectrocket

mongodb_consistent_backup/Oplog/Resolver/Resolver.py

@@ -148,5 +148,7 @@ def run(self):
             self.stopped = True
 
         for shard in self.resolver_state:
+            logging.info("Getting shard oplog state %s" % (shard))
             self.resolver_summary[shard] = self.resolver_state[shard].get()
+        logging.info("Returning self.resolver_summary")
         return self.resolver_summary

mongodb_consistent_backup/Oplog/Tailer/TailThread.py

@@ -89,7 +89,7 @@ def status(self):
 
     def connect(self):
         if not self.db:
-            self.db = DB(self.uri, self.config, True, 'secondary', True)
+            self.db = DB(self.uri, self.config, False, 'secondary', True)
         return self.db.connection()
 
     def run(self):
@@ -169,6 +169,7 @@ def run(self):
             self.stopped = True
             self.state.set('running', False)
             self.timer.stop(self.timer_name)
+            logging.info("Thread is stopped and marked as stopped %s" % self.uri)
 
         if self.exit_code == 0:
             log_msg_extra = "%i oplog changes" % self.count

mongodb_consistent_backup/Oplog/Tailer/Tailer.py

@@ -102,8 +102,8 @@ def stop(self, kill=False, sleep_secs=3):
                     logging.warning("Could not get current optime from PRIMARY! Using now as a stop time")
                     timestamp = Timestamp(int(time()), 0)
 
-                # wait for replication to get in sync
-                while state.get('last_ts') and state.get('last_ts') < timestamp:
+                # wait for replication to get in sync making sure cursor has not been stopped in a race condition
+                while state.get('last_ts') and state.get('last_ts') < timestamp and not self.shards[shard]['thread'].stopped:
                     logging.info('Waiting for %s tailer to reach ts: %s, currrent: %s' % (uri, timestamp, state.get('last_ts')))
                     sleep(sleep_secs)