Bug fixes

Author: mailseth

src/modules/ObjectDetectionCoral/objectdetection_coral_multitpu.py

@@ -231,7 +231,7 @@ def main():
 
   thread_cnt = 16
   tot_infr_time = 0 
-  half_wall_start = 0 
+  half_wall_start = None
   half_infr_count = 0 
   if args.count > 1:
     with concurrent.futures.ThreadPoolExecutor(max_workers=thread_cnt) as executor:
@@ -244,9 +244,9 @@ def main():
           tot_infr_time += infr_time
 
           # Start a timer for the last ~half of the run for more accurate benchmark
-          if chunk_i > (args.count-1 / 3):
+          if chunk_i > (args.count-1) / 3.0:
             half_infr_count += 1
-            if half_wall_start == 0:
+            if half_wall_start is None:
               half_wall_start = time.perf_counter()
 
         # Uncomment for testing
@@ -255,7 +255,6 @@ def main():
         # time.sleep(random.randint(0,INTERPRETER_LIFESPAN_SECONDS*3))
   else:
     start = time.perf_counter()
-    half_wall_start = time.perf_counter()
 
   # snapshot = tracemalloc.take_snapshot()
   # top_stats = snapshot.statistics('lineno')
@@ -267,15 +266,18 @@ def main():
   tot_infr_time += infr_time
   half_infr_count += 1
   wall_time = time.perf_counter() - start
-  half_wall_time = time.perf_counter() - half_wall_start
+
+  half_wall_time = 0.0
+  if half_wall_start is not None:
+    half_wall_time = time.perf_counter() - half_wall_start
 
   print('completed one run every %.2fms for %d runs; %.2fms wall time for a single run' %
                             (wall_time * 1000 / args.count, args.count,
                             (time.perf_counter() - start_one) * 1000))
 
   print('%.2fms avg time blocked across %d threads; %.2fms ea for final %d inferences' %
                             (tot_infr_time / args.count, thread_cnt,
-                             half_wall_time * 1000 / half_infr_count, half_infr_count)
+                             half_wall_time * 1000 / half_infr_count, half_infr_count))
 
   print('-------RESULTS--------')
   if not objs:

src/modules/ObjectDetectionCoral/tpu_runner.py

@@ -21,6 +21,7 @@
 import logging
 import queue
 import gc
+import math
 
 from datetime import datetime
 
@@ -147,7 +148,8 @@ def _interpreter_runner(self, seg_idx: int):
                 self.interpreter = None
                 self.input_details = None
                 self.output_details = None
-                self.rebalancing_lock.release()
+                if self.rebalancing_lock.locked():
+                    self.rebalancing_lock.release()
                 return
 
             start_inference_time = time.perf_counter_ns()
@@ -466,7 +468,7 @@ def _watchdog(self):
                 time.time() - self.watchdog_time > self.max_idle_secs_before_recycle:
                 logging.warning("No work in {} seconds, watchdog shutting down TPUs.".format(self.max_idle_secs_before_recycle))
                 self.runner_lock.acquire(timeout=MAX_WAIT_TIME)
-                if self.pipe:
+                if self.pipe.first_name:
                     self.pipe.delete()
                 self.runner_lock.release()
                 # Pipeline will reinitialize itself as needed
@@ -726,7 +728,7 @@ def _delete(self):
     def pipeline_ok(self) -> bool:
         """ Check we have valid interpreters """
         with self.runner_lock:
-            return self.pipe and any(self.pipe.interpreters)
+            return bool(self.pipe and any(self.pipe.interpreters))
 
     def process_image(self,
                       options:Options,
@@ -1142,9 +1144,17 @@ def _resize_and_chop_tiles(self,
             self.printed_shape_map[key] = True
 
         # Do chunking
+        # Image will not be an even fit in at least one dimension; space tiles appropriately.
         tiles = []
-        for x_off in range(0, image.width - options.tile_overlap, int(ceil((image.width - m_width)/tiles_x))):
-            for y_off in range(0, image.height - options.tile_overlap, int(ceil((image.height - m_height)/tiles_y))):
+        step_x = 1
+        if tiles_x > 1:
+            step_x = int(math.ceil((image.width - m_width)/(tiles_x-1)))
+        step_y = 1
+        if tiles_y > 1:
+            step_y = int(math.ceil((image.height - m_height)/(tiles_y-1)))
+
+        for x_off in range(0, max(image.width - m_width, 0) + tiles_x, step_x):
+            for y_off in range(0, max(image.height - m_height, 0) + tiles_y, step_y):
                 # Adjust contrast on a per-chunk basis; we will likely be quantizing the image during scaling
                 image_chunk = ImageOps.autocontrast(image.crop((x_off,
                                                                 y_off,
@@ -1158,9 +1168,6 @@ def _resize_and_chop_tiles(self,
 
                 tiles.append((cropped_arr.astype(self.input_details['dtype']), resamp_info))
 
-                # Check again with various scales
-                #Image.fromarray((tiles[-1][0] + 128).astype(np.uint8)).save("test.png")
-
 
         return tiles