Merge pull request #97 from seth-planet/tpu_runner3

Coral Object Detection: Various optimizations & fix YOLOv8 image for value scaling bug

Author: ChrisMaunder

src/modules/ObjectDetectionCoral/modulesettings.json

@@ -25,7 +25,7 @@
         "RuntimeLocation": "Local",       // Can be Local, Shared or System
         "PostStartPauseSecs": 1,          // Generally 1 if using GPU, 0 for CPU
         "Queue": "objectdetection_queue", // We make all Object detectors use the same queue.
-        "Parallelism": 1                  // 0 = Default = number of CPUs / 2
+        "Parallelism": 16                 // Should probably be TPU count * 2; I don't see harm in overprovisioning threads
       },
 
       "ModelRequirements" : [{
@@ -36,7 +36,7 @@
 
       "EnvironmentVariables": {
         "CPAI_CORAL_MULTI_TPU":  "true",
-        "CPAI_CORAL_MODEL_NAME": "MobileNet SSD", // "MobileNet SSD", "EfficientDet-Lite", "YOLOv5"
+        "CPAI_CORAL_MODEL_NAME": "MobileNet SSD", // "MobileNet SSD", "EfficientDet-Lite", "YOLOv5", "YOLOv8"
 
         "MODELS_DIR": "%CURRENT_MODULE_PATH%/assets",
         "MODEL_SIZE": "Small"
@@ -85,16 +85,16 @@
         ],
         "DownloadableModels":[
 
-          { "Name": "EfficientDet Large",  "Filename": "objectdetection-efficientdet-large-edgetpu.zip",  "Folder": "assets", "Description": "EfficientDet object detection, Large",  "FileSizeKb": 275800, "PreInstall": false },
-          { "Name": "EfficientDet Medium", "Filename": "objectdetection-efficientdet-medium-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Medium", "FileSizeKb": 275800, "PreInstall": false },
-          { "Name": "EfficientDet Small",  "Filename": "objectdetection-efficientdet-small-edgetpu.zip",  "Folder": "assets", "Description": "EfficientDet object detection, Small",  "FileSizeKb": 275800, "PreInstall": false },
-          { "Name": "EfficientDet Tiny",   "Filename": "objectdetection-efficientdet-tiny-edgetpu.zip",   "Folder": "assets", "Description": "EfficientDet object detection, Tiny",   "FileSizeKb": 275800, "PreInstall": false },
-
           { "Name": "MobileNet Large",  "Filename": "objectdetection-mobilenet-large-edgetpu.zip",  "Folder": "assets", "Description": "MobileNet object detection, Large",  "FileSizeKb": 275800, "PreInstall": true },
           { "Name": "MobileNet Medium", "Filename": "objectdetection-mobilenet-medium-edgetpu.zip", "Folder": "assets", "Description": "MobileNet object detection, Medium", "FileSizeKb": 275800, "PreInstall": true },
           { "Name": "MobileNet Small",  "Filename": "objectdetection-mobilenet-small-edgetpu.zip",  "Folder": "assets", "Description": "MobileNet object detection, Small",  "FileSizeKb": 275800, "PreInstall": true },
           { "Name": "MobileNet Tiny",   "Filename": "objectdetection-mobilenet-tiny-edgetpu.zip",   "Folder": "assets", "Description": "MobileNet object detection, Tiny",   "FileSizeKb": 275800, "PreInstall": true },
 
+          { "Name": "EfficientDet Large",  "Filename": "objectdetection-efficientdet-large-edgetpu.zip",  "Folder": "assets", "Description": "EfficientDet object detection, Large",  "FileSizeKb": 275800, "PreInstall": false },
+          { "Name": "EfficientDet Medium", "Filename": "objectdetection-efficientdet-medium-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Medium", "FileSizeKb": 275800, "PreInstall": false },
+          { "Name": "EfficientDet Small",  "Filename": "objectdetection-efficientdet-small-edgetpu.zip",  "Folder": "assets", "Description": "EfficientDet object detection, Small",  "FileSizeKb": 275800, "PreInstall": false },
+          { "Name": "EfficientDet Tiny",   "Filename": "objectdetection-efficientdet-tiny-edgetpu.zip",   "Folder": "assets", "Description": "EfficientDet object detection, Tiny",   "FileSizeKb": 275800, "PreInstall": false },
+
           { "Name": "YOLOv5 Large",  "Filename": "objectdetection-yolov5-large-edgetpu.zip",  "Folder": "assets", "Description": "YOLOv5 object detection, Large",  "FileSizeKb": 275800, "PreInstall": false },
           { "Name": "YOLOv5 Medium", "Filename": "objectdetection-yolov5-medium-edgetpu.zip", "Folder": "assets", "Description": "YOLOv5 object detection, Medium", "FileSizeKb": 275800, "PreInstall": false },
           { "Name": "YOLOv5 Small",  "Filename": "objectdetection-yolov5-small-edgetpu.zip",  "Folder": "assets", "Description": "YOLOv5 object detection, Small",  "FileSizeKb": 275800, "PreInstall": false },
@@ -112,9 +112,9 @@
           "Label": "Model",
           "Options": [
               { "Label": "MobileNet SSD",     "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "MobileNet SSD"     },
+              { "Label": "EfficientDet Lite", "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "EfficientDet-Lite" },
               { "Label": "YOLOv5",            "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "YOLOv5"            },
-              { "Label": "YOLOv8",            "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "YOLOv8"            },
-              { "Label": "EfficientDet Lite", "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "EfficientDet-Lite" }
+              { "Label": "YOLOv8",            "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "YOLOv8"            }
           ]
         },
         {

src/modules/ObjectDetectionCoral/objectdetection_coral_multitpu.py

@@ -231,20 +231,28 @@ def main():
 
   thread_cnt = 16
   tot_infr_time = 0 
+  half_wall_start = None
+  half_infr_count = 0 
   if args.count > 1:
     with concurrent.futures.ThreadPoolExecutor(max_workers=thread_cnt) as executor:
       start = time.perf_counter()
       for chunk_i in range(0, args.count-1, thread_cnt*8):
         fs = [executor.submit(_tpu_runner.process_image, options, copy.copy(image), args.threshold)
               for i in range(min(thread_cnt*8, args.count-1 - chunk_i))]
         for f in concurrent.futures.as_completed(fs):
-          _, infr_time = f.result()
+          _, infr_time, _ = f.result()
           tot_infr_time += infr_time
+
+          # Start a timer for the last ~half of the run for more accurate benchmark
+          if chunk_i > (args.count-1) / 3.0:
+            half_infr_count += 1
+            if half_wall_start is None:
+              half_wall_start = time.perf_counter()
 
         # Uncomment for testing
         # import random
         # logging.info("Pause")
-        # time.sleep(random.randint(0,INTERPRETER_LIFESPAN_SECONDS*2))
+        # time.sleep(random.randint(0,INTERPRETER_LIFESPAN_SECONDS*3))
   else:
     start = time.perf_counter()
 
@@ -254,20 +262,22 @@ def main():
   #   print(stat)
 
   start_one = time.perf_counter()
-  objs, infr_time = _tpu_runner.process_image(options, image, args.threshold)
+  objs, infr_time, _ = _tpu_runner.process_image(options, copy.copy(image), args.threshold)
   tot_infr_time += infr_time
+  half_infr_count += 1
   wall_time = time.perf_counter() - start
-  print('completed one run every %.2f ms for %d runs; %.2f ms wall time for a single run' %
+
+  half_wall_time = 0.0
+  if half_wall_start is not None:
+    half_wall_time = time.perf_counter() - half_wall_start
+  
+  print('completed one run every %.2fms for %d runs; %.2fms wall time for a single run' %
                             (wall_time * 1000 / args.count, args.count,
                             (time.perf_counter() - start_one) * 1000))
 
-  # Optimizing the number of segments used for a model would result in the
-  # lowest average time spent adjusted for number of TPUs used. At some point,
-  # adding additional segments just removes from the pool of TPUs you can use
-  # for parallelism.
-  print('%.2f ms avg time blocked across %d threads; %.2f avg TPU * ms / run' %
+  print('%.2fms avg time blocked across %d threads; %.2fms ea for final %d inferences' %
                             (tot_infr_time / args.count, thread_cnt,
-                             len(_tpu_runner.pipe.tpu_list) * wall_time * 1000 / args.count))
+                             half_wall_time * 1000 / half_infr_count, half_infr_count))
 
   print('-------RESULTS--------')
   if not objs:

src/modules/ObjectDetectionCoral/options.py

@@ -7,7 +7,7 @@
     print("Unable to import ModuleOptions, running with defaults")
     class ModuleOptions:
         module_path = '.'
-        def getEnvVariable(self, a, b):
+        def getEnvVariable(a, b):
             return b
 
 class Settings:
@@ -163,7 +163,7 @@ def __init__(self):
 
         self.MAX_PIPELINE_QUEUE_LEN             = 1000      # Multi-only
         self.TILE_OVERLAP                       = 15        # Multi-only.
-        self.DOWNSAMPLE_BY                      = 5.2       # Multi-only. Smaller number results in more tiles generated
+        self.DOWNSAMPLE_BY                      = 6.0       # Multi-only. Smaller number results in more tiles generated
         self.IOU_THRESHOLD                      = 0.1       # Multi-only
 
         # ----------------------------------------------------------------------

src/modules/ObjectDetectionCoral/pipelined_model_runner.py

@@ -2,6 +2,7 @@
 import subprocess
 import time
 import shutil
+import re
 
 #'''
 fn_list = [
@@ -11,12 +12,12 @@
     #'ssd_mobilenet_v1_coco_quant_postprocess',
     'tf2_ssd_mobilenet_v1_fpn_640x640_coco17_ptq',
     #'efficientdet_lite0_320_ptq',
-    'efficientdet_lite1_384_ptq',
+    #'efficientdet_lite1_384_ptq',
     'efficientdet_lite2_448_ptq',
     'efficientdet_lite3_512_ptq',
     'efficientdet_lite3x_640_ptq',
     #'yolov5n-int8',
-    'yolov5s-int8',
+    #'yolov5s-int8',
     'yolov5m-int8',
     'yolov5l-int8',
     #'yolov8n_416_640px', # lg 1st seg
@@ -192,7 +193,7 @@ def seg_exists(filename, segment_type, segment_count):
 
 MAX_TPU_COUNT = 8
 
-#'''
+'''
 # Generate segment files
 for sn in range(1,MAX_TPU_COUNT+1):
     for fn in fn_list:
@@ -313,15 +314,12 @@ def seg_exists(filename, segment_type, segment_count):
                       seg_list + ["--labels","coral/pycoral/test_data/coco_labels.txt","--input","/home/seth/coral/pycoral/test_data/grace_hopper.bmp",
                       "--count","1000","--num-tpus",str(num_tpus)]
                 print(cmd)
-                start_time = time.perf_counter()
-                subprocess.run(cmd)
-                timings.append(((time.perf_counter() - start_time), num_tpus, fn, seg_type, sn))
+                c = subprocess.run(cmd, capture_output=True)
+                print(c.stdout)
+                print(c.stderr)
+                ms_time = float(re.compile(r'threads; ([\d\.]+)ms ea').findall(c.stdout)[0])
+                timings.append((ms_time, num_tpus, fn, seg_type, sn))
 
-            # Find segment pipeline efficencies
-            #cmd = ["/home/seth/libcoral/out/k8/tools/model_pipelining_performance_analysis","--data_dir",seg_dir+seg_type,
-            #       "--model_list",fn,"--num_segments_list",','.join([str(i) for i in range(1,max_seg+1)])]
-            #print(cmd)
-            #subprocess.run(cmd)
         timings = sorted(timings, key=lambda t: t[0])
 
         # Print the top three