Merge pull request #82 from fastlabel/bugfix/789-limitation-size

Add request size check

Author: ryoKaz

README.md

@@ -433,7 +433,7 @@ task = client.create_multi_image_task(
 ```
 
 ##### Limitation
-
+* You can upload up to a total size of 512 MB.
 * You can upload up to 250 files in total.
 
 #### Find Task
@@ -565,6 +565,9 @@ task_id = client.create_video_task(
 )
 ```
 
+##### Limitation
+* You can upload up to a size of 250 MB.
+
 #### Find Task
 
 Find a single task.
@@ -669,6 +672,9 @@ task_id = client.create_video_classification_task(
 )
 ```
 
+##### Limitation
+* You can upload up to a size of 250 MB.
+
 #### Find Task
 
 Find a single task.

fastlabel/__init__.py

@@ -477,6 +477,7 @@ def create_multi_image_task(
             raise FastLabelInvalidException(
                 "Folder does not have any file.", 422)
         contents = []
+        contents_size = 0
         for file_path in file_paths:
             if not utils.is_image_supported_ext(file_path):
                 raise FastLabelInvalidException(
@@ -491,6 +492,11 @@ def create_multi_image_task(
                 "name": os.path.basename(file_path),
                 "file": file
             })
+            contents_size += utils.get_json_length(contents[-1])
+            if contents_size > const.SUPPORTED_CONTENTS_SIZE:
+                raise FastLabelInvalidException(
+                    f"Supported contents size is under {const.SUPPORTED_CONTENTS_SIZE}.", 422)
+
         payload = {"project": project, "name": name, "contents": contents}
         if status:
             payload["status"] = status
@@ -537,6 +543,10 @@ def create_video_task(
         if not utils.is_video_supported_ext(file_path):
             raise FastLabelInvalidException(
                 "Supported extensions are mp4.", 422)
+        if os.path.getsize(file_path) > const.SUPPORTED_VIDEO_SIZE:
+            raise FastLabelInvalidException(
+                f"Supported video size is under 250 MB.", 422)
+
         file = utils.base64_encode(file_path)
         payload = {"project": project, "name": name, "file": file}
         if status:
@@ -586,6 +596,10 @@ def create_video_classification_task(
         if not utils.is_video_supported_ext(file_path):
             raise FastLabelInvalidException(
                 "Supported extensions are mp4.", 422)
+        if os.path.getsize(file_path) > const.SUPPORTED_VIDEO_SIZE:
+            raise FastLabelInvalidException(
+                f"Supported video size is under 250 MB.", 422)
+
         file = utils.base64_encode(file_path)
         payload = {"project": project, "name": name, "file": file}
         if status:

fastlabel/const.py

@@ -3,11 +3,19 @@
 # only 57 types
 COLOR_PALETTE = [0, 0, 0, 228, 26, 28, 55, 126, 184, 77, 175, 74, 152, 78, 163, 255, 127, 0, 255, 255, 51, 166, 86, 40, 247, 129, 191, 153, 153, 153, 102, 194, 165, 252, 141, 98, 141, 160, 203, 231, 138, 195, 166, 216, 84, 255, 217, 47, 229, 196, 148, 179, 179, 179, 141, 211, 199, 255, 255, 179, 190, 186, 218, 251, 128, 114, 128, 177, 211, 253, 180, 98, 179, 222, 105, 252, 205, 229, 217, 217, 217, 188, 128, 189, 204, 235, 197, 255, 237, 111, 166, 206, 227, 31, 120, 180, 178, 223, 138, 51, 160, 44, 251, 154, 153, 227, 26, 28, 253, 191, 111, 255, 127, 0, 202, 178, 214, 106, 61, 154, 255, 255, 153, 177, 89, 40, 127, 201, 127, 190, 174, 212, 253, 192, 134, 255, 255, 153, 56, 108, 176, 240, 2, 127, 191, 91, 22, 102, 102, 102, 27, 158, 119, 217, 95, 2, 117, 112, 179, 231, 41, 138, 102, 166, 30, 230, 171, 2, 166, 118, 29, 102, 102, 102]
 
+# under 512 MB. Actual size is 536870888 bytes, but to consider other attributes, minus 888 bytes.
+# Because of V8's limitation, API only can accept the JSON string that length is under this.
+SUPPORTED_CONTENTS_SIZE = 536870000
+
+# API can accept under 250 MB ( 250 * 1024 * 1024 )
+SUPPORTED_VIDEO_SIZE = 262144000
+
+
 class AnnotationType(Enum):
     bbox = "bbox"
     polygon = "polygon"
     keypoint = "keypoint"
     line = "line"
     segmentation = "segmentation"
     classification = "classification"
-    pose_estimation = "pose_estimation"
+    pose_estimation = "pose_estimation"

fastlabel/utils.py

@@ -2,6 +2,7 @@
 import base64
 import numpy as np
 import geojson
+import json
 from typing import List
 
 
@@ -15,7 +16,7 @@ def is_image_supported_ext(file_path: str) -> bool:
 
 
 def is_video_supported_ext(file_path: str) -> bool:
-    return file_path.lower().endswith(('.mp4'))
+    return file_path.lower().endswith('.mp4')
 
 
 def get_basename(file_path: str) -> str:
@@ -44,13 +45,14 @@ def reverse_points(points: List[int]) -> List[int]:
                 0, points[index])
     return reversed_points
 
+
 def is_clockwise(points: list) -> bool:
     """
     points: [x1, y1, x2, y2, x3, y3, ... xn, yn]
-    Sum over the edges, (x2 − x1)(y2 + y1). 
+    Sum over the edges, (x2 − x1)(y2 + y1).
     If the result is positive the curve is clockwise, if it's negative the curve is counter-clockwise.
-    
-    The above is assumes a normal Cartesian coordinate system. 
+
+    The above is assumes a normal Cartesian coordinate system.
     HTML5 canvas, use an inverted Y-axis.
     Therefore If the area is negative, the curve is clockwise.
     """
@@ -65,4 +67,10 @@ def is_clockwise(points: list) -> bool:
 
     if sum_edges < 0:
         return True
-    return False 
+    return False
+
+
+def get_json_length(value) -> int:
+    json_str = json.dumps(value)
+    return len(json_str)
+