From fcd5cb43929bf608783653e47c1c696bf6c98867 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 23 Apr 2025 16:05:12 +0200 Subject: [PATCH 01/11] Try to build on Windows --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 70f0f2a7..15b06a3d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: windows-latest strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] From 52ec796ed0294414801e3d068c27361cc9b915b4 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 23 Apr 2025 16:49:56 +0200 Subject: [PATCH 02/11] Update test.yml --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 15b06a3d..2d2f59e8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,8 @@ on: jobs: build: runs-on: windows-latest + env: + PYTHONUTF8: 1 strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] From bfb19e1a61a786dcc4ff6dcbdeb38281fb892ef1 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 23 Apr 2025 17:12:52 +0200 Subject: [PATCH 03/11] Trying to make it cross platform --- roboflow/util/folderparser.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/roboflow/util/folderparser.py b/roboflow/util/folderparser.py index eff2a118..75abda41 100644 --- a/roboflow/util/folderparser.py +++ b/roboflow/util/folderparser.py @@ -13,9 +13,8 @@ def parsefolder(folder): - folder = folder.strip() - if folder.endswith("/"): - folder = folder[:-1] + # Remove trailing spaces and slashes + folder = folder.strip().rstrip("\\/") if not os.path.exists(folder): raise Exception(f"folder does not exist. {folder}") files = _list_files(folder) @@ -53,8 +52,12 @@ def _list_files(folder): for root, dirs, files in os.walk(folder): for file in files: file_path = os.path.join(root, file) - filedescriptors.append(_describe_file(file_path.split(folder)[1])) - filedescriptors = sorted(filedescriptors, key=lambda x: _alphanumkey(x["file"])) + rel = os.path.relpath(file_path, folder) + filedescriptors.append(_describe_file(f"/{rel}")) + filedescriptors = sorted( + filedescriptors, + key=lambda x: _alphanumkey(x["file"]), + ) return filedescriptors @@ -64,7 +67,7 @@ def _add_indices(files): def _describe_file(f): - name = f.split("/")[-1] + name = os.path.basename(f) dirname = os.path.dirname(f) fullkey, extension = os.path.splitext(f) fullkey2 = fullkey.replace("/labels", "").replace("/images", "") From d519f424b5c5eeb2e75c97157014d2beeebaa793 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 23 Apr 2025 17:13:41 +0200 Subject: [PATCH 04/11] fixup! Trying to make it cross platform --- .github/workflows/test.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d2f59e8..570da3d1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,12 +8,13 @@ on: jobs: build: - runs-on: windows-latest - env: - PYTHONUTF8: 1 strategy: matrix: + os: ["ubuntu-latest", "windows-latest"] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + runs-on: ${{ matrix.os }} + env: + PYTHONUTF8: 1 steps: - name: 🛎️ Checkout From 47fec9a9f597cfdb6f0c8ef3d4f82589570752f6 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 23 Apr 2025 17:50:39 +0200 Subject: [PATCH 05/11] fixup! fixup! Trying to make it cross platform --- tests/test_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_version.py b/tests/test_version.py index f13479bc..3697e1f5 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -59,7 +59,7 @@ def test_download_returns_dataset(self, *_): self.assertEqual(dataset.name, self.version.name) self.assertEqual(dataset.version, self.version.version) self.assertEqual(dataset.model_format, "coco") - self.assertEqual(dataset.location, "/my-spot") + self.assertEqual(dataset.location, os.path.abspath("/my-spot")) class TestExport(unittest.TestCase): From 98ff2bdbd1454ca63c608d4f6cb3f94b1a6481cc Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 23 Apr 2025 18:16:44 +0200 Subject: [PATCH 06/11] fixup! fixup! fixup! Trying to make it cross platform --- roboflow/util/folderparser.py | 4 ++-- tests/util/test_folderparser.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/roboflow/util/folderparser.py b/roboflow/util/folderparser.py index 75abda41..dc183588 100644 --- a/roboflow/util/folderparser.py +++ b/roboflow/util/folderparser.py @@ -53,7 +53,7 @@ def _list_files(folder): for file in files: file_path = os.path.join(root, file) rel = os.path.relpath(file_path, folder) - filedescriptors.append(_describe_file(f"/{rel}")) + filedescriptors.append(_describe_file(f"{os.sep}{rel}")) filedescriptors = sorted( filedescriptors, key=lambda x: _alphanumkey(x["file"]), @@ -70,7 +70,7 @@ def _describe_file(f): name = os.path.basename(f) dirname = os.path.dirname(f) fullkey, extension = os.path.splitext(f) - fullkey2 = fullkey.replace("/labels", "").replace("/images", "") + fullkey2 = fullkey.replace(f"{os.sep}labels", "").replace(f"{os.sep}images", "") key = os.path.splitext(name)[0] return { "file": f, diff --git a/tests/util/test_folderparser.py b/tests/util/test_folderparser.py index d4d497cb..078002d0 100644 --- a/tests/util/test_folderparser.py +++ b/tests/util/test_folderparser.py @@ -17,12 +17,14 @@ def test_parse_sharks_coco(self): sharksfolder = f"{thisdir}/../datasets/sharks-tiny-coco" parsed = folderparser.parsefolder(sharksfolder) testImagePath = "/train/sharks_mp4-20_jpg.rf.90ba2e8e9ca0613f71359efb7ed48b26.jpg" + print("PARSED", parsed["images"]) testImage = [i for i in parsed["images"] if i["file"] == testImagePath][0] assert len(json.loads(testImage["annotationfile"]["rawText"])["annotations"]) == 5 def test_parse_sharks_createml(self): sharksfolder = f"{thisdir}/../datasets/sharks-tiny-createml" parsed = folderparser.parsefolder(sharksfolder) + print("PARSED", parsed["images"]) testImagePath = "/train/sharks_mp4-20_jpg.rf.5359121123e86e016401ea2731e47949.jpg" testImage = [i for i in parsed["images"] if i["file"] == testImagePath][0] imgParsedAnnotations = json.loads(testImage["annotationfile"]["rawText"]) From ae045ac6c3a6a6b3323b83d5990249f475f4c911 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 23 Apr 2025 18:31:48 +0200 Subject: [PATCH 07/11] fixup! fixup! fixup! fixup! Trying to make it cross platform --- tests/util/test_folderparser.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/util/test_folderparser.py b/tests/util/test_folderparser.py index 078002d0..37929759 100644 --- a/tests/util/test_folderparser.py +++ b/tests/util/test_folderparser.py @@ -1,10 +1,15 @@ import json +import os import unittest -from os.path import abspath, dirname from roboflow.util import folderparser -thisdir = dirname(abspath(__file__)) +thisdir = os.path.dirname(os.path.abspath(__file__)) + + +def _find_test_image(image, images): + image = image.replace("/", os.sep) + return next(i for i in images if i["file"] == image) class TestFolderParser(unittest.TestCase): @@ -18,7 +23,7 @@ def test_parse_sharks_coco(self): parsed = folderparser.parsefolder(sharksfolder) testImagePath = "/train/sharks_mp4-20_jpg.rf.90ba2e8e9ca0613f71359efb7ed48b26.jpg" print("PARSED", parsed["images"]) - testImage = [i for i in parsed["images"] if i["file"] == testImagePath][0] + testImage = _find_test_image(testImagePath, parsed["images"]) assert len(json.loads(testImage["annotationfile"]["rawText"])["annotations"]) == 5 def test_parse_sharks_createml(self): @@ -26,7 +31,7 @@ def test_parse_sharks_createml(self): parsed = folderparser.parsefolder(sharksfolder) print("PARSED", parsed["images"]) testImagePath = "/train/sharks_mp4-20_jpg.rf.5359121123e86e016401ea2731e47949.jpg" - testImage = [i for i in parsed["images"] if i["file"] == testImagePath][0] + testImage = _find_test_image(testImagePath, parsed["images"]) imgParsedAnnotations = json.loads(testImage["annotationfile"]["rawText"]) assert len(imgParsedAnnotations) == 1 imgReference = imgParsedAnnotations[0] @@ -36,7 +41,7 @@ def test_parse_sharks_yolov9(self): def test(sharksfolder): parsed = folderparser.parsefolder(sharksfolder) testImagePath = "/train/images/sharks_mp4-20_jpg.rf.5359121123e86e016401ea2731e47949.jpg" - testImage = [i for i in parsed["images"] if i["file"] == testImagePath][0] + testImage = _find_test_image(testImagePath, parsed["images"]) expectAnnotationFile = "/train/labels/sharks_mp4-20_jpg.rf.5359121123e86e016401ea2731e47949.txt" assert testImage["annotationfile"]["file"] == expectAnnotationFile assert testImage["annotationfile"]["labelmap"] == {0: "fish", 1: "primary", 2: "shark"} @@ -48,7 +53,7 @@ def test_parse_mosquitos_csv(self): folder = f"{thisdir}/../datasets/mosquitos" parsed = folderparser.parsefolder(folder) testImagePath = "/train_10308.jpeg" - testImage = [i for i in parsed["images"] if i["file"] == testImagePath][0] + testImage = _find_test_image(testImagePath, parsed["images"]) assert testImage["annotationfile"]["name"] == "annotation.csv" expected = "img_fName,img_w,img_h,class_label,bbx_xtl,bbx_ytl,bbx_xbr,bbx_ybr\n" expected += "train_10308.jpeg,1058,943,japonicus/koreicus,28,187,908,815\n" @@ -58,7 +63,7 @@ def test_paligemma_format(self): folder = f"{thisdir}/../datasets/paligemma" parsed = folderparser.parsefolder(folder) testImagePath = "/dataset/de48275e1ff70fab78bee31e09fc896d_png.rf.01a97b1ad053aa1e6525ac0451cee8b7.jpg" - testImage = [i for i in parsed["images"] if i["file"] == testImagePath][0] + testImage = _find_test_image(testImagePath, parsed["images"]) assert testImage["annotationfile"]["name"] == "annotation.jsonl" expected = ( '{"image": "de48275e1ff70fab78bee31e09fc896d_png.rf.01a97b1ad053aa1e6525ac0451cee8b7.jpg",' From c4e36deba957d6bfa873b7b9f691572ba590670f Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 23 Apr 2025 18:42:16 +0200 Subject: [PATCH 08/11] fixup! fixup! fixup! fixup! fixup! Trying to make it cross platform --- tests/util/test_folderparser.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/util/test_folderparser.py b/tests/util/test_folderparser.py index 37929759..dc556686 100644 --- a/tests/util/test_folderparser.py +++ b/tests/util/test_folderparser.py @@ -7,9 +7,12 @@ thisdir = os.path.dirname(os.path.abspath(__file__)) +def patch_sep(filename): + return filename.replace("/", os.sep) + + def _find_test_image(image, images): - image = image.replace("/", os.sep) - return next(i for i in images if i["file"] == image) + return next(i for i in images if i["file"] == patch_sep(image)) class TestFolderParser(unittest.TestCase): @@ -22,14 +25,12 @@ def test_parse_sharks_coco(self): sharksfolder = f"{thisdir}/../datasets/sharks-tiny-coco" parsed = folderparser.parsefolder(sharksfolder) testImagePath = "/train/sharks_mp4-20_jpg.rf.90ba2e8e9ca0613f71359efb7ed48b26.jpg" - print("PARSED", parsed["images"]) testImage = _find_test_image(testImagePath, parsed["images"]) assert len(json.loads(testImage["annotationfile"]["rawText"])["annotations"]) == 5 def test_parse_sharks_createml(self): sharksfolder = f"{thisdir}/../datasets/sharks-tiny-createml" parsed = folderparser.parsefolder(sharksfolder) - print("PARSED", parsed["images"]) testImagePath = "/train/sharks_mp4-20_jpg.rf.5359121123e86e016401ea2731e47949.jpg" testImage = _find_test_image(testImagePath, parsed["images"]) imgParsedAnnotations = json.loads(testImage["annotationfile"]["rawText"]) @@ -43,7 +44,7 @@ def test(sharksfolder): testImagePath = "/train/images/sharks_mp4-20_jpg.rf.5359121123e86e016401ea2731e47949.jpg" testImage = _find_test_image(testImagePath, parsed["images"]) expectAnnotationFile = "/train/labels/sharks_mp4-20_jpg.rf.5359121123e86e016401ea2731e47949.txt" - assert testImage["annotationfile"]["file"] == expectAnnotationFile + assert testImage["annotationfile"]["file"] == patch_sep(expectAnnotationFile) assert testImage["annotationfile"]["labelmap"] == {0: "fish", 1: "primary", 2: "shark"} test(f"{thisdir}/../datasets/sharks-tiny-yolov9") From 930fe84e32d4488ebeff9ee8e306b9e3e3747026 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Thu, 24 Apr 2025 12:06:51 +0200 Subject: [PATCH 09/11] fixup! fixup! fixup! fixup! fixup! fixup! Trying to make it cross platform --- tests/util/test_folderparser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/util/test_folderparser.py b/tests/util/test_folderparser.py index dc556686..49bbe73e 100644 --- a/tests/util/test_folderparser.py +++ b/tests/util/test_folderparser.py @@ -44,6 +44,7 @@ def test(sharksfolder): testImagePath = "/train/images/sharks_mp4-20_jpg.rf.5359121123e86e016401ea2731e47949.jpg" testImage = _find_test_image(testImagePath, parsed["images"]) expectAnnotationFile = "/train/labels/sharks_mp4-20_jpg.rf.5359121123e86e016401ea2731e47949.txt" + print("TESTIMAGE", expectAnnotationFile, testImage) assert testImage["annotationfile"]["file"] == patch_sep(expectAnnotationFile) assert testImage["annotationfile"]["labelmap"] == {0: "fish", 1: "primary", 2: "shark"} From 55d39458b62a10734259bbc82da8a3d026d0a515 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Thu, 24 Apr 2025 12:18:13 +0200 Subject: [PATCH 10/11] fixup! fixup! fixup! fixup! fixup! fixup! fixup! Trying to make it cross platform --- roboflow/util/folderparser.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/roboflow/util/folderparser.py b/roboflow/util/folderparser.py index dc183588..4541860a 100644 --- a/roboflow/util/folderparser.py +++ b/roboflow/util/folderparser.py @@ -132,16 +132,16 @@ def _build_image_and_annotation_maps(annotationFiles): ) if parsedType == "coco": for imageRef in parsed["images"]: - imgRefMap[f"{filename}/{imageRef['file_name']}"] = imageRef + imgRefMap[os.path.join(filename, imageRef['file_name'])] = imageRef for annotation in parsed["annotations"]: - annotationMap[f"{dirname}/{annotation['image_id']}"].append(annotation) + annotationMap[os.path.join(dirname, annotation['image_id'])].append(annotation) return imgRefMap, annotationMap def _filterIndividualAnnotations(image, annotation, format, imgRefMap, annotationMap): parsed = annotation["parsed"] if format == "coco": - imgReference = imgRefMap.get(f"{annotation['file']}/{image['name']}") + imgReference = imgRefMap.get(os.path.join(annotation['file'], image['name'])) if imgReference: # workaround to make Annotations.js correctly identify this as coco in the backend fake_annotation = { @@ -153,7 +153,9 @@ def _filterIndividualAnnotations(image, annotation, format, imgRefMap, annotatio "iscrowd": 0, } _annotation = {"name": "annotation.coco.json"} - annotations_for_image = annotationMap.get(f"{image['dirname']}/{imgReference['id']}", []) + annotations_for_image = annotationMap.get( + os.path.join(image['dirname'], imgReference['id']), [] + ) _annotation["rawText"] = json.dumps( { "info": parsed["info"], @@ -255,7 +257,7 @@ def _map_labelmaps_to_annotations(annotations, labelmaps): if not labelmaps: return labelmapmap = {lm["dirname"]: lm for lm in labelmaps} - rootLabelmap = labelmapmap.get("/") + rootLabelmap = labelmapmap.get(os.sep) if len(labelmapmap) < len(labelmaps): print("warning: unexpectedly found multiple labelmaps per directory") print([lm["file"] for lm in labelmaps]) From 9c2b20e342bb00540ae491e742f9dca42fff97ef Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 24 Apr 2025 10:18:30 +0000 Subject: [PATCH 11/11] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- roboflow/util/folderparser.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/roboflow/util/folderparser.py b/roboflow/util/folderparser.py index 4541860a..286c3a56 100644 --- a/roboflow/util/folderparser.py +++ b/roboflow/util/folderparser.py @@ -132,16 +132,16 @@ def _build_image_and_annotation_maps(annotationFiles): ) if parsedType == "coco": for imageRef in parsed["images"]: - imgRefMap[os.path.join(filename, imageRef['file_name'])] = imageRef + imgRefMap[os.path.join(filename, imageRef["file_name"])] = imageRef for annotation in parsed["annotations"]: - annotationMap[os.path.join(dirname, annotation['image_id'])].append(annotation) + annotationMap[os.path.join(dirname, annotation["image_id"])].append(annotation) return imgRefMap, annotationMap def _filterIndividualAnnotations(image, annotation, format, imgRefMap, annotationMap): parsed = annotation["parsed"] if format == "coco": - imgReference = imgRefMap.get(os.path.join(annotation['file'], image['name'])) + imgReference = imgRefMap.get(os.path.join(annotation["file"], image["name"])) if imgReference: # workaround to make Annotations.js correctly identify this as coco in the backend fake_annotation = { @@ -153,9 +153,7 @@ def _filterIndividualAnnotations(image, annotation, format, imgRefMap, annotatio "iscrowd": 0, } _annotation = {"name": "annotation.coco.json"} - annotations_for_image = annotationMap.get( - os.path.join(image['dirname'], imgReference['id']), [] - ) + annotations_for_image = annotationMap.get(os.path.join(image["dirname"], imgReference["id"]), []) _annotation["rawText"] = json.dumps( { "info": parsed["info"],