diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 70f0f2a7..570da3d1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,10 +8,13 @@ on: jobs: build: - runs-on: ubuntu-latest strategy: matrix: + os: ["ubuntu-latest", "windows-latest"] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + runs-on: ${{ matrix.os }} + env: + PYTHONUTF8: 1 steps: - name: 🛎️ Checkout diff --git a/roboflow/util/folderparser.py b/roboflow/util/folderparser.py index eff2a118..286c3a56 100644 --- a/roboflow/util/folderparser.py +++ b/roboflow/util/folderparser.py @@ -13,9 +13,8 @@ def parsefolder(folder): - folder = folder.strip() - if folder.endswith("/"): - folder = folder[:-1] + # Remove trailing spaces and slashes + folder = folder.strip().rstrip("\\/") if not os.path.exists(folder): raise Exception(f"folder does not exist. {folder}") files = _list_files(folder) @@ -53,8 +52,12 @@ def _list_files(folder): for root, dirs, files in os.walk(folder): for file in files: file_path = os.path.join(root, file) - filedescriptors.append(_describe_file(file_path.split(folder)[1])) - filedescriptors = sorted(filedescriptors, key=lambda x: _alphanumkey(x["file"])) + rel = os.path.relpath(file_path, folder) + filedescriptors.append(_describe_file(f"{os.sep}{rel}")) + filedescriptors = sorted( + filedescriptors, + key=lambda x: _alphanumkey(x["file"]), + ) return filedescriptors @@ -64,10 +67,10 @@ def _add_indices(files): def _describe_file(f): - name = f.split("/")[-1] + name = os.path.basename(f) dirname = os.path.dirname(f) fullkey, extension = os.path.splitext(f) - fullkey2 = fullkey.replace("/labels", "").replace("/images", "") + fullkey2 = fullkey.replace(f"{os.sep}labels", "").replace(f"{os.sep}images", "") key = os.path.splitext(name)[0] return { "file": f, @@ -129,16 +132,16 @@ def _build_image_and_annotation_maps(annotationFiles): ) if parsedType == "coco": for imageRef in parsed["images"]: - imgRefMap[f"{filename}/{imageRef['file_name']}"] = imageRef + imgRefMap[os.path.join(filename, imageRef["file_name"])] = imageRef for annotation in parsed["annotations"]: - annotationMap[f"{dirname}/{annotation['image_id']}"].append(annotation) + annotationMap[os.path.join(dirname, annotation["image_id"])].append(annotation) return imgRefMap, annotationMap def _filterIndividualAnnotations(image, annotation, format, imgRefMap, annotationMap): parsed = annotation["parsed"] if format == "coco": - imgReference = imgRefMap.get(f"{annotation['file']}/{image['name']}") + imgReference = imgRefMap.get(os.path.join(annotation["file"], image["name"])) if imgReference: # workaround to make Annotations.js correctly identify this as coco in the backend fake_annotation = { @@ -150,7 +153,7 @@ def _filterIndividualAnnotations(image, annotation, format, imgRefMap, annotatio "iscrowd": 0, } _annotation = {"name": "annotation.coco.json"} - annotations_for_image = annotationMap.get(f"{image['dirname']}/{imgReference['id']}", []) + annotations_for_image = annotationMap.get(os.path.join(image["dirname"], imgReference["id"]), []) _annotation["rawText"] = json.dumps( { "info": parsed["info"], @@ -252,7 +255,7 @@ def _map_labelmaps_to_annotations(annotations, labelmaps): if not labelmaps: return labelmapmap = {lm["dirname"]: lm for lm in labelmaps} - rootLabelmap = labelmapmap.get("/") + rootLabelmap = labelmapmap.get(os.sep) if len(labelmapmap) < len(labelmaps): print("warning: unexpectedly found multiple labelmaps per directory") print([lm["file"] for lm in labelmaps]) diff --git a/tests/test_version.py b/tests/test_version.py index f13479bc..3697e1f5 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -59,7 +59,7 @@ def test_download_returns_dataset(self, *_): self.assertEqual(dataset.name, self.version.name) self.assertEqual(dataset.version, self.version.version) self.assertEqual(dataset.model_format, "coco") - self.assertEqual(dataset.location, "/my-spot") + self.assertEqual(dataset.location, os.path.abspath("/my-spot")) class TestExport(unittest.TestCase): diff --git a/tests/util/test_folderparser.py b/tests/util/test_folderparser.py index d4d497cb..49bbe73e 100644 --- a/tests/util/test_folderparser.py +++ b/tests/util/test_folderparser.py @@ -1,10 +1,18 @@ import json +import os import unittest -from os.path import abspath, dirname from roboflow.util import folderparser -thisdir = dirname(abspath(__file__)) +thisdir = os.path.dirname(os.path.abspath(__file__)) + + +def patch_sep(filename): + return filename.replace("/", os.sep) + + +def _find_test_image(image, images): + return next(i for i in images if i["file"] == patch_sep(image)) class TestFolderParser(unittest.TestCase): @@ -17,14 +25,14 @@ def test_parse_sharks_coco(self): sharksfolder = f"{thisdir}/../datasets/sharks-tiny-coco" parsed = folderparser.parsefolder(sharksfolder) testImagePath = "/train/sharks_mp4-20_jpg.rf.90ba2e8e9ca0613f71359efb7ed48b26.jpg" - testImage = [i for i in parsed["images"] if i["file"] == testImagePath][0] + testImage = _find_test_image(testImagePath, parsed["images"]) assert len(json.loads(testImage["annotationfile"]["rawText"])["annotations"]) == 5 def test_parse_sharks_createml(self): sharksfolder = f"{thisdir}/../datasets/sharks-tiny-createml" parsed = folderparser.parsefolder(sharksfolder) testImagePath = "/train/sharks_mp4-20_jpg.rf.5359121123e86e016401ea2731e47949.jpg" - testImage = [i for i in parsed["images"] if i["file"] == testImagePath][0] + testImage = _find_test_image(testImagePath, parsed["images"]) imgParsedAnnotations = json.loads(testImage["annotationfile"]["rawText"]) assert len(imgParsedAnnotations) == 1 imgReference = imgParsedAnnotations[0] @@ -34,9 +42,10 @@ def test_parse_sharks_yolov9(self): def test(sharksfolder): parsed = folderparser.parsefolder(sharksfolder) testImagePath = "/train/images/sharks_mp4-20_jpg.rf.5359121123e86e016401ea2731e47949.jpg" - testImage = [i for i in parsed["images"] if i["file"] == testImagePath][0] + testImage = _find_test_image(testImagePath, parsed["images"]) expectAnnotationFile = "/train/labels/sharks_mp4-20_jpg.rf.5359121123e86e016401ea2731e47949.txt" - assert testImage["annotationfile"]["file"] == expectAnnotationFile + print("TESTIMAGE", expectAnnotationFile, testImage) + assert testImage["annotationfile"]["file"] == patch_sep(expectAnnotationFile) assert testImage["annotationfile"]["labelmap"] == {0: "fish", 1: "primary", 2: "shark"} test(f"{thisdir}/../datasets/sharks-tiny-yolov9") @@ -46,7 +55,7 @@ def test_parse_mosquitos_csv(self): folder = f"{thisdir}/../datasets/mosquitos" parsed = folderparser.parsefolder(folder) testImagePath = "/train_10308.jpeg" - testImage = [i for i in parsed["images"] if i["file"] == testImagePath][0] + testImage = _find_test_image(testImagePath, parsed["images"]) assert testImage["annotationfile"]["name"] == "annotation.csv" expected = "img_fName,img_w,img_h,class_label,bbx_xtl,bbx_ytl,bbx_xbr,bbx_ybr\n" expected += "train_10308.jpeg,1058,943,japonicus/koreicus,28,187,908,815\n" @@ -56,7 +65,7 @@ def test_paligemma_format(self): folder = f"{thisdir}/../datasets/paligemma" parsed = folderparser.parsefolder(folder) testImagePath = "/dataset/de48275e1ff70fab78bee31e09fc896d_png.rf.01a97b1ad053aa1e6525ac0451cee8b7.jpg" - testImage = [i for i in parsed["images"] if i["file"] == testImagePath][0] + testImage = _find_test_image(testImagePath, parsed["images"]) assert testImage["annotationfile"]["name"] == "annotation.jsonl" expected = ( '{"image": "de48275e1ff70fab78bee31e09fc896d_png.rf.01a97b1ad053aa1e6525ac0451cee8b7.jpg",'