diff --git a/.gitattributes b/.gitattributes index d448f5b..1243391 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ src/resolver_athena_client/generated/* linguist-generated=true +tests/functional/e2e/testcases/**/*.jpg filter=lfs diff=lfs merge=lfs -text diff --git a/tests/functional/e2e/__init__.py b/tests/functional/e2e/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional/e2e/test_classify_single.py b/tests/functional/e2e/test_classify_single.py new file mode 100644 index 0000000..867195f --- /dev/null +++ b/tests/functional/e2e/test_classify_single.py @@ -0,0 +1,69 @@ +from pathlib import Path + +import pytest + +from resolver_athena_client.client.athena_client import AthenaClient +from resolver_athena_client.client.athena_options import AthenaOptions +from resolver_athena_client.client.channel import ( + CredentialHelper, + create_channel_with_credentials, +) +from resolver_athena_client.client.models import ImageData +from tests.functional.e2e.testcases.parser import ( + AthenaTestCase, + load_test_cases, +) + +TEST_CASES = load_test_cases("benign_model") + +FP_ERROR_TOLERANCE = 1e-4 + + +@pytest.mark.asyncio +@pytest.mark.functional +@pytest.mark.parametrize("test_case", TEST_CASES, ids=lambda tc: tc.id) +async def test_classify_single( + athena_options: AthenaOptions, + credential_helper: CredentialHelper, + test_case: AthenaTestCase, +) -> None: + """Functional test for ClassifySingle endpoint and API methods. + + This test creates a unique test image for each iteration and classifies it. + The test runs multiple iterations to ensure consistent behavior. + """ + + # Create gRPC channel with credentials + channel = await create_channel_with_credentials( + athena_options.host, credential_helper + ) + with Path.open(Path(test_case.filepath), "rb") as f: + image_bytes = f.read() + + async with AthenaClient(channel, athena_options) as client: + image_data = ImageData(image_bytes) + + # Classify with auto-generated correlation ID + result = await client.classify_single(image_data) + + if result.error.code: + msg = f"Image Result Error: {result.error.message}" + pytest.fail(msg) + + actual_output = {c.label: c.weight for c in result.classifications} + assert set(test_case.expected_output.keys()).issubset( + set(actual_output.keys()) + ), ( + "Expected output to contain labels: ", + f"{test_case.expected_output.keys() - actual_output.keys()}", + ) + + max_diff = max( + abs(test_case.expected_output[label] - actual_output[label]) + for label in test_case.expected_output + ) + assert max_diff < FP_ERROR_TOLERANCE, ( + "Output weights differ from expected by more than", + f" {FP_ERROR_TOLERANCE}: ", + f"expected={test_case.expected_output}, actual={actual_output}", + ) diff --git a/tests/functional/e2e/testcases/__init__.py b/tests/functional/e2e/testcases/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional/e2e/testcases/benign_model/expected_outputs.json b/tests/functional/e2e/testcases/benign_model/expected_outputs.json new file mode 100644 index 0000000..e92de3d --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/expected_outputs.json @@ -0,0 +1,573 @@ +{ + "classification_labels": [ + "Lakeside", + "Mountains", + "Trees", + "Seashore", + "Ducks", + "Goose" + ], + "images": [ + [ + "mountains/pexels-pixabay-210243.jpg", + [ + 0.0, + 0.9304999709129333, + 0.0, + 0.0, + 0.0, + 0.06949999928474426 + ] + ], + [ + "mountains/pexels-pixabay-326235.jpg", + [ + 0.0, + 0.9350000023841858, + 0.0, + 0.0, + 0.0, + 0.06499999761581421 + ] + ], + [ + "mountains/pexels-audio-visual-art-9849264.jpg", + [ + 0.0, + 0.85589998960495, + 0.0, + 0.0, + 0.0, + 0.14409999549388885 + ] + ], + [ + "mountains/pexels-joyston-judah-331625-933054.jpg", + [ + 0.0, + 0.9096999764442444, + 0.0, + 0.0, + 0.0, + 0.09030000120401382 + ] + ], + [ + "mountains/pexels-suketdedhia-671659.jpg", + [ + 0.0, + 0.913100004196167, + 0.0, + 0.0, + 0.0, + 0.0869000032544136 + ] + ], + [ + "mountains/pexels-eberhardgross-1303431.jpg", + [ + 0.0, + 0.9350000023841858, + 0.0, + 0.0, + 0.0, + 0.06499999761581421 + ] + ], + [ + "mountains/pexels-dreamypixel-547115.jpg", + [ + 0.0, + 0.5285999774932861, + 0.0, + 0.4564000070095062, + 0.0, + 0.014999999664723873 + ] + ], + [ + "mountains/pexels-riciardus-789380.jpg", + [ + 0.0, + 0.6215000152587891, + 0.25040000677108765, + 0.0, + 0.03680000081658363, + 0.09130000323057175 + ] + ], + [ + "trees/pexels-jakkel-418831.jpg", + [ + 0.0, + 0.0, + 0.838699996471405, + 0.0, + 0.0, + 0.16130000352859497 + ] + ], + [ + "trees/pexels-luisdalvan-1770809.jpg", + [ + 0.0, + 0.0, + 0.9559999704360962, + 0.0, + 0.0, + 0.04399999976158142 + ] + ], + [ + "trees/pexels-veeterzy-38136.jpg", + [ + 0.0, + 0.0, + 0.9815000295639038, + 0.0, + 0.0, + 0.01850000023841858 + ] + ], + [ + "trees/pexels-minan1398-1006121.jpg", + [ + 0.0, + 0.0, + 0.9172000288963318, + 0.0, + 0.0, + 0.0828000009059906 + ] + ], + [ + "trees/pexels-umkreisel-app-957024.jpg", + [ + 0.0, + 0.0, + 0.9758999943733215, + 0.0, + 0.0, + 0.02410000003874302 + ] + ], + [ + "trees/pexels-pixabay-101529.jpg", + [ + 0.0, + 0.0, + 0.9096999764442444, + 0.0, + 0.0, + 0.09030000120401382 + ] + ], + [ + "trees/pexels-pixabay-33109.jpg", + [ + 0.0, + 0.0, + 0.902400016784668, + 0.0, + 0.0, + 0.09759999811649323 + ] + ], + [ + "trees/pexels-packermann-1666021.jpg", + [ + 0.17499999701976776, + 0.6215000152587891, + 0.03680000081658363, + 0.0, + 0.0, + 0.16670000553131104 + ] + ], + [ + "ducks/pexels-pixabay-64225.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.5285999774932861, + 0.4713999927043915 + ] + ], + [ + "ducks/pexels-weekendplayer-724372.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.17499999701976776, + 0.824999988079071 + ] + ], + [ + "ducks/pexels-aidanj-660266.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.847100019454956, + 0.15289999544620514 + ] + ], + [ + "ducks/pexels-pixabay-112548.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.838699996471405, + 0.16130000352859497 + ] + ], + [ + "ducks/pexels-kasperphotography-998989.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + "ducks/pexels-pixabay-162316.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.8314999938011169, + 0.16850000619888306 + ] + ], + [ + "ducks/pexels-jeswin-2020323.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.8314999938011169, + 0.16850000619888306 + ] + ], + [ + "ducks/pexels-theotherkev-2264552.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.03680000081658363, + 0.9631999731063843 + ] + ], + [ + "goose/pexels-kevin-caicedo-601330023-17826550.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.03680000081658363, + 0.9631999731063843 + ] + ], + [ + "goose/pexels-pixabay-53531.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + "goose/pexels-pixabay-55832.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + "goose/pexels-ragasirtahk-730278.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + "goose/pexels-orlovamaria-4947364.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + "goose/pexels-brett-sayles-2009775.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + "goose/pexels-sugarpose-33189129.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + "goose/pexels-brett-sayles-1143924.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + "seashore/pexels-fabianwiktor-994605.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.902400016784668, + 0.0, + 0.09759999811649323 + ] + ], + [ + "seashore/pexels-conojeghuo-127673.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.85589998960495, + 0.0, + 0.14409999549388885 + ] + ], + [ + "seashore/pexels-asadphoto-240526.jpg", + [ + 0.0, + 0.0, + 0.03680000081658363, + 0.838699996471405, + 0.0, + 0.12449999898672104 + ] + ], + [ + "seashore/pexels-pok-rie-33563-1726310.jpg", + [ + 0.03680000081658363, + 0.03680000081658363, + 0.12229999899864197, + 0.6215000152587891, + 0.03680000081658363, + 0.14579999446868896 + ] + ], + [ + "seashore/pexels-julia-kuzenkov-442028-1974521.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.9350000023841858, + 0.0, + 0.06499999761581421 + ] + ], + [ + "seashore/pexels-asadphoto-1450360.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.8226000070571899, + 0.0, + 0.17739999294281006 + ] + ], + [ + "seashore/pexels-pixabay-221471.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.9617000222206116, + 0.0, + 0.03830000013113022 + ] + ], + [ + "seashore/pexels-photoklickr-78359-244517.jpg", + [ + 0.0, + 0.0, + 0.0, + 0.8837000131607056, + 0.0, + 0.11630000174045563 + ] + ], + [ + "lakeside/pexels-galerieb-2301172.jpg", + [ + 0.3450999855995178, + 0.5285999774932861, + 0.04919999837875366, + 0.0, + 0.0, + 0.0771000012755394 + ] + ], + [ + "lakeside/pexels-souvenirpixels-1396740.jpg", + [ + 0.17499999701976776, + 0.25040000677108765, + 0.03680000081658363, + 0.25040000677108765, + 0.0, + 0.2874000072479248 + ] + ], + [ + "lakeside/pexels-souvenirpixels-1578751.jpg", + [ + 0.6274999976158142, + 0.06840000301599503, + 0.03680000081658363, + 0.0, + 0.03680000081658363, + 0.2304999977350235 + ] + ], + [ + "lakeside/pexels-pixabay-221395.jpg", + [ + 0.25040000677108765, + 0.6215000152587891, + 0.0, + 0.0, + 0.0, + 0.12809999287128448 + ] + ], + [ + "lakeside/pexels-liger-pham-232622-1108701.jpg", + [ + 0.7907000184059143, + 0.03680000081658363, + 0.0, + 0.03680000081658363, + 0.0, + 0.13570000231266022 + ] + ], + [ + "lakeside/pexels-dongdilac-30871071.jpg", + [ + 0.8198000192642212, + 0.0, + 0.03680000081658363, + 0.0, + 0.0, + 0.14339999854564667 + ] + ], + [ + "lakeside/pexels-kalju-liiv-118775-1548181.jpg", + [ + 0.6542999744415283, + 0.04919999837875366, + 0.0, + 0.03680000081658363, + 0.0, + 0.2597000002861023 + ] + ], + [ + "lakeside/pexels-pixabay-210288.jpg", + [ + 0.9258999824523926, + 0.0, + 0.0, + 0.0, + 0.0, + 0.07410000264644623 + ] + ], + [ + "lakeside/pexels-pixabay-210290.jpg", + [ + 0.9258999824523926, + 0.0, + 0.0, + 0.0, + 0.0, + 0.07410000264644623 + ] + ], + [ + "lakeside/pexels-pixabay-210289.jpg", + [ + 0.9258999824523926, + 0.0, + 0.0, + 0.0, + 0.0, + 0.07410000264644623 + ] + ], + [ + "lakeside/pexels-pixabay-210291.jpg", + [ + 0.9258999824523926, + 0.0, + 0.0, + 0.0, + 0.0, + 0.07410000264644623 + ] + ] + ] +} \ No newline at end of file diff --git a/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-aidanj-660266.jpg b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-aidanj-660266.jpg new file mode 100644 index 0000000..e546a3c --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-aidanj-660266.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:581cf5b937320696d301003d45037d6b9c4be4580a5c4a975f1b5b237adbc241 +size 627287 diff --git a/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-jeswin-2020323.jpg b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-jeswin-2020323.jpg new file mode 100644 index 0000000..ed0b1c0 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-jeswin-2020323.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb8dd8f361532788ad57b56bab63a31cb99fd3a09233773b6aa536debed42f6f +size 562626 diff --git a/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-kasperphotography-998989.jpg b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-kasperphotography-998989.jpg new file mode 100644 index 0000000..9429e2c --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-kasperphotography-998989.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b6ef2b28349b45c04f2811726bc37c9c99a2fdafdd7c2a56f6cd546f939f91f +size 566713 diff --git a/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-pixabay-112548.jpg b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-pixabay-112548.jpg new file mode 100644 index 0000000..c9e869d --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-pixabay-112548.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d2c14ff42f9d8182176857bc9cbf785991d63c019221c486e36eca705da73b4 +size 324647 diff --git a/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-pixabay-162316.jpg b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-pixabay-162316.jpg new file mode 100644 index 0000000..d38aa15 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-pixabay-162316.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:464edf442fb4766c7f869f7166e7bcfb433fc5aabd036508cac057e4785c522b +size 1150727 diff --git a/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-pixabay-64225.jpg b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-pixabay-64225.jpg new file mode 100644 index 0000000..2d2beda --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-pixabay-64225.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44b5daaafd728e588bc4f0e9eea1c4211bd9fda7fbdf95bd963e444081f597d2 +size 791187 diff --git a/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-theotherkev-2264552.jpg b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-theotherkev-2264552.jpg new file mode 100644 index 0000000..52ee990 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-theotherkev-2264552.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8df2f13117c0dd65c3a135768506358f4606a86877c9c0400f627ce531b640d +size 557890 diff --git a/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-weekendplayer-724372.jpg b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-weekendplayer-724372.jpg new file mode 100644 index 0000000..b84b80c --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/ducks/pexels-weekendplayer-724372.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec65dc1ace257c3af1431e88f2dd30520ce8b74562b53f17026e9368db593777 +size 1439612 diff --git a/tests/functional/e2e/testcases/benign_model/images/goose/pexels-brett-sayles-1143924.jpg b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-brett-sayles-1143924.jpg new file mode 100644 index 0000000..0104db1 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-brett-sayles-1143924.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ac4ad10cb47a694eeb300e3811f54174acff736ced12ab26a795ef2fdd9efa6 +size 2250406 diff --git a/tests/functional/e2e/testcases/benign_model/images/goose/pexels-brett-sayles-2009775.jpg b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-brett-sayles-2009775.jpg new file mode 100644 index 0000000..e27debd --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-brett-sayles-2009775.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b554b40212e4e77512caf87baebef050cc5c7761e8b7b2daac14bdff71e3696 +size 1159840 diff --git a/tests/functional/e2e/testcases/benign_model/images/goose/pexels-kevin-caicedo-601330023-17826550.jpg b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-kevin-caicedo-601330023-17826550.jpg new file mode 100644 index 0000000..25d75c5 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-kevin-caicedo-601330023-17826550.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23b4d3f5dc3ea43b0b07d7ffdc51b1262dac40574aa0a753efabd71b44c45882 +size 797125 diff --git a/tests/functional/e2e/testcases/benign_model/images/goose/pexels-orlovamaria-4947364.jpg b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-orlovamaria-4947364.jpg new file mode 100644 index 0000000..f6ee3e8 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-orlovamaria-4947364.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85c52adac35724ff7e6b1c77540b69135f9f8dd5408bd4f1067e35cb76cffd9a +size 1133237 diff --git a/tests/functional/e2e/testcases/benign_model/images/goose/pexels-pixabay-53531.jpg b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-pixabay-53531.jpg new file mode 100644 index 0000000..8092b64 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-pixabay-53531.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bbc4d6144486f65b1a25e4a78f14b201dcd3ca25b27a952f41cbd7ede7563d6 +size 311763 diff --git a/tests/functional/e2e/testcases/benign_model/images/goose/pexels-pixabay-55832.jpg b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-pixabay-55832.jpg new file mode 100644 index 0000000..8e8ea10 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-pixabay-55832.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2862051f32dd9c75a759a2290236eba2d8f03862af581c7d83a5fcd249946947 +size 444207 diff --git a/tests/functional/e2e/testcases/benign_model/images/goose/pexels-ragasirtahk-730278.jpg b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-ragasirtahk-730278.jpg new file mode 100644 index 0000000..109c533 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-ragasirtahk-730278.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43835947bb249da2562a5546913c6b190fcb965f859fc8436839c2622e32646e +size 668721 diff --git a/tests/functional/e2e/testcases/benign_model/images/goose/pexels-sugarpose-33189129.jpg b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-sugarpose-33189129.jpg new file mode 100644 index 0000000..e204e2e --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/goose/pexels-sugarpose-33189129.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d96b3c05b0ebc3369f60e629c440eb5895b57d87e831ea21e00995589d60f5f +size 1475698 diff --git a/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-dongdilac-30871071.jpg b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-dongdilac-30871071.jpg new file mode 100644 index 0000000..120d5c0 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-dongdilac-30871071.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:314b6b29d8a64c59c1e1dc7380af9553b52fc905e13ef6947793ed49c5343c1d +size 3364263 diff --git a/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-galerieb-2301172.jpg b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-galerieb-2301172.jpg new file mode 100644 index 0000000..9d231ea --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-galerieb-2301172.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bbb95b75eb4c823bdda61701942c59a6cae84007da782c4688413e3d56b7faa +size 1859534 diff --git a/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-kalju-liiv-118775-1548181.jpg b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-kalju-liiv-118775-1548181.jpg new file mode 100644 index 0000000..6cc45f2 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-kalju-liiv-118775-1548181.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3965e652acfe6bc04eef3c2b25e42ec6b19bb94d5caf4d7bc457bc77941d646e +size 1518816 diff --git a/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-liger-pham-232622-1108701.jpg b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-liger-pham-232622-1108701.jpg new file mode 100644 index 0000000..b3bd4ed --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-liger-pham-232622-1108701.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bd1dc8dfebf31d55e72904d4484cfa68484943115241be7f3633defb132138f +size 1316127 diff --git a/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-pixabay-210288.jpg b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-pixabay-210288.jpg new file mode 100644 index 0000000..0596fb0 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-pixabay-210288.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6b08c324c4f70ed4ba13b2e579350d78fe563103bc9aa3dfe297c5710781556 +size 844521 diff --git a/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-pixabay-221395.jpg b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-pixabay-221395.jpg new file mode 100644 index 0000000..506c15a --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-pixabay-221395.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d1fcdc594c53f021d5d426e15fd7371497561bdc502754cf06831df8d147c27 +size 1222610 diff --git a/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-souvenirpixels-1396740.jpg b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-souvenirpixels-1396740.jpg new file mode 100644 index 0000000..7dc991c --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-souvenirpixels-1396740.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2174f2e42c51ad0802e5238002a4b3a721c939f6ceb62e1f0b1fe46e9ae3b958 +size 843693 diff --git a/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-souvenirpixels-1578751.jpg b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-souvenirpixels-1578751.jpg new file mode 100644 index 0000000..797a10c --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/lakeside/pexels-souvenirpixels-1578751.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62202ad15041df0ac9a01b9f6f9c2289ec3fca3506b8058553009c37063e8375 +size 763322 diff --git a/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-audio-visual-art-9849264.jpg b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-audio-visual-art-9849264.jpg new file mode 100644 index 0000000..8d53980 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-audio-visual-art-9849264.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5fe4db9610dd838209d24dce13f56954e3bbc575ade4735d565aa21c4edab31 +size 1082218 diff --git a/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-dreamypixel-547115.jpg b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-dreamypixel-547115.jpg new file mode 100644 index 0000000..17002ef --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-dreamypixel-547115.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c948519b798f5ad6f23832203905c47460839eecac4049b2626ae8a0886a8b7a +size 1360143 diff --git a/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-eberhardgross-1303431.jpg b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-eberhardgross-1303431.jpg new file mode 100644 index 0000000..dac4eb4 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-eberhardgross-1303431.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:247b018cd32e3c1ece1d79f0f6b6b15e29fd3403b4019971006f641aed86bfd4 +size 1235474 diff --git a/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-joyston-judah-331625-933054.jpg b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-joyston-judah-331625-933054.jpg new file mode 100644 index 0000000..b03d8d5 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-joyston-judah-331625-933054.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c51cb3bee5a7012ada4e9f74622fd7ae780df45916c34e8d13c2aefbc302372c +size 235414 diff --git a/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-pixabay-210243.jpg b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-pixabay-210243.jpg new file mode 100644 index 0000000..4665f95 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-pixabay-210243.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:240a803d85b05f30ca8bf3bac98ba44610689d54e88f4898772a2ad4acb5b7bf +size 503519 diff --git a/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-pixabay-326235.jpg b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-pixabay-326235.jpg new file mode 100644 index 0000000..772e8d0 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-pixabay-326235.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99d0c47c0d199fe1742299f7b7ca6a79f2615ae27ecf6fc97557ce158df28008 +size 1226113 diff --git a/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-riciardus-789380.jpg b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-riciardus-789380.jpg new file mode 100644 index 0000000..2020e5f --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-riciardus-789380.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c566962c41c6b9ec11e025e5bf7eb772d47dd972cf4418b5aa1bf446c283f288 +size 1692155 diff --git a/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-suketdedhia-671658.jpg b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-suketdedhia-671658.jpg new file mode 100644 index 0000000..7e40524 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/mountains/pexels-suketdedhia-671658.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:300b938dde11160520bb0dacb595b23abb49ae3590eb47ef361893d9df4f08b8 +size 1618048 diff --git a/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-asadphoto-1450360.jpg b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-asadphoto-1450360.jpg new file mode 100644 index 0000000..20b1542 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-asadphoto-1450360.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54c7a6e867b273fac86eead859e466dde44f350e6fd349b35da4a2793783770c +size 4335903 diff --git a/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-asadphoto-240526.jpg b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-asadphoto-240526.jpg new file mode 100644 index 0000000..d91ca26 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-asadphoto-240526.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b21eed9fc2c78eeab9f3500925230652a356d4181b0878e0cf3a1f6f394d1d17 +size 1064621 diff --git a/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-conojeghuo-127673.jpg b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-conojeghuo-127673.jpg new file mode 100644 index 0000000..bb5f1b9 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-conojeghuo-127673.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8533ca6c6742904c3f8cbb77b6a924e7bbc1187701bc7cd033cf35b1f043d95 +size 1553777 diff --git a/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-fabianwiktor-994605.jpg b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-fabianwiktor-994605.jpg new file mode 100644 index 0000000..bd98392 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-fabianwiktor-994605.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20e445dde30fa210ef79fb8f3a463cedcb29b983459d1781412f5966333004da +size 790401 diff --git a/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-julia-kuzenkov-442028-1974521.jpg b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-julia-kuzenkov-442028-1974521.jpg new file mode 100644 index 0000000..ae929b1 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-julia-kuzenkov-442028-1974521.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d236a139146ef9a2067607be153d53ce3934d973316450a1336a17bff410be8 +size 1534452 diff --git a/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-photoklickr-78359-244517.jpg b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-photoklickr-78359-244517.jpg new file mode 100644 index 0000000..1bb6650 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-photoklickr-78359-244517.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:358f92de361759662dc1f43ca0357f3d685c61a93c882d238f77249b50a284e5 +size 776559 diff --git a/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-pixabay-221471.jpg b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-pixabay-221471.jpg new file mode 100644 index 0000000..a727153 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-pixabay-221471.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d0fe9a674dfb2dd4b1c768dea39730dfe6336239292d1704adcf6feb3357a43 +size 251582 diff --git a/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-pok-rie-33563-1726310.jpg b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-pok-rie-33563-1726310.jpg new file mode 100644 index 0000000..b6072e3 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/seashore/pexels-pok-rie-33563-1726310.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90c6169c51935cc78febbf407cfcc596aa3cbb3dbb0ee6fbd935878c8942aeec +size 2400795 diff --git a/tests/functional/e2e/testcases/benign_model/images/trees/pexels-jakkel-418831.jpg b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-jakkel-418831.jpg new file mode 100644 index 0000000..57822ea --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-jakkel-418831.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3abe457a6e03c3d6151bbaf91d993e79e84e063e7630ced6eb7abc318bf143ac +size 3821224 diff --git a/tests/functional/e2e/testcases/benign_model/images/trees/pexels-luisdalvan-1770809.jpg b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-luisdalvan-1770809.jpg new file mode 100644 index 0000000..c380668 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-luisdalvan-1770809.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88927f37fdf3b02c69891153f64dabd26e02be79936cdd8e6a9d7a720aeea14a +size 3532518 diff --git a/tests/functional/e2e/testcases/benign_model/images/trees/pexels-minan1398-1006121.jpg b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-minan1398-1006121.jpg new file mode 100644 index 0000000..954de40 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-minan1398-1006121.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf9a8593b587f7ba857fc03f631ee0648aa67a7755a5d4775673c58241601011 +size 2236774 diff --git a/tests/functional/e2e/testcases/benign_model/images/trees/pexels-packermann-1666021.jpg b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-packermann-1666021.jpg new file mode 100644 index 0000000..4bc7e98 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-packermann-1666021.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7db9a069737ae5a9d7f31890a6a3514d77178002fcd4a1de88c7618f3f24930 +size 2588416 diff --git a/tests/functional/e2e/testcases/benign_model/images/trees/pexels-pixabay-101529.jpg b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-pixabay-101529.jpg new file mode 100644 index 0000000..530610f --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-pixabay-101529.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e39921b35f5983f533f9ecc2a208517b8046189309b127cdc78212eb39a3226 +size 908869 diff --git a/tests/functional/e2e/testcases/benign_model/images/trees/pexels-pixabay-33109.jpg b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-pixabay-33109.jpg new file mode 100644 index 0000000..7631ab6 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-pixabay-33109.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e6eff0edfef18572a278546cb5af26011afb1de1a952dcf9289775c8c31f0f2 +size 2396026 diff --git a/tests/functional/e2e/testcases/benign_model/images/trees/pexels-umkreisel-app-957024.jpg b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-umkreisel-app-957024.jpg new file mode 100644 index 0000000..ac68e38 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-umkreisel-app-957024.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2965f0f7b5e7ea90d1bb040d0a98b730fbf010b90d37ac6773a0dc9b8ab7b47b +size 7868847 diff --git a/tests/functional/e2e/testcases/benign_model/images/trees/pexels-veeterzy-38136.jpg b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-veeterzy-38136.jpg new file mode 100644 index 0000000..9b43682 --- /dev/null +++ b/tests/functional/e2e/testcases/benign_model/images/trees/pexels-veeterzy-38136.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db2bb4601eb1566dec7cc1da9d11bb02a04c7cc913ae85d6a6e8251a26f0b61e +size 4411358 diff --git a/tests/functional/e2e/testcases/parser.py b/tests/functional/e2e/testcases/parser.py new file mode 100644 index 0000000..a76a0fd --- /dev/null +++ b/tests/functional/e2e/testcases/parser.py @@ -0,0 +1,44 @@ +import json +from pathlib import Path + +# These files were NOT in the original Roke test set and were added later so +# manually excluded from tests. +EXCLUDED_FILENAMES = [ + "mountains/pexels-suketdedhia-671659.jpg", + "lakeside/pexels-pixabay-210289.jpg", + "lakeside/pexels-pixabay-210290.jpg", + "lakeside/pexels-pixabay-210291.jpg", +] + + +class AthenaTestCase: + def __init__( + self, + filepath: str, + expected_output: list[float], + classification_labels: list[str], + ) -> None: + self.id: str = "/".join( + filepath.split("/")[-2:] + ) # e.g. "ducks/duck1.jpg" + self.filepath: str = filepath + self.expected_output: dict[str, float] = dict( + zip(classification_labels, expected_output, strict=True) + ) + self.classification_labels: list[str] = classification_labels + + +def load_test_cases(dirname: str = "benign_model") -> list[AthenaTestCase]: + with Path.open( + Path(Path(__file__).parent / dirname / "expected_outputs.json"), + ) as f: + test_cases = json.load(f) + return [ + AthenaTestCase( + str(Path(Path(__file__).parent / dirname / "images" / item[0])), + item[1], + test_cases["classification_labels"], + ) + for item in test_cases["images"] + if item[0] not in EXCLUDED_FILENAMES + ]