From 8e288ab071d141dbe4448b420966edb039d05105 Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Thu, 18 Dec 2025 10:51:26 -0600 Subject: [PATCH 1/2] tests: cache and verify hash value of sample images --- .github/workflows/build_wheels.yml | 10 +++++ .github/workflows/tox_matrix.yml | 17 +++++-- Jenkinsfile | 2 +- ci/docker/linux/jenkins/Dockerfile | 13 +++++- ci/docker/linux/tox/Dockerfile | 18 +++++++- scripts/resources/windows/tox/Dockerfile | 17 ++++++- tests/CMakeLists.txt | 24 +++++++--- tests/conftest.py | 56 ++++++++++++++++-------- tox.ini | 1 + 9 files changed, 125 insertions(+), 33 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 6769d6b8..6e82bd9a 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -31,6 +31,15 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Cache sample files + id: cache_samples + uses: actions/cache@v4 + with: + path: "${{ github.workspace }}/sampledata/" + key: sample-images + - name: download sample files + if: steps.cache_samples.outputs.cache-hit != 'true' + run: curl -L https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz --create-dirs -o "${{ github.workspace }}/sampledata/sample_images.tar.gz" - uses: actions/setup-python@v5 with: python-version: '3.11' @@ -51,6 +60,7 @@ jobs: # uses: pypa/cibuildwheel@v2.11.2 env: CONAN_COMPILER_VERSION: ${{ matrix.compiler_version }} + SAMPLE_IMAGES_ARCHIVE: "${{ github.workspace }}/sampledata/sample_images.tar.gz" # CIBW_SOME_OPTION: value - uses: actions/upload-artifact@v4 diff --git a/.github/workflows/tox_matrix.yml b/.github/workflows/tox_matrix.yml index ad127242..cc774c24 100644 --- a/.github/workflows/tox_matrix.yml +++ b/.github/workflows/tox_matrix.yml @@ -42,8 +42,16 @@ jobs: fail-fast: false name: Python ${{ matrix.python-version }} ${{ matrix.os }} build steps: - - uses: actions/checkout@v3 + - name: Cache sample files + id: cache_samples + uses: actions/cache@v4 + with: + path: "${{ github.workspace }}/sampledata/" + key: sample-images + - name: download sample files + if: steps.cache_samples.outputs.cache-hit != 'true' + run: curl -L https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz --create-dirs -o "${{ github.workspace }}/sampledata/sample_images.tar.gz" - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -51,11 +59,11 @@ jobs: - name: "install Python dependencies" run: | pip install uv - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: cache with: path: "${{ matrix.conan_user_home }}/.conan2" - key: ${{ runner.os }}-${{ hashFiles('**/conanfile.py') }} + key: ${{ runner.os }}-${{ hashFiles('**/conanfile.py', 'conan.lock') }} - name: Build conan packages on Non-Windows Operating Systems if: ${{ !contains(matrix.os, 'windows') && steps.cache.outputs.cache-hit != 'true' }} @@ -80,10 +88,11 @@ jobs: "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 && uv run --only-group tox --with tox-uv tox -e ${{ steps.tox-env.outputs.result }} env: CONAN_USER_HOME: ${{ matrix.conan_user_home }} + SAMPLE_IMAGES_ARCHIVE: "${{ github.workspace }}/sampledata/sample_images.tar.gz" - name: Run tox if: "!contains(matrix.os, 'windows')" run: cc --version && cc -dumpfullversion -dumpversion && uv run --only-group tox --with tox-uv tox -e ${{ steps.tox-env.outputs.result }} -vvv env: CONAN_COMPILER_LIBCXX: ${{ matrix.compiler_libcxx }} CONAN_USER_HOME: ${{ matrix.conan_user_home }} - + SAMPLE_IMAGES_ARCHIVE: "${{ github.workspace }}/sampledata/sample_images.tar.gz" diff --git a/Jenkinsfile b/Jenkinsfile index 48ee695f..80b05882 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -106,7 +106,7 @@ def test_cpp_code(buildPath){ tee('logs/cmake-build.log'){ sh(label: 'Building C++ Code', script: """conan install . -if ${buildPath} - cmake -B ${buildPath} -Wdev -DCMAKE_TOOLCHAIN_FILE=build/conan_paths.cmake -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true -DBUILD_TESTING:BOOL=true -DCMAKE_CXX_FLAGS="-fprofile-arcs -ftest-coverage -Wall -Wextra" + cmake -B ${buildPath} -Wdev -DSAMPLE_IMAGES_ARCHIVE=\${SAMPLE_IMAGES_ARCHIVE} -DCMAKE_TOOLCHAIN_FILE=build/conan_paths.cmake -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true -DBUILD_TESTING:BOOL=true -DCMAKE_CXX_FLAGS="-fprofile-arcs -ftest-coverage -Wall -Wextra" cmake --build ${buildPath} -j \$(grep -c ^processor /proc/cpuinfo) """ ) diff --git a/ci/docker/linux/jenkins/Dockerfile b/ci/docker/linux/jenkins/Dockerfile index f52cfa80..070ea916 100644 --- a/ci/docker/linux/jenkins/Dockerfile +++ b/ci/docker/linux/jenkins/Dockerfile @@ -1,6 +1,8 @@ ARG CONAN_USER_HOME=/conan ARG CONAN_HOME=${CONAN_USER_HOME}/.conan2 +ARG SAMPLE_IMAGES_URL=https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz + ARG PYTHON_VERSION=latest ARG CONAN_CENTER_PROXY_V2_URL=https://center2.conan.io @@ -33,6 +35,12 @@ RUN mkdir -p /.cache/pip && \ #============================================================================== +FROM base_image AS sample_files_downloader +ARG SAMPLE_IMAGES_URL +ENV EXPECTED_SHA256="0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063" +RUN mkdir /sampledata && wget $SAMPLE_IMAGES_URL -P /sampledata && echo "$EXPECTED_SHA256 /sampledata/sample_images.tar.gz" | sha256sum --check + + FROM base_image AS sonar_builder ARG SONAR_USER_HOME @@ -141,11 +149,14 @@ COPY --from=dr_memory_builder /opt/drmemory /opt/drmemory/ RUN ln -s /opt/drmemory/bin64/drmemory /usr/local/bin/drmemory && \ drmemory -version +COPY --from=sample_files_downloader /sampledata /sampledata + ARG CONAN_USER_HOME ARG CONAN_HOME COPY --from=conan_builder --chmod=777 ${CONAN_HOME} ${CONAN_HOME} ENV CONAN_USER_HOME=${CONAN_USER_HOME}\ - CONAN_HOME=${CONAN_HOME} + CONAN_HOME=${CONAN_HOME}\ + SAMPLE_IMAGES_ARCHIVE=/sampledata/sample_images.tar.gz # To help mark the image as a CI image so it can be cleaned up more easily LABEL purpose=ci diff --git a/ci/docker/linux/tox/Dockerfile b/ci/docker/linux/tox/Dockerfile index 8f8705d2..21a44d38 100644 --- a/ci/docker/linux/tox/Dockerfile +++ b/ci/docker/linux/tox/Dockerfile @@ -11,6 +11,8 @@ ARG CONAN_CENTER_PROXY_V2_URL=https://center2.conan.io # If you want to use a diffrent remote for Conan, such as a proxy. Set the CONAN_CENTER_PROXY_V2_URL # Not this is only for building the image. The actual conan center proxy URL is set in the remotes.json file. +ARG SAMPLE_IMAGES_URL=https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz + FROM ghcr.io/astral-sh/uv:latest AS uv_builder FROM ubuntu:24.04 AS base @@ -25,6 +27,18 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ < /tmp/apt-packages.txt xargs apt-get install -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* + +# ============================================================================== +FROM base AS sample_files_downloader +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + apt-get update && \ + apt-get install -y wget + +ARG SAMPLE_IMAGES_URL +ENV EXPECTED_SHA256="0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063" + +RUN mkdir /sampledata && wget $SAMPLE_IMAGES_URL -P /sampledata && echo "$EXPECTED_SHA256 /sampledata/sample_images.tar.gz" | sha256sum --check + # ============================================================================== FROM base AS conan_builder @@ -74,9 +88,11 @@ FROM base COPY --from=uv_builder /uv /uvx /bin/ ARG CONAN_USER_HOME ARG CONAN_HOME +COPY --from=sample_files_downloader /sampledata /sampledata COPY --from=conan_builder --chmod=777 ${CONAN_HOME}/ ${CONAN_HOME}/ ENV CONAN_USER_HOME=${CONAN_USER_HOME}\ - CONAN_HOME=${CONAN_HOME} + CONAN_HOME=${CONAN_HOME}\ + SAMPLE_IMAGES_ARCHIVE=/sampledata/sample_images.tar.gz # To help mark the image as a CI image so it can be cleaned up more easily LABEL purpose=ci diff --git a/scripts/resources/windows/tox/Dockerfile b/scripts/resources/windows/tox/Dockerfile index 6a32316f..2c52c728 100644 --- a/scripts/resources/windows/tox/Dockerfile +++ b/scripts/resources/windows/tox/Dockerfile @@ -15,6 +15,8 @@ ARG PIP_DOWNLOAD_CACHE=c:/users/containeradministrator/appdata/local/pip ARG CHOCOLATEY_SOURCE=https://chocolatey.org/api/v2 ARG chocolateyVersion +ARG SAMPLE_IMAGES_URL=https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz + FROM ${FROM_IMAGE} AS certsgen RUN certutil -generateSSTFromWU roots.sst @@ -61,6 +63,16 @@ RUN Set-ExecutionPolicy Bypass -Scope Process -Force; ` Write-Host "Verifying installed packages - Done" +# ============================================================================== +FROM base_builder AS sample_files_downloader +ARG SAMPLE_IMAGES_URL +ENV EXPECTED_SHA256="0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063" +ENV downloadPath="C:/sampledata/" +RUN if (-not (Test-Path -Path ${env:downloadPath} -PathType Container)) {` + New-Item -Path ${env:downloadPath} -ItemType Directory -Force` + }; ` + Invoke-WebRequest -Uri ${env:SAMPLE_IMAGES_URL} -OutFile "${env:downloadPath}\sample_images.tar.gz" + # ============================================================================== FROM base_builder AS conan_builder ARG CONAN_HOME @@ -105,11 +117,14 @@ RUN New-Item -type directory -path ${Env:PIP_DOWNLOAD_CACHE} -Force | Out-Null ; New-Item -type directory -path ${Env:UV_CACHE_DIR} -Force | Out-Null ARG CONAN_HOME COPY --from=conan_builder ${CONAN_HOME}/ ${CONAN_HOME}/ +COPY --from=sample_files_downloader c:/sampledata/ C:/sampledata/ + ARG CONAN_USER_HOME ENV CONAN_USER_HOME=${CONAN_USER_HOME}` CONAN_HOME=${CONAN_HOME}` PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE}` - UV_CACHE_DIR=${UV_CACHE_DIR} + UV_CACHE_DIR=${UV_CACHE_DIR}` + SAMPLE_IMAGES_ARCHIVE=C:/sampledata/sample_images.tar.gz WORKDIR C:/src ENV DISTUTILS_USE_SDK=1 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0fd7b832..e4b26cbd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,15 +1,25 @@ include(FetchContent) if (BUILD_TESTING AND Catch2_FOUND) include(Catch) - FetchContent_Declare(test_images + set(SAMPLE_IMAGES_ARCHIVE "" CACHE FILEPATH "Path to sample images archive. If not set, it will be downloaded from the internet.") + if(SAMPLE_IMAGES_ARCHIVE) + set(TEST_IMAGE_PATH "${CMAKE_CURRENT_BINARY_DIR}/sample_images/") + file(ARCHIVE_EXTRACT + INPUT "${SAMPLE_IMAGES_ARCHIVE}" + DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/" + ) + else () + FetchContent_Declare(test_images URL https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz + URL_HASH SHA256=0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063 SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/sample_images - ) - FetchContent_GetProperties(test_images) - FetchContent_MakeAvailable(test_images) - find_path(TEST_IMAGE_PATH dummy.jp2 - PATHS ${test_images_SOURCE_DIR}) - set(TEST_IMAGE_PATH ${TEST_IMAGE_PATH}/) + ) + FetchContent_GetProperties(test_images) + FetchContent_MakeAvailable(test_images) + find_path(TEST_IMAGE_PATH dummy.jp2 + PATHS ${test_images_SOURCE_DIR}) + set(TEST_IMAGE_PATH ${TEST_IMAGE_PATH}/) + endif () set(PYTHON_TEST_FILES test_core.py) find_package(Python3 COMPONENTS Interpreter) if(pyexiv2bind_generate_python_bindings) diff --git a/tests/conftest.py b/tests/conftest.py index 8f518f6b..d39d125a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,24 +1,37 @@ import os import shutil import tarfile +import hashlib import pytest import urllib.request +SAMPLE_IMAGES_SHA256 = "0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063" +SAMPLE_IMAGES_URL = "https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz" -def download_images(url, destination, download_path): +# To save time from redownloading the sample_images.tar.gz file every time, +# download it locally and set the environment variable SAMPLE_IMAGES_ARCHIVE +# to the path of the downloaded file - print("Downloading {}".format(url)) - urllib.request.urlretrieve(url, - filename=os.path.join(download_path, "sample_images.tar.gz")) - if not os.path.exists(os.path.join(download_path, "sample_images.tar.gz")): +def download_images(url, download_path): + + print(f"Downloading {url}") + output = os.path.join(download_path, "sample_images.tar.gz") + urllib.request.urlretrieve(url, filename=output) + if not os.path.exists(output): raise FileNotFoundError("sample images not download") - print("Extracting images") - with tarfile.open(os.path.join(download_path, "sample_images.tar.gz"), "r:gz") as archive_file: - for item in archive_file.getmembers(): - print("Extracting {}".format(item.name)) - archive_file.extract(item, path=destination) - pass + return output +def extract_images(path, destination): + print("Extracting images") + with tarfile.open(path, "r:gz") as archive_file: + for item in archive_file.getmembers(): + print("Extracting {}".format(item.name)) + archive_file.extract(item, path=destination) + +def verify_hash(path, sha256_hash): + with open(path, "rb") as f: + file_hash = hashlib.sha256(f.read()).hexdigest() + assert file_hash == sha256_hash @pytest.fixture(scope="session") @@ -28,16 +41,23 @@ def sample_images_readonly(tmpdir_factory): sample_images_path = os.path.join(test_path, "sample_images") download_path = tmpdir_factory.mktemp("downloaded_archives", numbered=False) if os.path.exists(sample_images_path): - print("{} already exits".format(sample_images_path)) + print(f"{sample_images_path} already exits") else: - print("Downloading sample images") - download_images(url="https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz", - destination=test_path, - download_path=download_path) - + archive = os.getenv('SAMPLE_IMAGES_ARCHIVE') + if not archive: + print("Downloading sample images") + archive = download_images( + url=SAMPLE_IMAGES_URL, + download_path=download_path + ) + if not os.path.exists(archive): + raise FileNotFoundError(f"sample image archive not found. {archive} does not exist.") + verify_hash(archive, sha256_hash=SAMPLE_IMAGES_SHA256) + extract_images(path=archive, destination=test_path) yield sample_images_path - shutil.rmtree(test_path) + if os.path.exists(test_path): + shutil.rmtree(test_path) @pytest.fixture diff --git a/tox.ini b/tox.ini index 6af6d1f1..9f486ad0 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,7 @@ passenv = LIB LIBPATH DISTUTILS_USE_SDK + SAMPLE_IMAGES_ARCHIVE commands_pre = {env_bin_dir}{/}python {tox_root}/scripts/get_linked_exiv2_version.py From a0b49de566196e963cb7ae4b8ad4ba81c20814c1 Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Thu, 18 Dec 2025 13:30:08 -0600 Subject: [PATCH 2/2] ci: run git clean with -dfx --- Jenkinsfile | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 80b05882..553d3cbd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -143,7 +143,7 @@ def windows_wheels(pythonVersions, testPackages, params, wheelStashes){ script: "docker image rm --no-prune ${dockerImageName}", returnStatus: true ) - bat "${tool(name: 'Default', type: 'git')} clean -dfx" + bat "${tool(name: 'Default', type: 'git')} clean -dffx" } } } @@ -188,7 +188,7 @@ def windows_wheels(pythonVersions, testPackages, params, wheelStashes){ } } } finally { - bat "${tool(name: 'Default', type: 'git')} clean -dfx" + bat "${tool(name: 'Default', type: 'git')} clean -dffx" } } } @@ -236,7 +236,7 @@ def linux_wheels(pythonVersions, testPackages, params, wheelStashes){ } } } finally { - sh "${tool(name: 'Default', type: 'git')} clean -dfx" + sh "${tool(name: 'Default', type: 'git')} clean -dffx" } } } @@ -269,7 +269,7 @@ def linux_wheels(pythonVersions, testPackages, params, wheelStashes){ } } } finally { - sh "${tool(name: 'Default', type: 'git')} clean -dfx" + sh "${tool(name: 'Default', type: 'git')} clean -dffx" } } } @@ -327,7 +327,7 @@ def mac_wheels(pythonVersions, testPackages, params, wheelStashes){ wheelStashes << "python${pythonVersion} mac ${arch} wheel" archiveArtifacts artifacts: 'dist/*.whl' } finally { - sh "${tool(name: 'Default', type: 'git')} clean -dfx" + sh "${tool(name: 'Default', type: 'git')} clean -dffx" } } } @@ -353,7 +353,7 @@ def mac_wheels(pythonVersions, testPackages, params, wheelStashes){ } } } finally { - sh "${tool(name: 'Default', type: 'git')} clean -dfx" + sh "${tool(name: 'Default', type: 'git')} clean -dffx" } } } @@ -401,7 +401,7 @@ def mac_wheels(pythonVersions, testPackages, params, wheelStashes){ archiveArtifacts artifacts: 'dist/*.whl' } } finally{ - sh "${tool(name: 'Default', type: 'git')} clean -dfx" + sh "${tool(name: 'Default', type: 'git')} clean -dffx" } } } @@ -434,7 +434,7 @@ def mac_wheels(pythonVersions, testPackages, params, wheelStashes){ } archiveArtifacts artifacts: 'dist/*.whl' } finally { - sh "${tool(name: 'Default', type: 'git')} clean -dfx" + sh "${tool(name: 'Default', type: 'git')} clean -dffx" } } } @@ -884,7 +884,7 @@ pipeline { } post{ cleanup{ - sh "git clean -dfx" + sh "git clean -dffx" } } } @@ -920,7 +920,7 @@ pipeline { ).trim().split('\n') } } finally{ - sh "${tool(name: 'Default', type: 'git')} clean -dfx" + sh "${tool(name: 'Default', type: 'git')} clean -dffx" } } @@ -952,7 +952,7 @@ pipeline { } } } finally{ - sh "${tool(name: 'Default', type: 'git')} clean -dfx" + sh "${tool(name: 'Default', type: 'git')} clean -dffx" } } } finally { @@ -1002,7 +1002,7 @@ pipeline { } } } finally{ - bat "${tool(name: 'Default', type: 'git')} clean -dfx" + bat "${tool(name: 'Default', type: 'git')} clean -dffx" } } parallel( @@ -1051,7 +1051,7 @@ pipeline { } } } finally { - bat "${tool(name: 'Default', type: 'git')} clean -dfx" + bat "${tool(name: 'Default', type: 'git')} clean -dffx" } } finally{ if (image){ @@ -1204,7 +1204,7 @@ pipeline { } } }finally{ - sh "${tool(name: 'Default', type: 'git')} clean -dfx" + sh "${tool(name: 'Default', type: 'git')} clean -dffx" } } } @@ -1276,7 +1276,7 @@ pipeline { } } } finally { - bat "${tool(name: 'Default', type: 'git')} clean -dfx" + bat "${tool(name: 'Default', type: 'git')} clean -dffx" } } @@ -1352,7 +1352,7 @@ pipeline { } } } finally{ - sh "${tool(name: 'Default', type: 'git')} clean -dfx" + sh "${tool(name: 'Default', type: 'git')} clean -dffx" } } finally{ if(image){