Skip to content

tests: cache and verify hash value of sample images #247

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
henryborchers merged 2 commits into from
Dec 18, 2025
Merged

tests: cache and verify hash value of sample images #247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8e288ab
tests: cache and verify hash value of sample images
henryborchers Dec 18, 2025
a0b49de
ci: run git clean with -dfx
henryborchers Dec 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions .github/workflows/build_wheels.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,15 @@ jobs:

steps:
- uses: actions/checkout@v4
- name: Cache sample files
id: cache_samples
uses: actions/cache@v4
with:
path: "${{ github.workspace }}/sampledata/"
key: sample-images
- name: download sample files
if: steps.cache_samples.outputs.cache-hit != 'true'
run: curl -L https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz --create-dirs -o "${{ github.workspace }}/sampledata/sample_images.tar.gz"
- uses: actions/setup-python@v5
with:
python-version: '3.11'
Expand All @@ -51,6 +60,7 @@ jobs:
# uses: pypa/cibuildwheel@v2.11.2
env:
CONAN_COMPILER_VERSION: ${{ matrix.compiler_version }}
SAMPLE_IMAGES_ARCHIVE: "${{ github.workspace }}/sampledata/sample_images.tar.gz"

# CIBW_SOME_OPTION: value
- uses: actions/upload-artifact@v4
Expand Down
17 changes: 13 additions & 4 deletions .github/workflows/tox_matrix.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,20 +42,28 @@ jobs:
fail-fast: false
name: Python ${{ matrix.python-version }} ${{ matrix.os }} build
steps:

- uses: actions/checkout@v3
- name: Cache sample files
id: cache_samples
uses: actions/cache@v4
with:
path: "${{ github.workspace }}/sampledata/"
key: sample-images
- name: download sample files
if: steps.cache_samples.outputs.cache-hit != 'true'
run: curl -L https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz --create-dirs -o "${{ github.workspace }}/sampledata/sample_images.tar.gz"
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'pip' # caching pip dependencies
- name: "install Python dependencies"
run: |
pip install uv
- uses: actions/cache@v3
- uses: actions/cache@v4
id: cache
with:
path: "${{ matrix.conan_user_home }}/.conan2"
key: ${{ runner.os }}-${{ hashFiles('**/conanfile.py') }}
key: ${{ runner.os }}-${{ hashFiles('**/conanfile.py', 'conan.lock') }}

- name: Build conan packages on Non-Windows Operating Systems
if: ${{ !contains(matrix.os, 'windows') && steps.cache.outputs.cache-hit != 'true' }}
Expand All @@ -80,10 +88,11 @@ jobs:
"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 && uv run --only-group tox --with tox-uv tox -e ${{ steps.tox-env.outputs.result }}
env:
CONAN_USER_HOME: ${{ matrix.conan_user_home }}
SAMPLE_IMAGES_ARCHIVE: "${{ github.workspace }}/sampledata/sample_images.tar.gz"
- name: Run tox
if: "!contains(matrix.os, 'windows')"
run: cc --version && cc -dumpfullversion -dumpversion && uv run --only-group tox --with tox-uv tox -e ${{ steps.tox-env.outputs.result }} -vvv
env:
CONAN_COMPILER_LIBCXX: ${{ matrix.compiler_libcxx }}
CONAN_USER_HOME: ${{ matrix.conan_user_home }}

SAMPLE_IMAGES_ARCHIVE: "${{ github.workspace }}/sampledata/sample_images.tar.gz"
34 changes: 17 additions & 17 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ def test_cpp_code(buildPath){
tee('logs/cmake-build.log'){
sh(label: 'Building C++ Code',
script: """conan install . -if ${buildPath}
cmake -B ${buildPath} -Wdev -DCMAKE_TOOLCHAIN_FILE=build/conan_paths.cmake -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true -DBUILD_TESTING:BOOL=true -DCMAKE_CXX_FLAGS="-fprofile-arcs -ftest-coverage -Wall -Wextra"
cmake -B ${buildPath} -Wdev -DSAMPLE_IMAGES_ARCHIVE=\${SAMPLE_IMAGES_ARCHIVE} -DCMAKE_TOOLCHAIN_FILE=build/conan_paths.cmake -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true -DBUILD_TESTING:BOOL=true -DCMAKE_CXX_FLAGS="-fprofile-arcs -ftest-coverage -Wall -Wextra"
cmake --build ${buildPath} -j \$(grep -c ^processor /proc/cpuinfo)
"""
)
Expand Down Expand Up @@ -143,7 +143,7 @@ def windows_wheels(pythonVersions, testPackages, params, wheelStashes){
script: "docker image rm --no-prune ${dockerImageName}",
returnStatus: true
)
bat "${tool(name: 'Default', type: 'git')} clean -dfx"
bat "${tool(name: 'Default', type: 'git')} clean -dffx"
}
}
}
Expand Down Expand Up @@ -188,7 +188,7 @@ def windows_wheels(pythonVersions, testPackages, params, wheelStashes){
}
}
} finally {
bat "${tool(name: 'Default', type: 'git')} clean -dfx"
bat "${tool(name: 'Default', type: 'git')} clean -dffx"
}
}
}
Expand Down Expand Up @@ -236,7 +236,7 @@ def linux_wheels(pythonVersions, testPackages, params, wheelStashes){
}
}
} finally {
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
}
}
}
Expand Down Expand Up @@ -269,7 +269,7 @@ def linux_wheels(pythonVersions, testPackages, params, wheelStashes){
}
}
} finally {
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
}
}
}
Expand Down Expand Up @@ -327,7 +327,7 @@ def mac_wheels(pythonVersions, testPackages, params, wheelStashes){
wheelStashes << "python${pythonVersion} mac ${arch} wheel"
archiveArtifacts artifacts: 'dist/*.whl'
} finally {
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
}
}
}
Expand All @@ -353,7 +353,7 @@ def mac_wheels(pythonVersions, testPackages, params, wheelStashes){
}
}
} finally {
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
}
}
}
Expand Down Expand Up @@ -401,7 +401,7 @@ def mac_wheels(pythonVersions, testPackages, params, wheelStashes){
archiveArtifacts artifacts: 'dist/*.whl'
}
} finally{
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
}
}
}
Expand Down Expand Up @@ -434,7 +434,7 @@ def mac_wheels(pythonVersions, testPackages, params, wheelStashes){
}
archiveArtifacts artifacts: 'dist/*.whl'
} finally {
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
}
}
}
Expand Down Expand Up @@ -884,7 +884,7 @@ pipeline {
}
post{
cleanup{
sh "git clean -dfx"
sh "git clean -dffx"
}
}
}
Expand Down Expand Up @@ -920,7 +920,7 @@ pipeline {
).trim().split('\n')
}
} finally{
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
}
}

Expand Down Expand Up @@ -952,7 +952,7 @@ pipeline {
}
}
} finally{
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
}
}
} finally {
Expand Down Expand Up @@ -1002,7 +1002,7 @@ pipeline {
}
}
} finally{
bat "${tool(name: 'Default', type: 'git')} clean -dfx"
bat "${tool(name: 'Default', type: 'git')} clean -dffx"
}
}
parallel(
Expand Down Expand Up @@ -1051,7 +1051,7 @@ pipeline {
}
}
} finally {
bat "${tool(name: 'Default', type: 'git')} clean -dfx"
bat "${tool(name: 'Default', type: 'git')} clean -dffx"
}
} finally{
if (image){
Expand Down Expand Up @@ -1204,7 +1204,7 @@ pipeline {
}
}
}finally{
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
}
}
}
Expand Down Expand Up @@ -1276,7 +1276,7 @@ pipeline {
}
}
} finally {
bat "${tool(name: 'Default', type: 'git')} clean -dfx"
bat "${tool(name: 'Default', type: 'git')} clean -dffx"

}
}
Expand Down Expand Up @@ -1352,7 +1352,7 @@ pipeline {
}
}
} finally{
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
}
} finally{
if(image){
Expand Down
13 changes: 12 additions & 1 deletion ci/docker/linux/jenkins/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
ARG CONAN_USER_HOME=/conan
ARG CONAN_HOME=${CONAN_USER_HOME}/.conan2

ARG SAMPLE_IMAGES_URL=https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz

ARG PYTHON_VERSION=latest

ARG CONAN_CENTER_PROXY_V2_URL=https://center2.conan.io
Expand Down Expand Up @@ -33,6 +35,12 @@ RUN mkdir -p /.cache/pip && \
#==============================================================================


FROM base_image AS sample_files_downloader
ARG SAMPLE_IMAGES_URL
ENV EXPECTED_SHA256="0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063"
RUN mkdir /sampledata && wget $SAMPLE_IMAGES_URL -P /sampledata && echo "$EXPECTED_SHA256 /sampledata/sample_images.tar.gz" | sha256sum --check


FROM base_image AS sonar_builder
ARG SONAR_USER_HOME

Expand Down Expand Up @@ -141,11 +149,14 @@ COPY --from=dr_memory_builder /opt/drmemory /opt/drmemory/
RUN ln -s /opt/drmemory/bin64/drmemory /usr/local/bin/drmemory && \
drmemory -version

COPY --from=sample_files_downloader /sampledata /sampledata

ARG CONAN_USER_HOME
ARG CONAN_HOME
COPY --from=conan_builder --chmod=777 ${CONAN_HOME} ${CONAN_HOME}
ENV CONAN_USER_HOME=${CONAN_USER_HOME}\
CONAN_HOME=${CONAN_HOME}
CONAN_HOME=${CONAN_HOME}\
SAMPLE_IMAGES_ARCHIVE=/sampledata/sample_images.tar.gz

# To help mark the image as a CI image so it can be cleaned up more easily
LABEL purpose=ci
18 changes: 17 additions & 1 deletion ci/docker/linux/tox/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ ARG CONAN_CENTER_PROXY_V2_URL=https://center2.conan.io
# If you want to use a diffrent remote for Conan, such as a proxy. Set the CONAN_CENTER_PROXY_V2_URL
# Not this is only for building the image. The actual conan center proxy URL is set in the remotes.json file.

ARG SAMPLE_IMAGES_URL=https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz

FROM ghcr.io/astral-sh/uv:latest AS uv_builder

FROM ubuntu:24.04 AS base
Expand All @@ -25,6 +27,18 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
< /tmp/apt-packages.txt xargs apt-get install -y && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

# ==============================================================================
FROM base AS sample_files_downloader
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
apt-get update && \
apt-get install -y wget

ARG SAMPLE_IMAGES_URL
ENV EXPECTED_SHA256="0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063"

RUN mkdir /sampledata && wget $SAMPLE_IMAGES_URL -P /sampledata && echo "$EXPECTED_SHA256 /sampledata/sample_images.tar.gz" | sha256sum --check

# ==============================================================================

FROM base AS conan_builder
Expand Down Expand Up @@ -74,9 +88,11 @@ FROM base
COPY --from=uv_builder /uv /uvx /bin/
ARG CONAN_USER_HOME
ARG CONAN_HOME
COPY --from=sample_files_downloader /sampledata /sampledata
COPY --from=conan_builder --chmod=777 ${CONAN_HOME}/ ${CONAN_HOME}/
ENV CONAN_USER_HOME=${CONAN_USER_HOME}\
CONAN_HOME=${CONAN_HOME}
CONAN_HOME=${CONAN_HOME}\
SAMPLE_IMAGES_ARCHIVE=/sampledata/sample_images.tar.gz

# To help mark the image as a CI image so it can be cleaned up more easily
LABEL purpose=ci
17 changes: 16 additions & 1 deletion scripts/resources/windows/tox/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ ARG PIP_DOWNLOAD_CACHE=c:/users/containeradministrator/appdata/local/pip
ARG CHOCOLATEY_SOURCE=https://chocolatey.org/api/v2
ARG chocolateyVersion

ARG SAMPLE_IMAGES_URL=https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz

FROM ${FROM_IMAGE} AS certsgen
RUN certutil -generateSSTFromWU roots.sst

Expand Down Expand Up @@ -61,6 +63,16 @@ RUN Set-ExecutionPolicy Bypass -Scope Process -Force; `
Write-Host "Verifying installed packages - Done"


# ==============================================================================
FROM base_builder AS sample_files_downloader
ARG SAMPLE_IMAGES_URL
ENV EXPECTED_SHA256="0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063"
ENV downloadPath="C:/sampledata/"
RUN if (-not (Test-Path -Path ${env:downloadPath} -PathType Container)) {`
New-Item -Path ${env:downloadPath} -ItemType Directory -Force`
}; `
Invoke-WebRequest -Uri ${env:SAMPLE_IMAGES_URL} -OutFile "${env:downloadPath}\sample_images.tar.gz"

# ==============================================================================
FROM base_builder AS conan_builder
ARG CONAN_HOME
Expand Down Expand Up @@ -105,11 +117,14 @@ RUN New-Item -type directory -path ${Env:PIP_DOWNLOAD_CACHE} -Force | Out-Null ;
New-Item -type directory -path ${Env:UV_CACHE_DIR} -Force | Out-Null
ARG CONAN_HOME
COPY --from=conan_builder ${CONAN_HOME}/ ${CONAN_HOME}/
COPY --from=sample_files_downloader c:/sampledata/ C:/sampledata/

ARG CONAN_USER_HOME
ENV CONAN_USER_HOME=${CONAN_USER_HOME}`
CONAN_HOME=${CONAN_HOME}`
PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE}`
UV_CACHE_DIR=${UV_CACHE_DIR}
UV_CACHE_DIR=${UV_CACHE_DIR}`
SAMPLE_IMAGES_ARCHIVE=C:/sampledata/sample_images.tar.gz

WORKDIR C:/src
ENV DISTUTILS_USE_SDK=1
Expand Down
Loading
Loading