From cf6c084129a32478702b83ca4dc205edb8637259 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Thu, 16 Oct 2025 12:55:35 +0100 Subject: [PATCH 01/30] Update Dockerfile --- model-engine/Dockerfile | 47 ++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/model-engine/Dockerfile b/model-engine/Dockerfile index 45cd9630d..76c85ad01 100644 --- a/model-engine/Dockerfile +++ b/model-engine/Dockerfile @@ -1,42 +1,29 @@ -# syntax = docker/dockerfile:experimental - -FROM python:3.10.15-slim as model-engine +FROM cgr.dev/scale.com/python-fips:3.10.15-dev WORKDIR /workspace - -RUN apt-get update && apt-get install -y \ - apt-utils \ - dumb-init \ - git \ - ssh \ - htop \ - iftop \ - vim \ - curl \ - procps \ - libcurl4-openssl-dev \ - libssl-dev \ - python3-dev \ - gcc \ - build-essential \ - telnet \ - && rm -rf /var/lib/apt/lists/* +USER root + +RUN apk update && apk add htop \ + dumb-init \ + libssh \ + openssh-client \ + iftop \ + curl \ + curl-dev \ + procps \ + libcurl-openssl4 \ + vim \ + kubectl RUN curl -Lo /bin/aws-iam-authenticator https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.5.9/aws-iam-authenticator_0.5.9_linux_amd64 RUN chmod +x /bin/aws-iam-authenticator -# Install kubectl -RUN curl -LO "https://dl.k8s.io/release/v1.23.13/bin/linux/amd64/kubectl" \ - && chmod +x kubectl \ - && mv kubectl /usr/local/bin/kubectl - -# Pin pip version RUN pip install pip==24.2 RUN chmod -R 777 /workspace -# Install AWS CLI RUN pip install awscli==1.34.28 --no-cache-dir -## grab model_engine_server project (w/ requirements install layer caching) +COPY federal/sitecustomize.py /usr/lib/python3.10/site-packages/sitecustomize.py + WORKDIR /workspace/model-engine/ COPY model-engine/requirements-test.txt /workspace/model-engine/requirements-test.txt COPY model-engine/requirements.txt /workspace/model-engine/requirements.txt @@ -54,4 +41,6 @@ WORKDIR /workspace ENV PYTHONPATH /workspace ENV WORKSPACE /workspace +USER nonroot + EXPOSE 5000 From 0201f0ca43dfce1b588758b9f297df944e12d245 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Thu, 16 Oct 2025 18:15:16 +0100 Subject: [PATCH 02/30] Update circleci config to login to chainguard --- .circleci/config.yml | 27 +++++++++++++++++++++++++++ model-engine/Dockerfile | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4267200dd..e2a76776d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -105,6 +105,7 @@ jobs: executor: ubuntu-large steps: - checkout + - chainguard_login - run: name: Build Docker Image command: | @@ -116,6 +117,7 @@ jobs: - aws-cli/setup: role-arn: ${CIRCLECI_ROLE_ARN} aws-region: AWS_REGION + - chainguard_login - run: name: Build Docker Image command: | @@ -209,6 +211,31 @@ executors: resource_class: 2xlarge commands: + chainguard_login: + description: Authenticate to Chainguard Registry via OIDC + steps: + - run: + name: Login to Chainguard Registry + command: | + # Check if OIDC token is available + if [[ -z "${CIRCLE_OIDC_TOKEN}" ]]; then + echo "ERROR: CIRCLE_OIDC_TOKEN is not available" + exit 1 + fi + + # Exchange OIDC token for Chainguard token + CHAINGUARD_TOKEN=$(curl -X POST "https://issuer.enforce.dev/oauth/token" \ + -H "Content-Type: application/json" \ + -d "{ + \"audience\": \"scale.com\", + \"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", + \"subject_token\": \"${OIDC_TOKEN}\", + \"subject_token_type\": \"urn:ietf:params:oauth:token-type:id_token\", + \"requested_token_type\": \"urn:ietf:params:oauth:token-type:access_token\" + }" | jq -r '.access_token') + + # Login to Chainguard registry + echo "${CHAINGUARD_TOKEN}" | docker login cgr.dev -u "${CHAINGUARD_IDENTITY_ID}" --password-stdin environment_setup: description: Basic Environment setup steps: diff --git a/model-engine/Dockerfile b/model-engine/Dockerfile index 76c85ad01..f27ef57b4 100644 --- a/model-engine/Dockerfile +++ b/model-engine/Dockerfile @@ -1,4 +1,4 @@ -FROM cgr.dev/scale.com/python-fips:3.10.15-dev +FROM cgr.dev/scale.com/python-fips:3.10.19-dev WORKDIR /workspace USER root From 8ece2a5a37e30e2c01cb9b021e9b08f692990ab5 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Fri, 24 Oct 2025 16:22:55 +0100 Subject: [PATCH 03/30] fix typo in circleci config --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e2a76776d..14c11f2ad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -229,7 +229,7 @@ commands: -d "{ \"audience\": \"scale.com\", \"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", - \"subject_token\": \"${OIDC_TOKEN}\", + \"subject_token\": \"${CIRCLE_OIDC_TOKEN}\", \"subject_token_type\": \"urn:ietf:params:oauth:token-type:id_token\", \"requested_token_type\": \"urn:ietf:params:oauth:token-type:access_token\" }" | jq -r '.access_token') From c3ef4e54a22770a0754505f8ee44e692d03f9f19 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Fri, 24 Oct 2025 16:35:06 +0100 Subject: [PATCH 04/30] Add code to debug circleci errors --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 14c11f2ad..454f7b5f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -235,7 +235,9 @@ commands: }" | jq -r '.access_token') # Login to Chainguard registry - echo "${CHAINGUARD_TOKEN}" | docker login cgr.dev -u "${CHAINGUARD_IDENTITY_ID}" --password-stdin + echo "Chainguard token: ${CHAINGUARD_TOKEN}" + docker login "cgr.dev" --username "${CHAINGUARD_IDENTITY_ID}" --password "${CHAINGUARD_PULL_TOKEN}" + docker login cgr.dev -u "${CHAINGUARD_IDENTITY_ID}" --password "${CHAINGUARD_TOKEN}" environment_setup: description: Basic Environment setup steps: From 93f579a23f79b698ca6a1c2f9a3a05697a616273 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Fri, 24 Oct 2025 16:40:41 +0100 Subject: [PATCH 05/30] Debug missing chainguard token --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 454f7b5f9..f5edc4e19 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -235,9 +235,11 @@ commands: }" | jq -r '.access_token') # Login to Chainguard registry + echo "CIRCLE_OIDC_TOKEN: ${CIRCLE_OIDC_TOKEN}" + echo "CIRCLE_OIDC_TOKEN_V2: ${CIRCLE_OIDC_TOKEN_V2}" echo "Chainguard token: ${CHAINGUARD_TOKEN}" - docker login "cgr.dev" --username "${CHAINGUARD_IDENTITY_ID}" --password "${CHAINGUARD_PULL_TOKEN}" - docker login cgr.dev -u "${CHAINGUARD_IDENTITY_ID}" --password "${CHAINGUARD_TOKEN}" + echo "${CHAINGUARD_PULL_TOKEN}" | docker login "cgr.dev" --username "${CHAINGUARD_IDENTITY_ID}" --password-stdin + echo "${CHAINGUARD_TOKEN}" | docker login cgr.dev -u "${CHAINGUARD_IDENTITY_ID}" --password-stdin environment_setup: description: Basic Environment setup steps: From 38535175f1da2133b83cf50dfa19266ef28da6af Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Fri, 24 Oct 2025 16:45:51 +0100 Subject: [PATCH 06/30] Debug failing oidc token swap --- .circleci/config.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f5edc4e19..a85ed1625 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -223,8 +223,13 @@ commands: exit 1 fi + # Debug: Decode OIDC token to see claims + echo "=== Decoding OIDC Token Claims ===" + echo "${CIRCLE_OIDC_TOKEN}" | cut -d. -f2 | base64 -d 2>/dev/null | jq . || echo "Failed to decode token" + # Exchange OIDC token for Chainguard token - CHAINGUARD_TOKEN=$(curl -X POST "https://issuer.enforce.dev/oauth/token" \ + echo "=== Attempting Token Exchange ===" + TOKEN_RESPONSE=$(curl -s -X POST "https://issuer.enforce.dev/oauth/token" \ -H "Content-Type: application/json" \ -d "{ \"audience\": \"scale.com\", @@ -232,7 +237,12 @@ commands: \"subject_token\": \"${CIRCLE_OIDC_TOKEN}\", \"subject_token_type\": \"urn:ietf:params:oauth:token-type:id_token\", \"requested_token_type\": \"urn:ietf:params:oauth:token-type:access_token\" - }" | jq -r '.access_token') + }") + + echo "Token exchange response:" + echo "${TOKEN_RESPONSE}" | jq . + + CHAINGUARD_TOKEN=$(echo "${TOKEN_RESPONSE}" | jq -r '.access_token') # Login to Chainguard registry echo "CIRCLE_OIDC_TOKEN: ${CIRCLE_OIDC_TOKEN}" From 9dac2cd294fd3ebe3c7ffc4930da2fb389ad2e0a Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Fri, 24 Oct 2025 16:58:00 +0100 Subject: [PATCH 07/30] Update config --- .circleci/config.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a85ed1625..443f928e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -245,11 +245,8 @@ commands: CHAINGUARD_TOKEN=$(echo "${TOKEN_RESPONSE}" | jq -r '.access_token') # Login to Chainguard registry - echo "CIRCLE_OIDC_TOKEN: ${CIRCLE_OIDC_TOKEN}" - echo "CIRCLE_OIDC_TOKEN_V2: ${CIRCLE_OIDC_TOKEN_V2}" - echo "Chainguard token: ${CHAINGUARD_TOKEN}" echo "${CHAINGUARD_PULL_TOKEN}" | docker login "cgr.dev" --username "${CHAINGUARD_IDENTITY_ID}" --password-stdin - echo "${CHAINGUARD_TOKEN}" | docker login cgr.dev -u "${CHAINGUARD_IDENTITY_ID}" --password-stdin + # echo "${CHAINGUARD_TOKEN}" | docker login cgr.dev -u "${CHAINGUARD_IDENTITY_ID}" --password-stdin environment_setup: description: Basic Environment setup steps: From 8104ff19b9071016bea0d4a49f3c3481bf22d406 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Mon, 27 Oct 2025 10:17:09 +0000 Subject: [PATCH 08/30] Retry OIDC token swap with updated chainguard identity --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 443f928e6..a1b61628e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -245,8 +245,8 @@ commands: CHAINGUARD_TOKEN=$(echo "${TOKEN_RESPONSE}" | jq -r '.access_token') # Login to Chainguard registry - echo "${CHAINGUARD_PULL_TOKEN}" | docker login "cgr.dev" --username "${CHAINGUARD_IDENTITY_ID}" --password-stdin - # echo "${CHAINGUARD_TOKEN}" | docker login cgr.dev -u "${CHAINGUARD_IDENTITY_ID}" --password-stdin + # echo "${CHAINGUARD_PULL_TOKEN}" | docker login "cgr.dev" --username "${CHAINGUARD_IDENTITY_ID}" --password-stdin + echo "${CHAINGUARD_TOKEN}" | docker login cgr.dev -u "${CHAINGUARD_IDENTITY_ID}" --password-stdin environment_setup: description: Basic Environment setup steps: From 4ddee82581cd31b1b0ffc516226faf63e026d29b Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Mon, 27 Oct 2025 10:28:13 +0000 Subject: [PATCH 09/30] Update audience for token exchange request --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1b61628e..719557277 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -232,7 +232,7 @@ commands: TOKEN_RESPONSE=$(curl -s -X POST "https://issuer.enforce.dev/oauth/token" \ -H "Content-Type: application/json" \ -d "{ - \"audience\": \"scale.com\", + \"audience\": \"cgr.dev\", \"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token\": \"${CIRCLE_OIDC_TOKEN}\", \"subject_token_type\": \"urn:ietf:params:oauth:token-type:id_token\", From e5642e8cd93cc11326062c00cc3fe7c8075201ca Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Mon, 27 Oct 2025 11:11:59 +0000 Subject: [PATCH 10/30] Simplify chainguard authentication with chainctl --- .circleci/config.yml | 38 ++++++++------------------------------ 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 719557277..f4191345c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -214,39 +214,17 @@ commands: chainguard_login: description: Authenticate to Chainguard Registry via OIDC steps: + - run: + name: Install chainctl + command: | + curl -o chainctl "https://dl.enforce.dev/chainctl/latest/chainctl_$(uname -s | tr '[:upper:]' '[:lower:]')_$(uname -m | sed 's/aarch64/arm64/')" + sudo install -o $UID -g $(id -g) -m 0755 chainctl /usr/local/bin/ - run: name: Login to Chainguard Registry command: | - # Check if OIDC token is available - if [[ -z "${CIRCLE_OIDC_TOKEN}" ]]; then - echo "ERROR: CIRCLE_OIDC_TOKEN is not available" - exit 1 - fi - - # Debug: Decode OIDC token to see claims - echo "=== Decoding OIDC Token Claims ===" - echo "${CIRCLE_OIDC_TOKEN}" | cut -d. -f2 | base64 -d 2>/dev/null | jq . || echo "Failed to decode token" - - # Exchange OIDC token for Chainguard token - echo "=== Attempting Token Exchange ===" - TOKEN_RESPONSE=$(curl -s -X POST "https://issuer.enforce.dev/oauth/token" \ - -H "Content-Type: application/json" \ - -d "{ - \"audience\": \"cgr.dev\", - \"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", - \"subject_token\": \"${CIRCLE_OIDC_TOKEN}\", - \"subject_token_type\": \"urn:ietf:params:oauth:token-type:id_token\", - \"requested_token_type\": \"urn:ietf:params:oauth:token-type:access_token\" - }") - - echo "Token exchange response:" - echo "${TOKEN_RESPONSE}" | jq . - - CHAINGUARD_TOKEN=$(echo "${TOKEN_RESPONSE}" | jq -r '.access_token') - - # Login to Chainguard registry - # echo "${CHAINGUARD_PULL_TOKEN}" | docker login "cgr.dev" --username "${CHAINGUARD_IDENTITY_ID}" --password-stdin - echo "${CHAINGUARD_TOKEN}" | docker login cgr.dev -u "${CHAINGUARD_IDENTITY_ID}" --password-stdin + chainctl auth login --identity-token "${CIRCLE_OIDC_TOKEN}" --identity "${CHAINGUARD_IDENTITY_ID}" + CHAINGUARD_TOKEN=$(chainctl auth token --audience cgr.dev) + echo "${CHAINGUARD_TOKEN}" | docker login cgr.dev -u "oauth2accesstoken" --password-stdin environment_setup: description: Basic Environment setup steps: From 44b59ed8a5a4ed48deea353ad6519d1e35dca4d0 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Mon, 27 Oct 2025 11:14:47 +0000 Subject: [PATCH 11/30] Specify audience cgr.dev in auth login --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f4191345c..b8f1de1d6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -222,7 +222,7 @@ commands: - run: name: Login to Chainguard Registry command: | - chainctl auth login --identity-token "${CIRCLE_OIDC_TOKEN}" --identity "${CHAINGUARD_IDENTITY_ID}" + chainctl auth login --identity-token "${CIRCLE_OIDC_TOKEN}" --identity "${CHAINGUARD_IDENTITY_ID}" --audience cgr.dev CHAINGUARD_TOKEN=$(chainctl auth token --audience cgr.dev) echo "${CHAINGUARD_TOKEN}" | docker login cgr.dev -u "oauth2accesstoken" --password-stdin environment_setup: From 82fe322a348b75fb70f808cd568e0b08ae3c9516 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 11:34:01 +0000 Subject: [PATCH 12/30] Update system packages in Dockerfile --- model-engine/Dockerfile | 15 ++++++++++++--- {federal => model-engine}/sitecustomize.py | 0 2 files changed, 12 insertions(+), 3 deletions(-) rename {federal => model-engine}/sitecustomize.py (100%) diff --git a/model-engine/Dockerfile b/model-engine/Dockerfile index f27ef57b4..bd65d6c60 100644 --- a/model-engine/Dockerfile +++ b/model-engine/Dockerfile @@ -2,7 +2,8 @@ FROM cgr.dev/scale.com/python-fips:3.10.19-dev WORKDIR /workspace USER root -RUN apk update && apk add htop \ +RUN apk update && apk add \ + htop \ dumb-init \ libssh \ openssh-client \ @@ -12,7 +13,15 @@ RUN apk update && apk add htop \ procps \ libcurl-openssl4 \ vim \ - kubectl + kubectl \ + gcc \ + musl-dev \ + python3-dev \ + libffi-dev \ + openssl-dev \ + build-base \ + postgresql-dev \ + postgresql-libs RUN curl -Lo /bin/aws-iam-authenticator https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.5.9/aws-iam-authenticator_0.5.9_linux_amd64 RUN chmod +x /bin/aws-iam-authenticator @@ -22,7 +31,7 @@ RUN chmod -R 777 /workspace RUN pip install awscli==1.34.28 --no-cache-dir -COPY federal/sitecustomize.py /usr/lib/python3.10/site-packages/sitecustomize.py +COPY model-engine/sitecustomize.py /usr/lib/python3.10/site-packages/sitecustomize.py WORKDIR /workspace/model-engine/ COPY model-engine/requirements-test.txt /workspace/model-engine/requirements-test.txt diff --git a/federal/sitecustomize.py b/model-engine/sitecustomize.py similarity index 100% rename from federal/sitecustomize.py rename to model-engine/sitecustomize.py From 209a34a8bb0f50733d3ea2918bc5c6ab916a724d Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 11:44:46 +0000 Subject: [PATCH 13/30] Update Dockerfile packages for chainguard compatbility --- model-engine/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/model-engine/Dockerfile b/model-engine/Dockerfile index bd65d6c60..44598a1c1 100644 --- a/model-engine/Dockerfile +++ b/model-engine/Dockerfile @@ -15,13 +15,13 @@ RUN apk update && apk add \ vim \ kubectl \ gcc \ - musl-dev \ - python3-dev \ + glibc-dev \ + python-3.10-dev \ libffi-dev \ openssl-dev \ build-base \ postgresql-dev \ - postgresql-libs + libpq-16 RUN curl -Lo /bin/aws-iam-authenticator https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.5.9/aws-iam-authenticator_0.5.9_linux_amd64 RUN chmod +x /bin/aws-iam-authenticator From 6f7917984e691409f2a19a8b225dd36cfb127c97 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 12:59:39 +0000 Subject: [PATCH 14/30] update Dockerfile --- .circleci/config.yml | 6 +++--- model-engine/Dockerfile | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b8f1de1d6..62cb45b89 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -184,10 +184,10 @@ jobs: cat model-engine/values_circleci.yaml | envsubst > model-engine/values_circleci_subst.yaml helm install model-engine model-engine --values model-engine/values_circleci_subst.yaml --set tag=$CIRCLE_SHA1 --atomic --debug - run: - name: Change python version to 3.10.14 + name: Change python version to 3.10.19 command: | - pyenv install 3.10.14 - pyenv global 3.10.14 + pyenv install 3.10.19 + pyenv global 3.10.19 - run: name: Install integration test dependencies command: | diff --git a/model-engine/Dockerfile b/model-engine/Dockerfile index 44598a1c1..02b298ee6 100644 --- a/model-engine/Dockerfile +++ b/model-engine/Dockerfile @@ -14,6 +14,7 @@ RUN apk update && apk add \ libcurl-openssl4 \ vim \ kubectl \ + jq \ gcc \ glibc-dev \ python-3.10-dev \ @@ -42,14 +43,13 @@ RUN pip install -r requirements.txt --no-cache-dir RUN pip install -r requirements_override.txt --no-cache-dir COPY model-engine/setup.py /workspace/model-engine/setup.py COPY model-engine/model_engine_server /workspace/model-engine/model_engine_server +#COPY model-engine/service_configs /workspace/model-engine/service_configs RUN pip install -e . COPY integration_tests /workspace/integration_tests WORKDIR /workspace -ENV PYTHONPATH /workspace -ENV WORKSPACE /workspace - -USER nonroot +ENV PYTHONPATH=/workspace +ENV WORKSPACE=/workspace EXPOSE 5000 From ae1bb4e959f9ac86417bca6fbc7052fa1af0289b Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 14:38:43 +0000 Subject: [PATCH 15/30] Revert circleci python version to 3.10.14 --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 62cb45b89..b8f1de1d6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -184,10 +184,10 @@ jobs: cat model-engine/values_circleci.yaml | envsubst > model-engine/values_circleci_subst.yaml helm install model-engine model-engine --values model-engine/values_circleci_subst.yaml --set tag=$CIRCLE_SHA1 --atomic --debug - run: - name: Change python version to 3.10.19 + name: Change python version to 3.10.14 command: | - pyenv install 3.10.19 - pyenv global 3.10.19 + pyenv install 3.10.14 + pyenv global 3.10.14 - run: name: Install integration test dependencies command: | From a465e5133ab8c2456dff90237fc314c562fc9f8d Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 15:55:57 +0000 Subject: [PATCH 16/30] Update hardcoded model-engine image tag used in integration tests --- integration_tests/rest_api_utils.py | 6 +++--- model-engine/Dockerfile | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integration_tests/rest_api_utils.py b/integration_tests/rest_api_utils.py index 7db937dc6..630cfc668 100644 --- a/integration_tests/rest_api_utils.py +++ b/integration_tests/rest_api_utils.py @@ -78,7 +78,7 @@ def my_model(**keyword_args): "flavor": { "flavor": "streaming_enhanced_runnable_image", "repository": "model-engine", - "tag": "830c81ecba2a147022e504917c6ce18b00c2af44", + "tag": os.environ.get("GIT_TAG", "830c81ecba2a147022e504917c6ce18b00c2af44"), "command": [ "dumb-init", "--", @@ -269,7 +269,7 @@ def my_model(**keyword_args): CREATE_DOCKER_IMAGE_BATCH_JOB_BUNDLE_REQUEST: Dict[str, Any] = { "name": format_name("di_batch_job_bundle_1"), "image_repository": "model-engine", - "image_tag": "830c81ecba2a147022e504917c6ce18b00c2af44", + "image_tag": os.environ.get("GIT_TAG", "830c81ecba2a147022e504917c6ce18b00c2af44"), "command": ["jq", ".", "/launch_mount_location/file"], "env": {"ENV1": "VAL1"}, "mount_location": "/launch_mount_location/file", @@ -289,7 +289,7 @@ def my_model(**keyword_args): CREATE_FINE_TUNE_DI_BATCH_JOB_BUNDLE_REQUEST: Dict[str, Any] = { "name": format_name("fine_tune_di_batch_job_bundle_1"), "image_repository": "model-engine", - "image_tag": "830c81ecba2a147022e504917c6ce18b00c2af44", + "image_tag": os.environ.get("GIT_TAG", "830c81ecba2a147022e504917c6ce18b00c2af44"), "command": ["cat", "/launch_mount_location/file"], "env": {"ENV1": "VAL1"}, "mount_location": "/launch_mount_location/file", diff --git a/model-engine/Dockerfile b/model-engine/Dockerfile index 02b298ee6..05469ff0c 100644 --- a/model-engine/Dockerfile +++ b/model-engine/Dockerfile @@ -43,7 +43,7 @@ RUN pip install -r requirements.txt --no-cache-dir RUN pip install -r requirements_override.txt --no-cache-dir COPY model-engine/setup.py /workspace/model-engine/setup.py COPY model-engine/model_engine_server /workspace/model-engine/model_engine_server -#COPY model-engine/service_configs /workspace/model-engine/service_configs +COPY model-engine/service_configs /workspace/model-engine/service_configs RUN pip install -e . COPY integration_tests /workspace/integration_tests From 0eca9cb5f1b341f1cf527f40857be79941b0a25d Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 17:01:51 +0000 Subject: [PATCH 17/30] Fix CircleCI config trying to use hardcoded model-engine image tag for batch jobs pod --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b8f1de1d6..e37c15efd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -158,7 +158,11 @@ jobs: - run: name: Pre-load model-engine image to minikube command: | + # Load the base image for gateway/init containers minikube --logtostderr -v 1 image load model-engine:$CIRCLE_SHA1 + # Tag and load with ECR prefix for batch job containers + docker tag model-engine:$CIRCLE_SHA1 $CIRCLECI_AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/model-engine:$CIRCLE_SHA1 + minikube --logtostderr -v 1 image load $CIRCLECI_AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/model-engine:$CIRCLE_SHA1 - run: name: Pre-load integration test images to minikube command: | From fe8d76496682099a6db51be5375f876f1c446014 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 17:02:24 +0000 Subject: [PATCH 18/30] Mount service_config_circleci.yaml in batch job pods --- .../service_template_config_map.yaml | 36 ++++++++++++++++--- model-engine/Dockerfile | 1 - 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/charts/model-engine/templates/service_template_config_map.yaml b/charts/model-engine/templates/service_template_config_map.yaml index f1cee8c4d..5bb3e75d3 100644 --- a/charts/model-engine/templates/service_template_config_map.yaml +++ b/charts/model-engine/templates/service_template_config_map.yaml @@ -1025,12 +1025,20 @@ data: {{- toYaml . | nindent 12 }} {{- end }} serviceAccountName: {{ $launch_name }} - {{- if $require_aws_config }} volumes: + {{- if $require_aws_config }} - name: config-volume configMap: name: {{ $aws_config_map_name }} - {{- end }} + {{- end }} + {{- if $config_values }} + - name: {{ .Chart.Name }}-service-config-volume + configMap: + name: {{ $launch_name }}-service-config + items: + - key: launch_service_config + path: service_config.yaml + {{- end }} containers: - name: main image: {{ $gateway_repository }}:${GIT_TAG} @@ -1077,12 +1085,16 @@ data: cpu: 4 memory: 32Gi ephemeral-storage: 30Gi - {{- if $require_aws_config }} volumeMounts: + {{- if $require_aws_config }} - name: config-volume mountPath: /opt/.aws/config subPath: config - {{- end }} + {{- end }} + {{- if $config_values }} + - name: {{ .Chart.Name }}-service-config-volume + mountPath: /workspace/model-engine/service_configs + {{- end }} {{- range $device := tuple "cpu" "gpu" }} docker-image-batch-job-{{- $device }}.yaml: |- apiVersion: batch/v1 @@ -1134,6 +1146,14 @@ data: configMap: name: {{ $aws_config_map_name }} {{- end }} + {{- if $config_values }} + - name: {{ .Chart.Name }}-service-config-volume + configMap: + name: {{ $launch_name }}-service-config + items: + - key: launch_service_config + path: service_config.yaml + {{- end }} - name: workdir emptyDir: {} - name: dshm @@ -1178,6 +1198,10 @@ data: mountPath: /opt/.aws/config subPath: config {{- end }} + {{- if $config_values }} + - name: {{ .Chart.Name }}-service-config-volume + mountPath: /workspace/model-engine/service_configs + {{- end }} - name: workdir mountPath: ${MOUNT_PATH} - mountPath: /dev/shm @@ -1212,6 +1236,10 @@ data: mountPath: /opt/.aws/config subPath: config {{- end }} + {{- if $config_values }} + - name: {{ .Chart.Name }}-service-config-volume + mountPath: /workspace/model-engine/service_configs + {{- end }} - name: workdir mountPath: ${MOUNT_PATH} {{- end }} diff --git a/model-engine/Dockerfile b/model-engine/Dockerfile index 05469ff0c..bd24a4a9a 100644 --- a/model-engine/Dockerfile +++ b/model-engine/Dockerfile @@ -43,7 +43,6 @@ RUN pip install -r requirements.txt --no-cache-dir RUN pip install -r requirements_override.txt --no-cache-dir COPY model-engine/setup.py /workspace/model-engine/setup.py COPY model-engine/model_engine_server /workspace/model-engine/model_engine_server -COPY model-engine/service_configs /workspace/model-engine/service_configs RUN pip install -e . COPY integration_tests /workspace/integration_tests From b53b7c932b4bbb6be3d81790f858179733925512 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 17:48:27 +0000 Subject: [PATCH 19/30] Fix broken helm template --- .../model-engine/templates/service_template_config_map.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/model-engine/templates/service_template_config_map.yaml b/charts/model-engine/templates/service_template_config_map.yaml index 5bb3e75d3..dd791d554 100644 --- a/charts/model-engine/templates/service_template_config_map.yaml +++ b/charts/model-engine/templates/service_template_config_map.yaml @@ -1147,7 +1147,7 @@ data: name: {{ $aws_config_map_name }} {{- end }} {{- if $config_values }} - - name: {{ .Chart.Name }}-service-config-volume + - name: model-engine-service-config-volume configMap: name: {{ $launch_name }}-service-config items: @@ -1199,7 +1199,7 @@ data: subPath: config {{- end }} {{- if $config_values }} - - name: {{ .Chart.Name }}-service-config-volume + - name: model-engine-service-config-volume mountPath: /workspace/model-engine/service_configs {{- end }} - name: workdir @@ -1237,7 +1237,7 @@ data: subPath: config {{- end }} {{- if $config_values }} - - name: {{ .Chart.Name }}-service-config-volume + - name: model-engine-service-config-volume mountPath: /workspace/model-engine/service_configs {{- end }} - name: workdir From b054e67eb73822166ad0da685a108515342576c1 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 18:41:24 +0000 Subject: [PATCH 20/30] Add missing infra config and service template config to batch job pods --- .../service_template_config_map.yaml | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/charts/model-engine/templates/service_template_config_map.yaml b/charts/model-engine/templates/service_template_config_map.yaml index dd791d554..348d06799 100644 --- a/charts/model-engine/templates/service_template_config_map.yaml +++ b/charts/model-engine/templates/service_template_config_map.yaml @@ -1031,13 +1031,22 @@ data: configMap: name: {{ $aws_config_map_name }} {{- end }} + - name: service-template-config + configMap: + name: {{ $launch_name }}-service-template-config {{- if $config_values }} - - name: {{ .Chart.Name }}-service-config-volume + - name: service-config-volume configMap: name: {{ $launch_name }}-service-config items: - key: launch_service_config path: service_config.yaml + - name: infra-service-config-volume + configMap: + name: {{ $launch_name }}-service-config + items: + - key: infra_service_config + path: config.yaml {{- end }} containers: - name: main @@ -1091,9 +1100,13 @@ data: mountPath: /opt/.aws/config subPath: config {{- end }} + - name: service-template-config + mountPath: /workspace/model-engine/model_engine_server/infra/gateways/resources/templates {{- if $config_values }} - - name: {{ .Chart.Name }}-service-config-volume + - name: service-config-volume mountPath: /workspace/model-engine/service_configs + - name: infra-service-config-volume + mountPath: /workspace/model-engine/model_engine_server/core/configs {{- end }} {{- range $device := tuple "cpu" "gpu" }} docker-image-batch-job-{{- $device }}.yaml: |- @@ -1147,7 +1160,7 @@ data: name: {{ $aws_config_map_name }} {{- end }} {{- if $config_values }} - - name: model-engine-service-config-volume + - name: service-config-volume configMap: name: {{ $launch_name }}-service-config items: @@ -1199,7 +1212,7 @@ data: subPath: config {{- end }} {{- if $config_values }} - - name: model-engine-service-config-volume + - name: service-config-volume mountPath: /workspace/model-engine/service_configs {{- end }} - name: workdir @@ -1237,7 +1250,7 @@ data: subPath: config {{- end }} {{- if $config_values }} - - name: model-engine-service-config-volume + - name: service-config-volume mountPath: /workspace/model-engine/service_configs {{- end }} - name: workdir From c142b279f940885e8ceafcdd1b3eba31574ed886 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 19:47:17 +0000 Subject: [PATCH 21/30] remove redundant config for batch job pods --- .../model-engine/templates/service_template_config_map.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/charts/model-engine/templates/service_template_config_map.yaml b/charts/model-engine/templates/service_template_config_map.yaml index 348d06799..73e10f542 100644 --- a/charts/model-engine/templates/service_template_config_map.yaml +++ b/charts/model-engine/templates/service_template_config_map.yaml @@ -1031,9 +1031,6 @@ data: configMap: name: {{ $aws_config_map_name }} {{- end }} - - name: service-template-config - configMap: - name: {{ $launch_name }}-service-template-config {{- if $config_values }} - name: service-config-volume configMap: @@ -1100,8 +1097,6 @@ data: mountPath: /opt/.aws/config subPath: config {{- end }} - - name: service-template-config - mountPath: /workspace/model-engine/model_engine_server/infra/gateways/resources/templates {{- if $config_values }} - name: service-config-volume mountPath: /workspace/model-engine/service_configs From f312dfea7751d520852c6978ce99e342e537a6f7 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 20:25:16 +0000 Subject: [PATCH 22/30] enable SHA256 checksums for Celery S3 backend to avoid MD5 decoding issues when creating endpoints --- charts/model-engine/values_circleci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/model-engine/values_circleci.yaml b/charts/model-engine/values_circleci.yaml index ac0074eb1..2d9e70979 100644 --- a/charts/model-engine/values_circleci.yaml +++ b/charts/model-engine/values_circleci.yaml @@ -39,6 +39,10 @@ balloons: # commit from which the image was built. # tag: context: circleci + +# Enable SHA256 checksums for Celery S3 backend to avoid MD5 issues +celery_enable_sha256: true + image: gatewayRepository: model-engine builderRepository: model-engine From f742965bd55e23ccae1b52d3794f1c175bbb164e Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 21:35:47 +0000 Subject: [PATCH 23/30] Fix failing md5 monkey patch --- .../model_engine_server/common/settings.py | 3 ++- .../core/celery/celery_autoscaler.py | 3 ++- .../services/live_endpoint_builder_service.py | 6 +++-- model-engine/sitecustomize.py | 25 ++++++++++++++----- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/model-engine/model_engine_server/common/settings.py b/model-engine/model_engine_server/common/settings.py index 99adc4d7a..64594bfb2 100644 --- a/model-engine/model_engine_server/common/settings.py +++ b/model-engine/model_engine_server/common/settings.py @@ -61,7 +61,8 @@ def generate_destination(user_id: str, endpoint_name: str, endpoint_type: str) - def _generate_deployment_name_parts(user_id: str, endpoint_name: str) -> List[str]: - user_endpoint_hash = hashlib.md5((user_id + endpoint_name).encode("utf-8")).hexdigest() + # Use MD5 for deployment name hashing (non-security purpose) - FIPS compliant + user_endpoint_hash = hashlib.new('md5', (user_id + endpoint_name).encode("utf-8"), usedforsecurity=False).hexdigest() return [ DEPLOYMENT_PREFIX, user_id[:24], diff --git a/model-engine/model_engine_server/core/celery/celery_autoscaler.py b/model-engine/model_engine_server/core/celery/celery_autoscaler.py index 1b74f279b..12d7d475c 100644 --- a/model-engine/model_engine_server/core/celery/celery_autoscaler.py +++ b/model-engine/model_engine_server/core/celery/celery_autoscaler.py @@ -69,7 +69,8 @@ class CeleryAutoscalerParams: def _hash_any_to_int(data: Any): - return int(hashlib.md5(str(data).encode()).hexdigest(), 16) # nosemgrep + # Use MD5 for hashing (non-security purpose) - FIPS compliant with usedforsecurity=False + return int(hashlib.new('md5', str(data).encode(), usedforsecurity=False).hexdigest(), 16) # nosemgrep async def list_deployments(core_api, apps_api) -> Dict[Tuple[str, str], CeleryAutoscalerParams]: diff --git a/model-engine/model_engine_server/infra/services/live_endpoint_builder_service.py b/model-engine/model_engine_server/infra/services/live_endpoint_builder_service.py index 1b32104a5..782b5ca02 100644 --- a/model-engine/model_engine_server/infra/services/live_endpoint_builder_service.py +++ b/model-engine/model_engine_server/infra/services/live_endpoint_builder_service.py @@ -599,7 +599,8 @@ def _get_inject_bundle_image_params( bundle_id = model_bundle.id service_image_str = "-".join([base_image_params.image_tag, GIT_TAG, bundle_id]) # nosemgrep - service_image_hash = hashlib.md5(str(service_image_str).encode("utf-8")).hexdigest() + # Use MD5 for image tag hashing (non-security purpose, required for Docker compatibility) + service_image_hash = hashlib.new('md5', str(service_image_str).encode("utf-8"), usedforsecurity=False).hexdigest() service_image_tag = f"inject-bundle-image-{service_image_hash}" ecr_repo = base_image_params.repo @@ -812,7 +813,8 @@ def _get_restricted_env_vars(env_vars: Dict[str, str]) -> Set[str]: def _get_requirements_hash(requirements: List[str]) -> str: """Identifying hash for endpoint's Python requirements.""" # nosemgrep - return hashlib.md5("\n".join(sorted(requirements)).encode("utf-8")).hexdigest()[:6] + # Use MD5 for requirements hashing (non-security purpose) + return hashlib.new('md5', "\n".join(sorted(requirements)).encode("utf-8"), usedforsecurity=False).hexdigest()[:6] @staticmethod def _get_image_tag(base_image_tag: str, git_tag: str, requirements_hash: str) -> str: diff --git a/model-engine/sitecustomize.py b/model-engine/sitecustomize.py index c643a56b1..1ca5ec8bb 100644 --- a/model-engine/sitecustomize.py +++ b/model-engine/sitecustomize.py @@ -1,17 +1,30 @@ import hashlib -# Replace md5 with sha256 for FIPS compliance -hashlib.md5 = hashlib.sha256 +# FIPS compliance note: +# Python 3.9+ supports hashlib.new('md5', usedforsecurity=False) which allows +# MD5 usage for non-cryptographic purposes even in FIPS mode. +# This is preferred over monkey-patching as it doesn't break code that needs +# MD5 for compatibility with external systems (e.g., Docker image tags). -# Also patch SQLAlchemy specifically +# For SQLAlchemy, we provide a FIPS-compliant md5_hex implementation try: from sqlalchemy.util import langhelpers - def sha256_hex(data): + def fips_md5_hex(data): + """ + FIPS-compliant MD5 hex function for SQLAlchemy. + Uses usedforsecurity=False to allow MD5 in FIPS mode for non-security purposes. + """ if isinstance(data, str): data = data.encode("utf-8") - return hashlib.sha256(data).hexdigest()[:8] # Match MD5 length expectation + # Use MD5 with usedforsecurity=False for FIPS compliance + # Falls back to SHA256 if MD5 is completely unavailable + try: + return hashlib.new('md5', data, usedforsecurity=False).hexdigest() + except (ValueError, TypeError): + # Fallback for older Python or if usedforsecurity not supported + return hashlib.sha256(data).hexdigest()[:32] # Truncate to MD5 length - langhelpers.md5_hex = sha256_hex + langhelpers.md5_hex = fips_md5_hex except ImportError: pass From c8a2c668d963930e2caa984369bb451245feb8df Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 21:44:06 +0000 Subject: [PATCH 24/30] bump sqlalchemy to 2.0.21 to address md5 FIPS compliance --- model-engine/Dockerfile | 2 -- model-engine/requirements.in | 2 +- model-engine/requirements.txt | 2 +- model-engine/sitecustomize.py | 30 ------------------------------ 4 files changed, 2 insertions(+), 34 deletions(-) delete mode 100644 model-engine/sitecustomize.py diff --git a/model-engine/Dockerfile b/model-engine/Dockerfile index bd24a4a9a..54c8b3b3d 100644 --- a/model-engine/Dockerfile +++ b/model-engine/Dockerfile @@ -32,8 +32,6 @@ RUN chmod -R 777 /workspace RUN pip install awscli==1.34.28 --no-cache-dir -COPY model-engine/sitecustomize.py /usr/lib/python3.10/site-packages/sitecustomize.py - WORKDIR /workspace/model-engine/ COPY model-engine/requirements-test.txt /workspace/model-engine/requirements-test.txt COPY model-engine/requirements.txt /workspace/model-engine/requirements.txt diff --git a/model-engine/requirements.in b/model-engine/requirements.in index d503f7b83..3d4162daa 100644 --- a/model-engine/requirements.in +++ b/model-engine/requirements.in @@ -46,7 +46,7 @@ rich~=12.6 sentencepiece==0.1.99 sh~=1.13 smart-open~=5.2 -sqlalchemy[asyncio]~=2.0.4 +sqlalchemy[asyncio]~=2.0.21 sse-starlette==1.6.1 sseclient-py==1.7.2 starlette[full]>=0.36.2 # not used directly, but needs to be pinned for Microsoft security scan diff --git a/model-engine/requirements.txt b/model-engine/requirements.txt index 6e784ecc9..f3fd86577 100644 --- a/model-engine/requirements.txt +++ b/model-engine/requirements.txt @@ -458,7 +458,7 @@ sniffio==1.3.0 # via # anyio # httpx -sqlalchemy[asyncio]==2.0.4 +sqlalchemy[asyncio]==2.0.21 # via # -r model-engine/requirements.in # alembic diff --git a/model-engine/sitecustomize.py b/model-engine/sitecustomize.py deleted file mode 100644 index 1ca5ec8bb..000000000 --- a/model-engine/sitecustomize.py +++ /dev/null @@ -1,30 +0,0 @@ -import hashlib - -# FIPS compliance note: -# Python 3.9+ supports hashlib.new('md5', usedforsecurity=False) which allows -# MD5 usage for non-cryptographic purposes even in FIPS mode. -# This is preferred over monkey-patching as it doesn't break code that needs -# MD5 for compatibility with external systems (e.g., Docker image tags). - -# For SQLAlchemy, we provide a FIPS-compliant md5_hex implementation -try: - from sqlalchemy.util import langhelpers - - def fips_md5_hex(data): - """ - FIPS-compliant MD5 hex function for SQLAlchemy. - Uses usedforsecurity=False to allow MD5 in FIPS mode for non-security purposes. - """ - if isinstance(data, str): - data = data.encode("utf-8") - # Use MD5 with usedforsecurity=False for FIPS compliance - # Falls back to SHA256 if MD5 is completely unavailable - try: - return hashlib.new('md5', data, usedforsecurity=False).hexdigest() - except (ValueError, TypeError): - # Fallback for older Python or if usedforsecurity not supported - return hashlib.sha256(data).hexdigest()[:32] # Truncate to MD5 length - - langhelpers.md5_hex = fips_md5_hex -except ImportError: - pass From 5e4fcf251d993753de3724b5faf2af9d57e10ac1 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 22:51:00 +0000 Subject: [PATCH 25/30] Fix black linting errors --- charts/model-engine/values_circleci.yaml | 1 - integration_tests/rest_api_utils.py | 6 +++--- model-engine/Dockerfile | 1 - model-engine/model_engine_server/common/settings.py | 4 +++- .../core/celery/celery_autoscaler.py | 4 +++- .../infra/services/live_endpoint_builder_service.py | 10 ++++++++-- 6 files changed, 17 insertions(+), 9 deletions(-) diff --git a/charts/model-engine/values_circleci.yaml b/charts/model-engine/values_circleci.yaml index 2d9e70979..f3f1f1c11 100644 --- a/charts/model-engine/values_circleci.yaml +++ b/charts/model-engine/values_circleci.yaml @@ -40,7 +40,6 @@ balloons: # tag: context: circleci -# Enable SHA256 checksums for Celery S3 backend to avoid MD5 issues celery_enable_sha256: true image: diff --git a/integration_tests/rest_api_utils.py b/integration_tests/rest_api_utils.py index 630cfc668..61b394b63 100644 --- a/integration_tests/rest_api_utils.py +++ b/integration_tests/rest_api_utils.py @@ -78,7 +78,7 @@ def my_model(**keyword_args): "flavor": { "flavor": "streaming_enhanced_runnable_image", "repository": "model-engine", - "tag": os.environ.get("GIT_TAG", "830c81ecba2a147022e504917c6ce18b00c2af44"), + "tag": os.environ.get("GIT_TAG"), "command": [ "dumb-init", "--", @@ -269,7 +269,7 @@ def my_model(**keyword_args): CREATE_DOCKER_IMAGE_BATCH_JOB_BUNDLE_REQUEST: Dict[str, Any] = { "name": format_name("di_batch_job_bundle_1"), "image_repository": "model-engine", - "image_tag": os.environ.get("GIT_TAG", "830c81ecba2a147022e504917c6ce18b00c2af44"), + "image_tag": os.environ.get("GIT_TAG"), "command": ["jq", ".", "/launch_mount_location/file"], "env": {"ENV1": "VAL1"}, "mount_location": "/launch_mount_location/file", @@ -289,7 +289,7 @@ def my_model(**keyword_args): CREATE_FINE_TUNE_DI_BATCH_JOB_BUNDLE_REQUEST: Dict[str, Any] = { "name": format_name("fine_tune_di_batch_job_bundle_1"), "image_repository": "model-engine", - "image_tag": os.environ.get("GIT_TAG", "830c81ecba2a147022e504917c6ce18b00c2af44"), + "image_tag": os.environ.get("GIT_TAG"), "command": ["cat", "/launch_mount_location/file"], "env": {"ENV1": "VAL1"}, "mount_location": "/launch_mount_location/file", diff --git a/model-engine/Dockerfile b/model-engine/Dockerfile index 54c8b3b3d..28b9d5761 100644 --- a/model-engine/Dockerfile +++ b/model-engine/Dockerfile @@ -1,6 +1,5 @@ FROM cgr.dev/scale.com/python-fips:3.10.19-dev WORKDIR /workspace -USER root RUN apk update && apk add \ htop \ diff --git a/model-engine/model_engine_server/common/settings.py b/model-engine/model_engine_server/common/settings.py index 64594bfb2..9942de555 100644 --- a/model-engine/model_engine_server/common/settings.py +++ b/model-engine/model_engine_server/common/settings.py @@ -62,7 +62,9 @@ def generate_destination(user_id: str, endpoint_name: str, endpoint_type: str) - def _generate_deployment_name_parts(user_id: str, endpoint_name: str) -> List[str]: # Use MD5 for deployment name hashing (non-security purpose) - FIPS compliant - user_endpoint_hash = hashlib.new('md5', (user_id + endpoint_name).encode("utf-8"), usedforsecurity=False).hexdigest() + user_endpoint_hash = hashlib.new( + "md5", (user_id + endpoint_name).encode("utf-8"), usedforsecurity=False + ).hexdigest() return [ DEPLOYMENT_PREFIX, user_id[:24], diff --git a/model-engine/model_engine_server/core/celery/celery_autoscaler.py b/model-engine/model_engine_server/core/celery/celery_autoscaler.py index 12d7d475c..78a4da3f5 100644 --- a/model-engine/model_engine_server/core/celery/celery_autoscaler.py +++ b/model-engine/model_engine_server/core/celery/celery_autoscaler.py @@ -70,7 +70,9 @@ class CeleryAutoscalerParams: def _hash_any_to_int(data: Any): # Use MD5 for hashing (non-security purpose) - FIPS compliant with usedforsecurity=False - return int(hashlib.new('md5', str(data).encode(), usedforsecurity=False).hexdigest(), 16) # nosemgrep + return int( + hashlib.new("md5", str(data).encode(), usedforsecurity=False).hexdigest(), 16 + ) # nosemgrep async def list_deployments(core_api, apps_api) -> Dict[Tuple[str, str], CeleryAutoscalerParams]: diff --git a/model-engine/model_engine_server/infra/services/live_endpoint_builder_service.py b/model-engine/model_engine_server/infra/services/live_endpoint_builder_service.py index 782b5ca02..3494ea774 100644 --- a/model-engine/model_engine_server/infra/services/live_endpoint_builder_service.py +++ b/model-engine/model_engine_server/infra/services/live_endpoint_builder_service.py @@ -600,7 +600,9 @@ def _get_inject_bundle_image_params( service_image_str = "-".join([base_image_params.image_tag, GIT_TAG, bundle_id]) # nosemgrep # Use MD5 for image tag hashing (non-security purpose, required for Docker compatibility) - service_image_hash = hashlib.new('md5', str(service_image_str).encode("utf-8"), usedforsecurity=False).hexdigest() + service_image_hash = hashlib.new( + "md5", str(service_image_str).encode("utf-8"), usedforsecurity=False + ).hexdigest() service_image_tag = f"inject-bundle-image-{service_image_hash}" ecr_repo = base_image_params.repo @@ -814,7 +816,11 @@ def _get_requirements_hash(requirements: List[str]) -> str: """Identifying hash for endpoint's Python requirements.""" # nosemgrep # Use MD5 for requirements hashing (non-security purpose) - return hashlib.new('md5', "\n".join(sorted(requirements)).encode("utf-8"), usedforsecurity=False).hexdigest()[:6] + return hashlib.new( + "md5", + "\n".join(sorted(requirements)).encode("utf-8"), + usedforsecurity=False, + ).hexdigest()[:6] @staticmethod def _get_image_tag(base_image_tag: str, git_tag: str, requirements_hash: str) -> str: From 506b0bf3363e95fefb508c774aed29b54cc90f13 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 22:53:57 +0000 Subject: [PATCH 26/30] wrap Dockerfile layers between root and nonroot user --- model-engine/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/model-engine/Dockerfile b/model-engine/Dockerfile index 28b9d5761..0a9f29ef7 100644 --- a/model-engine/Dockerfile +++ b/model-engine/Dockerfile @@ -1,5 +1,6 @@ FROM cgr.dev/scale.com/python-fips:3.10.19-dev WORKDIR /workspace +USER root RUN apk update && apk add \ htop \ @@ -48,4 +49,6 @@ WORKDIR /workspace ENV PYTHONPATH=/workspace ENV WORKSPACE=/workspace +USER nonroot + EXPOSE 5000 From 66cbd331bdb0eed5302278c74252d43e7fc51d74 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 23:07:52 +0000 Subject: [PATCH 27/30] Remove the federal/ directory since Dockerfile is now FIPS compliant and doesn't require monkey patching --- federal/Dockerfile.chainguard | 47 ----------------------------------- 1 file changed, 47 deletions(-) delete mode 100644 federal/Dockerfile.chainguard diff --git a/federal/Dockerfile.chainguard b/federal/Dockerfile.chainguard deleted file mode 100644 index b28d96caf..000000000 --- a/federal/Dockerfile.chainguard +++ /dev/null @@ -1,47 +0,0 @@ -# federal/Dockerfile.chainguard -FROM cgr.dev/scale.com/python-fips:3.10.15-dev -WORKDIR /workspace -USER root - -RUN apk update && apk add htop \ - dumb-init \ - libssh \ - openssh-client \ - iftop \ - curl \ - curl-dev \ - procps \ - libcurl-openssl4 \ - vim \ - kubectl - -RUN curl -Lo /bin/aws-iam-authenticator https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.5.9/aws-iam-authenticator_0.5.9_linux_amd64 -RUN chmod +x /bin/aws-iam-authenticator - -RUN pip install pip==24.2 -RUN chmod -R 777 /workspace - -RUN pip install awscli==1.34.28 --no-cache-dir - -COPY federal/sitecustomize.py /usr/lib/python3.10/site-packages/sitecustomize.py - -WORKDIR /workspace/model-engine/ -COPY model-engine/requirements-test.txt /workspace/model-engine/requirements-test.txt -COPY model-engine/requirements.txt /workspace/model-engine/requirements.txt -COPY model-engine/requirements_override.txt /workspace/model-engine/requirements_override.txt -RUN pip install -r requirements-test.txt --no-cache-dir -RUN pip install -r requirements.txt --no-cache-dir -RUN pip install -r requirements_override.txt --no-cache-dir -COPY model-engine/setup.py /workspace/model-engine/setup.py -COPY model-engine/model_engine_server /workspace/model-engine/model_engine_server -RUN pip install -e . - -COPY integration_tests /workspace/integration_tests - -WORKDIR /workspace -ENV PYTHONPATH /workspace -ENV WORKSPACE /workspace - -USER nonroot - -EXPOSE 5000 \ No newline at end of file From e27a32dce0ffec8a04fb93725887864024e022d3 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Tue, 28 Oct 2025 23:12:22 +0000 Subject: [PATCH 28/30] set celery_enable_sha256 to true in all configs for FIPS compliance --- charts/model-engine/values.yaml | 2 +- charts/model-engine/values_circleci.yaml | 4 +--- charts/model-engine/values_sample.yaml | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/charts/model-engine/values.yaml b/charts/model-engine/values.yaml index b0335ddbb..24bc70493 100644 --- a/charts/model-engine/values.yaml +++ b/charts/model-engine/values.yaml @@ -3,7 +3,7 @@ spellbook: enabled: false # celery_enable_sha256 [optional] uses SHA256 hashes for federal compliance mode (FIPS, enhanced security) -celery_enable_sha256: null +celery_enable_sha256: true # debug_mode [optional] enables detailed debug logging for infrastructure components debug_mode: null diff --git a/charts/model-engine/values_circleci.yaml b/charts/model-engine/values_circleci.yaml index f3f1f1c11..94fbddaec 100644 --- a/charts/model-engine/values_circleci.yaml +++ b/charts/model-engine/values_circleci.yaml @@ -1,6 +1,6 @@ # This is a YAML-formatted file. -celery_enable_sha256: null +celery_enable_sha256: true celery_broker_type_redis: null debug_mode: null @@ -40,8 +40,6 @@ balloons: # tag: context: circleci -celery_enable_sha256: true - image: gatewayRepository: model-engine builderRepository: model-engine diff --git a/charts/model-engine/values_sample.yaml b/charts/model-engine/values_sample.yaml index 9068b5721..dc6ffa623 100644 --- a/charts/model-engine/values_sample.yaml +++ b/charts/model-engine/values_sample.yaml @@ -1,7 +1,7 @@ # This is a YAML-formatted file. # celery_enable_sha256 [optional] uses SHA256 hashes for federal compliance mode (FIPS, enhanced security) -celery_enable_sha256: null +celery_enable_sha256: true # debug_mode [optional] enables detailed debug logging for infrastructure components debug_mode: null From fb479f66f3f65cd6edb6790d68a53c5452719cb9 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Sun, 2 Nov 2025 23:51:29 +0000 Subject: [PATCH 29/30] make changes backwards compatible by having separate Dockerfiles --- charts/model-engine/values.yaml | 2 +- charts/model-engine/values_circleci.yaml | 3 +- charts/model-engine/values_sample.yaml | 2 +- model-engine/Dockerfile | 61 +++++++++++++----------- model-engine/Dockerfile.fips | 54 +++++++++++++++++++++ 5 files changed, 89 insertions(+), 33 deletions(-) create mode 100644 model-engine/Dockerfile.fips diff --git a/charts/model-engine/values.yaml b/charts/model-engine/values.yaml index 24bc70493..b0335ddbb 100644 --- a/charts/model-engine/values.yaml +++ b/charts/model-engine/values.yaml @@ -3,7 +3,7 @@ spellbook: enabled: false # celery_enable_sha256 [optional] uses SHA256 hashes for federal compliance mode (FIPS, enhanced security) -celery_enable_sha256: true +celery_enable_sha256: null # debug_mode [optional] enables detailed debug logging for infrastructure components debug_mode: null diff --git a/charts/model-engine/values_circleci.yaml b/charts/model-engine/values_circleci.yaml index 94fbddaec..ac0074eb1 100644 --- a/charts/model-engine/values_circleci.yaml +++ b/charts/model-engine/values_circleci.yaml @@ -1,6 +1,6 @@ # This is a YAML-formatted file. -celery_enable_sha256: true +celery_enable_sha256: null celery_broker_type_redis: null debug_mode: null @@ -39,7 +39,6 @@ balloons: # commit from which the image was built. # tag: context: circleci - image: gatewayRepository: model-engine builderRepository: model-engine diff --git a/charts/model-engine/values_sample.yaml b/charts/model-engine/values_sample.yaml index dc6ffa623..9068b5721 100644 --- a/charts/model-engine/values_sample.yaml +++ b/charts/model-engine/values_sample.yaml @@ -1,7 +1,7 @@ # This is a YAML-formatted file. # celery_enable_sha256 [optional] uses SHA256 hashes for federal compliance mode (FIPS, enhanced security) -celery_enable_sha256: true +celery_enable_sha256: null # debug_mode [optional] enables detailed debug logging for infrastructure components debug_mode: null diff --git a/model-engine/Dockerfile b/model-engine/Dockerfile index 0a9f29ef7..9d5602e1e 100644 --- a/model-engine/Dockerfile +++ b/model-engine/Dockerfile @@ -1,37 +1,42 @@ -FROM cgr.dev/scale.com/python-fips:3.10.19-dev +# syntax = docker/dockerfile:experimental + +FROM python:3.10.15-slim as model-engine WORKDIR /workspace -USER root - -RUN apk update && apk add \ - htop \ - dumb-init \ - libssh \ - openssh-client \ - iftop \ - curl \ - curl-dev \ - procps \ - libcurl-openssl4 \ - vim \ - kubectl \ - jq \ - gcc \ - glibc-dev \ - python-3.10-dev \ - libffi-dev \ - openssl-dev \ - build-base \ - postgresql-dev \ - libpq-16 + +RUN apt-get update && apt-get install -y \ + apt-utils \ + dumb-init \ + git \ + ssh \ + htop \ + iftop \ + vim \ + curl \ + procps \ + libcurl4-openssl-dev \ + libssl-dev \ + python3-dev \ + gcc \ + build-essential \ + telnet \ + && rm -rf /var/lib/apt/lists/* RUN curl -Lo /bin/aws-iam-authenticator https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.5.9/aws-iam-authenticator_0.5.9_linux_amd64 RUN chmod +x /bin/aws-iam-authenticator +# Install kubectl +RUN curl -LO "https://dl.k8s.io/release/v1.23.13/bin/linux/amd64/kubectl" \ + && chmod +x kubectl \ + && mv kubectl /usr/local/bin/kubectl + +# Pin pip version RUN pip install pip==24.2 RUN chmod -R 777 /workspace +# Install AWS CLI RUN pip install awscli==1.34.28 --no-cache-dir +## grab model_engine_server project (w/ requirements install layer caching) WORKDIR /workspace/model-engine/ COPY model-engine/requirements-test.txt /workspace/model-engine/requirements-test.txt COPY model-engine/requirements.txt /workspace/model-engine/requirements.txt @@ -46,9 +51,7 @@ RUN pip install -e . COPY integration_tests /workspace/integration_tests WORKDIR /workspace -ENV PYTHONPATH=/workspace -ENV WORKSPACE=/workspace - -USER nonroot +ENV PYTHONPATH /workspace +ENV WORKSPACE /workspace -EXPOSE 5000 +EXPOSE 5000 \ No newline at end of file diff --git a/model-engine/Dockerfile.fips b/model-engine/Dockerfile.fips new file mode 100644 index 000000000..0a9f29ef7 --- /dev/null +++ b/model-engine/Dockerfile.fips @@ -0,0 +1,54 @@ +FROM cgr.dev/scale.com/python-fips:3.10.19-dev +WORKDIR /workspace +USER root + +RUN apk update && apk add \ + htop \ + dumb-init \ + libssh \ + openssh-client \ + iftop \ + curl \ + curl-dev \ + procps \ + libcurl-openssl4 \ + vim \ + kubectl \ + jq \ + gcc \ + glibc-dev \ + python-3.10-dev \ + libffi-dev \ + openssl-dev \ + build-base \ + postgresql-dev \ + libpq-16 + +RUN curl -Lo /bin/aws-iam-authenticator https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.5.9/aws-iam-authenticator_0.5.9_linux_amd64 +RUN chmod +x /bin/aws-iam-authenticator + +RUN pip install pip==24.2 +RUN chmod -R 777 /workspace + +RUN pip install awscli==1.34.28 --no-cache-dir + +WORKDIR /workspace/model-engine/ +COPY model-engine/requirements-test.txt /workspace/model-engine/requirements-test.txt +COPY model-engine/requirements.txt /workspace/model-engine/requirements.txt +COPY model-engine/requirements_override.txt /workspace/model-engine/requirements_override.txt +RUN pip install -r requirements-test.txt --no-cache-dir +RUN pip install -r requirements.txt --no-cache-dir +RUN pip install -r requirements_override.txt --no-cache-dir +COPY model-engine/setup.py /workspace/model-engine/setup.py +COPY model-engine/model_engine_server /workspace/model-engine/model_engine_server +RUN pip install -e . + +COPY integration_tests /workspace/integration_tests + +WORKDIR /workspace +ENV PYTHONPATH=/workspace +ENV WORKSPACE=/workspace + +USER nonroot + +EXPOSE 5000 From 9b11f297dd554a8fc6339bf2b95a820477455d43 Mon Sep 17 00:00:00 2001 From: Valentine Dragan Date: Sun, 2 Nov 2025 23:53:07 +0000 Subject: [PATCH 30/30] formatting --- model-engine/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model-engine/Dockerfile b/model-engine/Dockerfile index 9d5602e1e..45cd9630d 100644 --- a/model-engine/Dockerfile +++ b/model-engine/Dockerfile @@ -54,4 +54,4 @@ WORKDIR /workspace ENV PYTHONPATH /workspace ENV WORKSPACE /workspace -EXPOSE 5000 \ No newline at end of file +EXPOSE 5000