CLOUDP-295785 - staging support for image building (#336)

# Summary

PR integrates `staging` build scenario with `atomic_pipeline.py` and the
e2e tests. `staging` repositories are created in AWS ECR under
`/staging` dir. List of new `staging/` repositories:

<img width="2239" height="756" alt="Screenshot 2025-08-13 at 14 39 55"
src="https://github.com/user-attachments/assets/a23a500f-d0bd-4f39-9a74-72a2ca3ad6f6"
/>

Example AWS page with details:

<img width="1654" height="994" alt="Screenshot 2025-08-11 at 21 26 51"
src="https://github.com/user-attachments/assets/4e059aeb-fac3-4208-8cd0-a00692f7d673"
/>

Previously all of the evergreen patch builds were targeting
`268558157000.dkr.ecr.us-east-1.amazonaws.com/dev` repository. Now,
depending on the `BUILD_SCENARIO` environment variable images will be
pushed to and pulled from
different registry:
 - PATCH_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com/dev"
-
STAGING_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com/staging"
 - RELEASE_REGISTRY="quay.io/mongodb"

Additional changes:
- re-added missing `latest` tag for staging builds. This is used for
local testing
- moved all registry variables declarations to `root-context`.
Previously they were duplicated in both
`scripts/dev/contexts/evg-private-context` and
`scripts/dev/contexts/local-defaults-context`
- fixes multi arch smoke tests configuration. We can now run them also
for released images (tbd)
- started to use `OPERATOR_VERSION` instead of `VERSION_ID`.
`VERSION_ID` was only related to patch builds, which did not apply for
staging and release builds
- BASE_REPO_URL_SHARED is now
`268558157000.dkr.ecr.us-east-1.amazonaws.com/staging` because all
latest tags are pushed on master builds
- `build_scenario` now needs to be passed explicitly to `pipeline.py` as
well as `version` (apart from `agent` image)

## Proof of Work

Building all images is passing for staging scenario ->
https://spruce.mongodb.com/version/68c7ce53672666000716401a/tasks?page=0&sorts=STATUS%3AASC%3BBASE_STATUS%3ADESC

Building all images is passing for release scenario (apart from kubectl
which requires tag) ->
https://spruce.mongodb.com/version/68c911ef8d38f100074fa2db/tasks?sorts=STATUS%3AASC%3BBASE_STATUS%3ADESC


Example image stored together with signature ->
https://us-east-1.console.aws.amazon.com/ecr/repositories/private/268558157000/staging/mongodb-kubernetes-init-database?region=us-east-1

## Remaining work

- [ ] Add guide on how to run `development` scenario for building the
images using evergreen and directly `pipeline.py`
- [ ] Fix `dummy` version in the `release_info.json` - this is already
planned in
[here](https://docs.google.com/document/d/1eJ8iKsI0libbpcJakGjxcPfbrTn8lmcZDbQH1UqMR_g/edit?tab=t.p76ry15gwmkk#bookmark=id.egtg1p95d1)

## Checklist

- [ ] Have you linked a jira ticket and/or is the ticket in the title?
- [ ] Have you checked whether your jira ticket required DOCSP changes?
- [ ] Have you added changelog file?
    - use `skip-changelog` label if not needed
- refer to [Changelog files and Release
Notes](https://github.com/mongodb/mongodb-kubernetes/blob/master/CONTRIBUTING.md#changelog-files-and-release-notes)
section in CONTRIBUTING.md for more details

Author: MaciejKaras

.evergreen-functions.yml

@@ -45,7 +45,8 @@ variables:
       - community_private_preview_pullsecret_dockerconfigjson
       - RELEASE_INITIAL_VERSION
       - RELEASE_INITIAL_COMMIT_SHA
-      - RELEASE_OPERATOR_VERSION
+      - OPERATOR_VERSION
+      - BUILD_SCENARIO
 
 functions:
 
@@ -509,43 +510,63 @@ functions:
       params:
         shell: bash
         working_dir: src/github.com/mongodb/mongodb-kubernetes
-        include_expansions_in_env:
-          - version_id
         add_to_path:
           - ${workdir}/bin
         binary: scripts/evergreen/e2e/build_tests_image_ibm.sh
 
-  pipeline_migrate_agents:
+  pipeline:
     - command: subprocess.exec
-      retry_on_failure: false
+      retry_on_failure: true
       type: setup
       params:
         shell: bash
         <<: *e2e_include_expansions_in_env
         working_dir: src/github.com/mongodb/mongodb-kubernetes
-        binary: scripts/release/pipeline_migrate_agent.sh
+        binary: scripts/release/pipeline.sh
+        env:
+          IMAGE_NAME: ${IMAGE_NAME}
 
-  pipeline:
+  pipeline_agent:
+    - command: subprocess.exec
+      retry_on_failure: true
+      type: setup
+      params:
+        shell: bash
+        <<: *e2e_include_expansions_in_env
+        working_dir: src/github.com/mongodb/mongodb-kubernetes
+        binary: scripts/release/pipeline_agent.sh
+        env:
+          FLAGS: ${FLAGS}
+
+  pipeline_ops_manager:
+    - command: subprocess.exec
+      retry_on_failure: true
+      type: setup
+      params:
+        shell: bash
+        <<: *e2e_include_expansions_in_env
+        working_dir: src/github.com/mongodb/mongodb-kubernetes
+        binary: scripts/release/pipeline_ops_manager.sh
+
+  pipeline_readiness_probe:
     - command: subprocess.exec
       retry_on_failure: true
       type: setup
       params:
         shell: bash
         <<: *e2e_include_expansions_in_env
         working_dir: src/github.com/mongodb/mongodb-kubernetes
-        binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name} ${all_agents} ${build_scenario}
+        binary: scripts/release/pipeline_readiness_probe.sh
 
-  # TODO: this function is very similar to pipeline and it will joined with it in the future
-  release_operator_pipeline:
+  pipeline_version_upgrade_hook:
     - command: subprocess.exec
       retry_on_failure: true
       type: setup
       params:
         shell: bash
         <<: *e2e_include_expansions_in_env
         working_dir: src/github.com/mongodb/mongodb-kubernetes
-        # By default, use the git tag that triggered the task which can be overridden with RELEASE_OPERATOR_VERSION
-        binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py ${image_name} --build-scenario release --version ${RELEASE_OPERATOR_VERSION|*triggered_by_git_tag}
+        binary: scripts/release/pipeline_version_upgrade_hook.sh
 
   teardown_cloud_qa_all:
     - command: shell.exec
@@ -846,7 +867,7 @@ functions:
           - macos_notary_secret
           - workdir
           - triggered_by_git_tag
-          - RELEASE_OPERATOR_VERSION
+          - OPERATOR_VERSION
         env:
           XDG_CONFIG_HOME: ${go_base_path}${workdir}
           GO111MODULE: "on"
@@ -856,7 +877,7 @@ functions:
         # shell.exec EVG Task doesn't have add_to_path, so we need to explicitly add the path export below.
         script: |
           set -Eeu pipefail
-          export GORELEASER_CURRENT_TAG=${RELEASE_OPERATOR_VERSION|*triggered_by_git_tag}
+          export GORELEASER_CURRENT_TAG=${OPERATOR_VERSION|*triggered_by_git_tag}
           export PATH=$GOROOT/bin:$PATH
           export GITHUB_TOKEN=${generated_token}
           ${workdir}/goreleaser release --clean

.evergreen-release.yml

@@ -12,8 +12,7 @@ tasks:
       - func: build_multi_cluster_binary
       - func: pipeline
         vars:
-          image_name: meko-tests
-          build_scenario: --build-scenario patch
+          IMAGE_NAME: meko-tests
 
   - name: release_operator
     tags: [ "image_release" ]
@@ -22,9 +21,9 @@ tasks:
       - func: clone
       - func: setup_building_host
       - func: quay_login
-      - func: release_operator_pipeline
+      - func: pipeline
         vars:
-          image_name: operator
+          IMAGE_NAME: operator
 
   # Releases init images to Quay
   - name: release_init_appdb
@@ -34,9 +33,9 @@ tasks:
       - func: clone
       - func: setup_building_host
       - func: quay_login
-      - func: release_operator_pipeline
+      - func: pipeline
         vars:
-          image_name: init-appdb
+          IMAGE_NAME: init-appdb
 
   - name: release_init_database
     tags: [ "image_release" ]
@@ -45,9 +44,9 @@ tasks:
       - func: clone
       - func: setup_building_host
       - func: quay_login
-      - func: release_operator_pipeline
+      - func: pipeline
         vars:
-          image_name: init-database
+          IMAGE_NAME: init-database
 
   - name: release_init_ops_manager
     tags: [ "image_release" ]
@@ -56,9 +55,9 @@ tasks:
       - func: clone
       - func: setup_building_host
       - func: quay_login
-      - func: release_operator_pipeline
+      - func: pipeline
         vars:
-          image_name: init-ops-manager
+          IMAGE_NAME: init-ops-manager
 
   - name: release_database
     tags: [ "image_release" ]
@@ -67,9 +66,9 @@ tasks:
       - func: clone
       - func: setup_building_host
       - func: quay_login
-      - func: release_operator_pipeline
+      - func: pipeline
         vars:
-          image_name: database
+          IMAGE_NAME: database
 
   - name: prepare_and_upload_openshift_bundles
     tags: [ "openshift_bundles" ]