From 2e03d4ad6ea147018a0bbdd481179bafdff59620 Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Fri, 27 Mar 2026 10:34:25 -0700 Subject: [PATCH 01/11] Set up the foundations for transitioning to Terraform --- .../cloud-build/terraform/triggers/.gitignore | 3 +++ .../cloud-build/terraform/triggers/README.md | 20 +++++++++++++++++ .../cloud-build/terraform/triggers/backend.tf | 6 +++++ .../cloud-build/terraform/triggers/locals.tf | 22 +++++++++++++++++++ .../terraform/triggers/providers.tf | 13 +++++++++++ .../terraform/triggers/terraform.tfvars | 4 ++++ .../terraform/triggers/variables.tf | 22 +++++++++++++++++++ 7 files changed, 90 insertions(+) create mode 100644 share/ramble/cloud-build/terraform/triggers/.gitignore create mode 100644 share/ramble/cloud-build/terraform/triggers/README.md create mode 100644 share/ramble/cloud-build/terraform/triggers/backend.tf create mode 100644 share/ramble/cloud-build/terraform/triggers/locals.tf create mode 100644 share/ramble/cloud-build/terraform/triggers/providers.tf create mode 100644 share/ramble/cloud-build/terraform/triggers/terraform.tfvars create mode 100644 share/ramble/cloud-build/terraform/triggers/variables.tf diff --git a/share/ramble/cloud-build/terraform/triggers/.gitignore b/share/ramble/cloud-build/terraform/triggers/.gitignore new file mode 100644 index 000000000..048ba8a6e --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/.gitignore @@ -0,0 +1,3 @@ +# We don't ignore any tfstate files, as those should reside on the GCS bucket instead +.terraform/ +.terraform.lock.hcl \ No newline at end of file diff --git a/share/ramble/cloud-build/terraform/triggers/README.md b/share/ramble/cloud-build/terraform/triggers/README.md new file mode 100644 index 000000000..a9f7e3504 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/README.md @@ -0,0 +1,20 @@ +# Ramble Cloud Build Image Triggers + +This directory contains Terraform configuration to deploy and manage Google Cloud Build Triggers used by the Ramble repository. + +## How to deploy + +The deployment states are stored in a GCS bucket, to allow for running Terraform from different locations. The bucket was created with: + +```bash +gcloud storage buckets create gs://ramble-terraform-state --project=ramble-eng --location=us-central1 +``` + +With the bucket created, the triggers can be deployed with: + +```bash +terraform init +# Optional to check the changes to be made +terraform plan +terraform apply --auto-approve +``` diff --git a/share/ramble/cloud-build/terraform/triggers/backend.tf b/share/ramble/cloud-build/terraform/triggers/backend.tf new file mode 100644 index 000000000..37d6fbb20 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/backend.tf @@ -0,0 +1,6 @@ +terraform { + backend "gcs" { + bucket = "ramble-terraform-state" + prefix = "terraform/state/image-triggers" + } +} diff --git a/share/ramble/cloud-build/terraform/triggers/locals.tf b/share/ramble/cloud-build/terraform/triggers/locals.tf new file mode 100644 index 000000000..19f39607f --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/locals.tf @@ -0,0 +1,22 @@ +locals { + # This matrix is used by both the image-build and unit-test triggers + image_matrix = [ + { python = "3.12.0", spack = "v0.21.2", base = "debian", base_ver = "12.5" }, + { python = "3.8.0", spack = "v0.21.2", base = "debian", base_ver = "12.5" }, + { python = "3.12.1", spack = "v0.22.1", base = "debian", base_ver = "12.5" }, + { python = "3.8.0", spack = "v0.22.1", base = "debian", base_ver = "12.5" }, + { python = "3.13.5", spack = "v1.0.0", base = "debian", base_ver = "12.5" }, + { python = "3.7.17", spack = "v1.0.0", base = "debian", base_ver = "12.5" }, + { python = "3.12.0", spack = "v0.21.2", base = "rockylinux", base_ver = "8" }, + { python = "3.7.0", spack = "v0.21.2", base = "rockylinux", base_ver = "8" }, + { python = "3.12.1", spack = "v0.22.1", base = "rockylinux", base_ver = "8" }, + { python = "3.7.0", spack = "v0.22.1", base = "rockylinux", base_ver = "8" }, + { python = "3.13.5", spack = "v1.0.0", base = "rockylinux", base_ver = "8" }, + { python = "3.7.17", spack = "v1.0.0", base = "rockylinux", base_ver = "8" }, + ] + + pm_map = { + "debian" = "apt" + "rockylinux" = "yum" + } +} diff --git a/share/ramble/cloud-build/terraform/triggers/providers.tf b/share/ramble/cloud-build/terraform/triggers/providers.tf new file mode 100644 index 000000000..b404ad86d --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/providers.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 7.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} diff --git a/share/ramble/cloud-build/terraform/triggers/terraform.tfvars b/share/ramble/cloud-build/terraform/triggers/terraform.tfvars new file mode 100644 index 000000000..1c92804de --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/terraform.tfvars @@ -0,0 +1,4 @@ +project_id = "ramble-eng" +region = "us-central1" +github_owner = "GoogleCloudPlatform" +github_repo = "ramble" diff --git a/share/ramble/cloud-build/terraform/triggers/variables.tf b/share/ramble/cloud-build/terraform/triggers/variables.tf new file mode 100644 index 000000000..393526d48 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/variables.tf @@ -0,0 +1,22 @@ +variable "project_id" { + type = string + description = "The GCP Project ID to host the Cloud Build triggers" +} + +variable "region" { + type = string + description = "The GCP Region to deploy the triggers into" + default = "us-central1" +} + +variable "github_owner" { + type = string + description = "The GitHub organization or user hosting the repository" + default = "GoogleCloudPlatform" +} + +variable "github_repo" { + type = string + description = "The GitHub repository name" + default = "ramble" +} From 48dca848247c36e04c4d55f0f10c547a4271c9b1 Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Fri, 27 Mar 2026 10:39:42 -0700 Subject: [PATCH 02/11] Add in the unit-test triggers --- .../terraform/triggers/pr_unit_tests.tf | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf diff --git a/share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf b/share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf new file mode 100644 index 000000000..5ed5cb35c --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf @@ -0,0 +1,35 @@ +resource "google_cloudbuild_trigger" "pr_unit_tests" { + for_each = { for img in local.image_matrix : "${img.base}${img.base_ver}-py${img.python}-spack${img.spack}" => img } + + name = "PR-Unit-Tests-${each.value.base}${replace(each.value.base_ver, ".", "-")}-${replace(each.value.spack, ".", "-")}spack-${replace(each.value.python, ".", "-")}python" + description = "Run unit tests and linting on Ramble pull requests" + + github { + owner = var.github_owner + name = var.github_repo + pull_request { + branch = "(?:main|develop)" + comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY" + } + } + + ignored_files = [ + "lib/ramble/docs/**" + ] + + included_files = [ + "lib/ramble/**", + "var/ramble/repos/**", + "share/ramble/cloud-build/**", + "conftest.py" + ] + + filename = "share/ramble/cloud-build/ramble-pr-unit-tests.yaml" + + substitutions = { + _BASE_IMG = each.value.base + _BASE_VER = each.value.base_ver + _PYTHON_VER = each.value.python + _SPACK_REF = each.value.spack + } +} From 9e2418a1dfa2eac80e5c1016e2214a6ac4049ca7 Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Fri, 27 Mar 2026 10:39:59 -0700 Subject: [PATCH 03/11] Add in the perf test triggers --- .../terraform/triggers/perf_tests.tf | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 share/ramble/cloud-build/terraform/triggers/perf_tests.tf diff --git a/share/ramble/cloud-build/terraform/triggers/perf_tests.tf b/share/ramble/cloud-build/terraform/triggers/perf_tests.tf new file mode 100644 index 000000000..71e4028f5 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/perf_tests.tf @@ -0,0 +1,56 @@ +resource "google_cloudbuild_trigger" "perf_test_pr" { + name = "PerfTest-PR-rockylinux8-v1-0-0spack-3-13-5python" + description = "Ramble perf tests for PR builds" + + github { + owner = var.github_owner + name = var.github_repo + pull_request { + branch = "(?:main|develop)" + comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY" + } + } + + include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS" + + filename = "share/ramble/cloud-build/ramble-perf-tests.yaml" + + substitutions = { + _SPACK_REF = "v1.0.0" + _PYTHON_VER = "3.13.5" + _BASE_IMG = "rockylinux" + _BASE_VER = "8" + _DATASET_ID = "ramble_metrics" + _PROJECT_ID = var.project_id + _TABLE_ID = "perf_test_durations" + _UPLOAD_TO_BQ = "false" + } +} + +resource "google_cloudbuild_trigger" "perf_test_push" { + name = "PerfTest-Push-rockylinux8-v1-0-0spack-3-13-5python" + description = "Continuous monitoring of Ramble performance for develop push" + + github { + owner = var.github_owner + name = var.github_repo + push { + branch = "^develop$" + } + } + + include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS" + + filename = "share/ramble/cloud-build/ramble-perf-tests.yaml" + + substitutions = { + _SPACK_REF = "v1.0.0" + _PYTHON_VER = "3.13.5" + _BASE_IMG = "rockylinux" + _BASE_VER = "8" + _DATASET_ID = "ramble_metrics" + _PROJECT_ID = var.project_id + _TABLE_ID = "perf_test_durations" + _UPLOAD_TO_BQ = "true" + } +} From 34623979d5899070ef9efdba5619666f9f4a61b9 Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Fri, 27 Mar 2026 10:40:19 -0700 Subject: [PATCH 04/11] Add in the image-build triggers --- .../terraform/triggers/image_builds.tf | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 share/ramble/cloud-build/terraform/triggers/image_builds.tf diff --git a/share/ramble/cloud-build/terraform/triggers/image_builds.tf b/share/ramble/cloud-build/terraform/triggers/image_builds.tf new file mode 100644 index 000000000..643af88dd --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/image_builds.tf @@ -0,0 +1,29 @@ +resource "google_cloudbuild_trigger" "image_builders" { + for_each = { for img in local.image_matrix : "${img.base}${img.base_ver}-py${img.python}-spack${img.spack}" => img } + + name = "ramble-image-builder-${each.value.base}${each.value.base_ver}-py${replace(each.value.python, ".", "-")}-spack${replace(each.value.spack, ".", "-")}" + description = "Build Ramble cloud build image for ${each.value.base} ${each.value.base_ver} with Python ${each.value.python} and Spack ${each.value.spack}" + + github { + owner = var.github_owner + name = var.github_repo + push { + branch = "^develop$" + } + } + + included_files = [ + "share/ramble/cloud-build/ramble-image-builder.yaml", + "share/ramble/cloud-build/Dockerfile-${local.pm_map[each.value.base]}" + ] + + filename = "share/ramble/cloud-build/ramble-image-builder.yaml" + + substitutions = { + _PYTHON_VER = each.value.python + _SPACK_REF = each.value.spack + _PKG_MANAGER = local.pm_map[each.value.base] + _BASE_IMG = each.value.base + _BASE_VER = each.value.base_ver + } +} From 005fa1cb7448cf8b6a7b968fae4b6938fdfccbf8 Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Fri, 27 Mar 2026 10:43:09 -0700 Subject: [PATCH 05/11] Add in the doc build trigger --- .../terraform/triggers/pr_doc_build.tf | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf diff --git a/share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf b/share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf new file mode 100644 index 000000000..abef999a9 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf @@ -0,0 +1,24 @@ +resource "google_cloudbuild_trigger" "pr_doc_build_tests" { + name = "PR-Doc-Build-Tests" + description = "A presubmit check for building Ramble documentation" + + github { + owner = var.github_owner + name = var.github_repo + pull_request { + branch = "(?:main|develop)" + comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY" + } + } + + include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS" + + filename = "share/ramble/cloud-build/ramble-pr-docs.yaml" + + substitutions = { + _BASE_IMG = "rockylinux" + _BASE_VER = "8" + _PYTHON_VER = "3.13.5" + _SPACK_REF = "v1.0.0" + } +} From a2dddd5a5a76048600b7cb89bc3147556561d4aa Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Fri, 27 Mar 2026 10:57:38 -0700 Subject: [PATCH 06/11] Add in the pr style and software conflict check triggers --- .../triggers/pr_software_conflicts.tf | 30 +++++++++++++++++++ .../terraform/triggers/pr_style.tf | 24 +++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf create mode 100644 share/ramble/cloud-build/terraform/triggers/pr_style.tf diff --git a/share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf b/share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf new file mode 100644 index 000000000..aef98dc82 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf @@ -0,0 +1,30 @@ +resource "google_cloudbuild_trigger" "pr_software_conflicts" { + name = "PR-Software-Conflicts-rockylinux8-v0-22-1spack-3-12-1python" + description = "Check for conflicts in application definitions on Ramble pull requests" + + github { + owner = var.github_owner + name = var.github_repo + pull_request { + branch = "(?:main|develop)" + comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY" + } + } + + include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS" + + filename = "share/ramble/cloud-build/ramble-pr-software-conflicts.yaml" + + included_files = [ + "var/ramble/repos/**", + "lib/ramble/ramble/**", + "share/ramble/cloud-build/**" + ] + + substitutions = { + _BASE_IMG = "rockylinux" + _BASE_VER = "8" + _PYTHON_VER = "3.12.1" + _SPACK_REF = "v0.22.1" + } +} diff --git a/share/ramble/cloud-build/terraform/triggers/pr_style.tf b/share/ramble/cloud-build/terraform/triggers/pr_style.tf new file mode 100644 index 000000000..6dea184a1 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/pr_style.tf @@ -0,0 +1,24 @@ +resource "google_cloudbuild_trigger" "pr_style" { + name = "PR-Style-rockylinux8-v0-22-1spack-3-12-1python" + description = "Run linting on Ramble pull requests" + + github { + owner = var.github_owner + name = var.github_repo + pull_request { + branch = "(?:main|develop)" + comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY" + } + } + + include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS" + + filename = "share/ramble/cloud-build/ramble-pr-style.yaml" + + substitutions = { + _BASE_IMG = "rockylinux" + _BASE_VER = "8" + _PYTHON_VER = "3.12.1" + _SPACK_REF = "v0.22.1" + } +} From a709f5bd26c52072ab32a048d39622549f58b43e Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Fri, 27 Mar 2026 10:58:02 -0700 Subject: [PATCH 07/11] Add in the pr image-build test trigger --- .../terraform/triggers/pr_image_build.tf | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 share/ramble/cloud-build/terraform/triggers/pr_image_build.tf diff --git a/share/ramble/cloud-build/terraform/triggers/pr_image_build.tf b/share/ramble/cloud-build/terraform/triggers/pr_image_build.tf new file mode 100644 index 000000000..b1c30d979 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/pr_image_build.tf @@ -0,0 +1,21 @@ +resource "google_cloudbuild_trigger" "pr_image_build_tests" { + name = "PR-Image-Build-Tests" + description = "A presubmit check for building images used by other cloud build triggers" + + github { + owner = var.github_owner + name = var.github_repo + pull_request { + branch = "(?:main|develop)" + comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY" + } + } + + include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS" + + filename = "share/ramble/cloud-build/ramble-pr-image-builds.yaml" + + included_files = [ + "share/ramble/cloud-build/**" + ] +} From fd5c7df2d05fbd30fdc22ffd79d9bd4394616c93 Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Fri, 27 Mar 2026 11:36:56 -0700 Subject: [PATCH 08/11] Reference image_matrix instead of hard-coding --- .../terraform/triggers/perf_tests.tf | 24 +++++++++++-------- .../terraform/triggers/pr_doc_build.tf | 12 ++++++---- .../triggers/pr_software_conflicts.tf | 14 +++++++---- .../terraform/triggers/pr_style.tf | 14 +++++++---- 4 files changed, 40 insertions(+), 24 deletions(-) diff --git a/share/ramble/cloud-build/terraform/triggers/perf_tests.tf b/share/ramble/cloud-build/terraform/triggers/perf_tests.tf index 71e4028f5..a79d010f2 100644 --- a/share/ramble/cloud-build/terraform/triggers/perf_tests.tf +++ b/share/ramble/cloud-build/terraform/triggers/perf_tests.tf @@ -1,5 +1,9 @@ +locals { + perf_test_env = local.image_matrix[10] +} + resource "google_cloudbuild_trigger" "perf_test_pr" { - name = "PerfTest-PR-rockylinux8-v1-0-0spack-3-13-5python" + name = "PerfTest-PR-${local.perf_test_env.base}${local.perf_test_env.base_ver}-${replace(local.perf_test_env.spack, ".", "-")}spack-${replace(local.perf_test_env.python, ".", "-")}python" description = "Ramble perf tests for PR builds" github { @@ -16,10 +20,10 @@ resource "google_cloudbuild_trigger" "perf_test_pr" { filename = "share/ramble/cloud-build/ramble-perf-tests.yaml" substitutions = { - _SPACK_REF = "v1.0.0" - _PYTHON_VER = "3.13.5" - _BASE_IMG = "rockylinux" - _BASE_VER = "8" + _SPACK_REF = local.perf_test_env.spack + _PYTHON_VER = local.perf_test_env.python + _BASE_IMG = local.perf_test_env.base + _BASE_VER = local.perf_test_env.base_ver _DATASET_ID = "ramble_metrics" _PROJECT_ID = var.project_id _TABLE_ID = "perf_test_durations" @@ -28,7 +32,7 @@ resource "google_cloudbuild_trigger" "perf_test_pr" { } resource "google_cloudbuild_trigger" "perf_test_push" { - name = "PerfTest-Push-rockylinux8-v1-0-0spack-3-13-5python" + name = "PerfTest-Push-${local.perf_test_env.base}${local.perf_test_env.base_ver}-${replace(local.perf_test_env.spack, ".", "-")}spack-${replace(local.perf_test_env.python, ".", "-")}python" description = "Continuous monitoring of Ramble performance for develop push" github { @@ -44,10 +48,10 @@ resource "google_cloudbuild_trigger" "perf_test_push" { filename = "share/ramble/cloud-build/ramble-perf-tests.yaml" substitutions = { - _SPACK_REF = "v1.0.0" - _PYTHON_VER = "3.13.5" - _BASE_IMG = "rockylinux" - _BASE_VER = "8" + _SPACK_REF = local.perf_test_env.spack + _PYTHON_VER = local.perf_test_env.python + _BASE_IMG = local.perf_test_env.base + _BASE_VER = local.perf_test_env.base_ver _DATASET_ID = "ramble_metrics" _PROJECT_ID = var.project_id _TABLE_ID = "perf_test_durations" diff --git a/share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf b/share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf index abef999a9..d035f7522 100644 --- a/share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf +++ b/share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf @@ -1,3 +1,7 @@ +locals { + pr_doc_env = local.image_matrix[10] +} + resource "google_cloudbuild_trigger" "pr_doc_build_tests" { name = "PR-Doc-Build-Tests" description = "A presubmit check for building Ramble documentation" @@ -16,9 +20,9 @@ resource "google_cloudbuild_trigger" "pr_doc_build_tests" { filename = "share/ramble/cloud-build/ramble-pr-docs.yaml" substitutions = { - _BASE_IMG = "rockylinux" - _BASE_VER = "8" - _PYTHON_VER = "3.13.5" - _SPACK_REF = "v1.0.0" + _BASE_IMG = local.pr_doc_env.base + _BASE_VER = local.pr_doc_env.base_ver + _PYTHON_VER = local.pr_doc_env.python + _SPACK_REF = local.pr_doc_env.spack } } diff --git a/share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf b/share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf index aef98dc82..de6f32455 100644 --- a/share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf +++ b/share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf @@ -1,5 +1,9 @@ +locals { + pr_software_conflicts_env = local.image_matrix[8] +} + resource "google_cloudbuild_trigger" "pr_software_conflicts" { - name = "PR-Software-Conflicts-rockylinux8-v0-22-1spack-3-12-1python" + name = "PR-Software-Conflicts-${local.pr_software_conflicts_env.base}${local.pr_software_conflicts_env.base_ver}-${replace(local.pr_software_conflicts_env.spack, ".", "-")}spack-${replace(local.pr_software_conflicts_env.python, ".", "-")}python" description = "Check for conflicts in application definitions on Ramble pull requests" github { @@ -22,9 +26,9 @@ resource "google_cloudbuild_trigger" "pr_software_conflicts" { ] substitutions = { - _BASE_IMG = "rockylinux" - _BASE_VER = "8" - _PYTHON_VER = "3.12.1" - _SPACK_REF = "v0.22.1" + _BASE_IMG = local.pr_software_conflicts_env.base + _BASE_VER = local.pr_software_conflicts_env.base_ver + _PYTHON_VER = local.pr_software_conflicts_env.python + _SPACK_REF = local.pr_software_conflicts_env.spack } } diff --git a/share/ramble/cloud-build/terraform/triggers/pr_style.tf b/share/ramble/cloud-build/terraform/triggers/pr_style.tf index 6dea184a1..d2838deca 100644 --- a/share/ramble/cloud-build/terraform/triggers/pr_style.tf +++ b/share/ramble/cloud-build/terraform/triggers/pr_style.tf @@ -1,5 +1,9 @@ +locals { + pr_style_env = local.image_matrix[8] +} + resource "google_cloudbuild_trigger" "pr_style" { - name = "PR-Style-rockylinux8-v0-22-1spack-3-12-1python" + name = "PR-Style-${local.pr_style_env.base}${local.pr_style_env.base_ver}-${replace(local.pr_style_env.spack, ".", "-")}spack-${replace(local.pr_style_env.python, ".", "-")}python" description = "Run linting on Ramble pull requests" github { @@ -16,9 +20,9 @@ resource "google_cloudbuild_trigger" "pr_style" { filename = "share/ramble/cloud-build/ramble-pr-style.yaml" substitutions = { - _BASE_IMG = "rockylinux" - _BASE_VER = "8" - _PYTHON_VER = "3.12.1" - _SPACK_REF = "v0.22.1" + _BASE_IMG = local.pr_style_env.base + _BASE_VER = local.pr_style_env.base_ver + _PYTHON_VER = local.pr_style_env.python + _SPACK_REF = local.pr_style_env.spack } } From 337facf3903635c2e6c099d9c15922792e56e700 Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Fri, 27 Mar 2026 20:59:29 -0700 Subject: [PATCH 09/11] Restructure to define images as a map --- .../terraform/triggers/image_builds.tf | 2 +- .../cloud-build/terraform/triggers/locals.tf | 31 ++++++++++--------- .../terraform/triggers/perf_tests.tf | 22 ++++++------- .../terraform/triggers/pr_doc_build.tf | 10 +++--- .../triggers/pr_software_conflicts.tf | 12 +++---- .../terraform/triggers/pr_style.tf | 12 +++---- .../terraform/triggers/pr_unit_tests.tf | 2 +- 7 files changed, 46 insertions(+), 45 deletions(-) diff --git a/share/ramble/cloud-build/terraform/triggers/image_builds.tf b/share/ramble/cloud-build/terraform/triggers/image_builds.tf index 643af88dd..d0c283a04 100644 --- a/share/ramble/cloud-build/terraform/triggers/image_builds.tf +++ b/share/ramble/cloud-build/terraform/triggers/image_builds.tf @@ -1,5 +1,5 @@ resource "google_cloudbuild_trigger" "image_builders" { - for_each = { for img in local.image_matrix : "${img.base}${img.base_ver}-py${img.python}-spack${img.spack}" => img } + for_each = local.image_map name = "ramble-image-builder-${each.value.base}${each.value.base_ver}-py${replace(each.value.python, ".", "-")}-spack${replace(each.value.spack, ".", "-")}" description = "Build Ramble cloud build image for ${each.value.base} ${each.value.base_ver} with Python ${each.value.python} and Spack ${each.value.spack}" diff --git a/share/ramble/cloud-build/terraform/triggers/locals.tf b/share/ramble/cloud-build/terraform/triggers/locals.tf index 19f39607f..9ce73960c 100644 --- a/share/ramble/cloud-build/terraform/triggers/locals.tf +++ b/share/ramble/cloud-build/terraform/triggers/locals.tf @@ -1,19 +1,20 @@ locals { - # This matrix is used by both the image-build and unit-test triggers - image_matrix = [ - { python = "3.12.0", spack = "v0.21.2", base = "debian", base_ver = "12.5" }, - { python = "3.8.0", spack = "v0.21.2", base = "debian", base_ver = "12.5" }, - { python = "3.12.1", spack = "v0.22.1", base = "debian", base_ver = "12.5" }, - { python = "3.8.0", spack = "v0.22.1", base = "debian", base_ver = "12.5" }, - { python = "3.13.5", spack = "v1.0.0", base = "debian", base_ver = "12.5" }, - { python = "3.7.17", spack = "v1.0.0", base = "debian", base_ver = "12.5" }, - { python = "3.12.0", spack = "v0.21.2", base = "rockylinux", base_ver = "8" }, - { python = "3.7.0", spack = "v0.21.2", base = "rockylinux", base_ver = "8" }, - { python = "3.12.1", spack = "v0.22.1", base = "rockylinux", base_ver = "8" }, - { python = "3.7.0", spack = "v0.22.1", base = "rockylinux", base_ver = "8" }, - { python = "3.13.5", spack = "v1.0.0", base = "rockylinux", base_ver = "8" }, - { python = "3.7.17", spack = "v1.0.0", base = "rockylinux", base_ver = "8" }, - ] + # This map holds the full list of available images. + # A trigger should always reference these images instead of hard-coding new ones. + image_map = { + "debian12-5-py3-12-0-spack-v0-21-2" = { python = "3.12.0", spack = "v0.21.2", base = "debian", base_ver = "12.5" }, + "debian12-5-py3-8-0-spack-v0-21-2" = { python = "3.8.0", spack = "v0.21.2", base = "debian", base_ver = "12.5" }, + "debian12-5-py3-12-1-spack-v0-22-1" = { python = "3.12.1", spack = "v0.22.1", base = "debian", base_ver = "12.5" }, + "debian12-5-py3-8-0-spack-v0-22-1" = { python = "3.8.0", spack = "v0.22.1", base = "debian", base_ver = "12.5" }, + "debian12-5-py3-13-5-spack-v1-0-0" = { python = "3.13.5", spack = "v1.0.0", base = "debian", base_ver = "12.5" }, + "debian12-5-py3-7-17-spack-v1-0-0" = { python = "3.7.17", spack = "v1.0.0", base = "debian", base_ver = "12.5" }, + "rocky8-py3-12-0-spack-v0-21-2" = { python = "3.12.0", spack = "v0.21.2", base = "rockylinux", base_ver = "8" }, + "rocky8-py3-7-0-spack-v0-21-2" = { python = "3.7.0", spack = "v0.21.2", base = "rockylinux", base_ver = "8" }, + "rocky8-py3-12-1-spack-v0-22-1" = { python = "3.12.1", spack = "v0.22.1", base = "rockylinux", base_ver = "8" }, + "rocky8-py3-7-0-spack-v0-22-1" = { python = "3.7.0", spack = "v0.22.1", base = "rockylinux", base_ver = "8" }, + "rocky8-py3-13-5-spack-v1-0-0" = { python = "3.13.5", spack = "v1.0.0", base = "rockylinux", base_ver = "8" }, + "rocky8-py3-7-17-spack-v1-0-0" = { python = "3.7.17", spack = "v1.0.0", base = "rockylinux", base_ver = "8" }, + } pm_map = { "debian" = "apt" diff --git a/share/ramble/cloud-build/terraform/triggers/perf_tests.tf b/share/ramble/cloud-build/terraform/triggers/perf_tests.tf index a79d010f2..39ddc0585 100644 --- a/share/ramble/cloud-build/terraform/triggers/perf_tests.tf +++ b/share/ramble/cloud-build/terraform/triggers/perf_tests.tf @@ -1,9 +1,9 @@ locals { - perf_test_env = local.image_matrix[10] + perf_test_img = local.image_map["rocky8-py3-13-5-spack-v1-0-0"] } resource "google_cloudbuild_trigger" "perf_test_pr" { - name = "PerfTest-PR-${local.perf_test_env.base}${local.perf_test_env.base_ver}-${replace(local.perf_test_env.spack, ".", "-")}spack-${replace(local.perf_test_env.python, ".", "-")}python" + name = "PerfTest-PR-${local.perf_test_img.base}${local.perf_test_img.base_ver}-${replace(local.perf_test_img.spack, ".", "-")}spack-${replace(local.perf_test_img.python, ".", "-")}python" description = "Ramble perf tests for PR builds" github { @@ -20,10 +20,10 @@ resource "google_cloudbuild_trigger" "perf_test_pr" { filename = "share/ramble/cloud-build/ramble-perf-tests.yaml" substitutions = { - _SPACK_REF = local.perf_test_env.spack - _PYTHON_VER = local.perf_test_env.python - _BASE_IMG = local.perf_test_env.base - _BASE_VER = local.perf_test_env.base_ver + _SPACK_REF = local.perf_test_img.spack + _PYTHON_VER = local.perf_test_img.python + _BASE_IMG = local.perf_test_img.base + _BASE_VER = local.perf_test_img.base_ver _DATASET_ID = "ramble_metrics" _PROJECT_ID = var.project_id _TABLE_ID = "perf_test_durations" @@ -32,7 +32,7 @@ resource "google_cloudbuild_trigger" "perf_test_pr" { } resource "google_cloudbuild_trigger" "perf_test_push" { - name = "PerfTest-Push-${local.perf_test_env.base}${local.perf_test_env.base_ver}-${replace(local.perf_test_env.spack, ".", "-")}spack-${replace(local.perf_test_env.python, ".", "-")}python" + name = "PerfTest-Push-${local.perf_test_img.base}${local.perf_test_img.base_ver}-${replace(local.perf_test_img.spack, ".", "-")}spack-${replace(local.perf_test_img.python, ".", "-")}python" description = "Continuous monitoring of Ramble performance for develop push" github { @@ -48,10 +48,10 @@ resource "google_cloudbuild_trigger" "perf_test_push" { filename = "share/ramble/cloud-build/ramble-perf-tests.yaml" substitutions = { - _SPACK_REF = local.perf_test_env.spack - _PYTHON_VER = local.perf_test_env.python - _BASE_IMG = local.perf_test_env.base - _BASE_VER = local.perf_test_env.base_ver + _SPACK_REF = local.perf_test_img.spack + _PYTHON_VER = local.perf_test_img.python + _BASE_IMG = local.perf_test_img.base + _BASE_VER = local.perf_test_img.base_ver _DATASET_ID = "ramble_metrics" _PROJECT_ID = var.project_id _TABLE_ID = "perf_test_durations" diff --git a/share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf b/share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf index d035f7522..6d499de97 100644 --- a/share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf +++ b/share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf @@ -1,5 +1,5 @@ locals { - pr_doc_env = local.image_matrix[10] + pr_doc_img = local.image_map["rocky8-py3-13-5-spack-v1-0-0"] } resource "google_cloudbuild_trigger" "pr_doc_build_tests" { @@ -20,9 +20,9 @@ resource "google_cloudbuild_trigger" "pr_doc_build_tests" { filename = "share/ramble/cloud-build/ramble-pr-docs.yaml" substitutions = { - _BASE_IMG = local.pr_doc_env.base - _BASE_VER = local.pr_doc_env.base_ver - _PYTHON_VER = local.pr_doc_env.python - _SPACK_REF = local.pr_doc_env.spack + _BASE_IMG = local.pr_doc_img.base + _BASE_VER = local.pr_doc_img.base_ver + _PYTHON_VER = local.pr_doc_img.python + _SPACK_REF = local.pr_doc_img.spack } } diff --git a/share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf b/share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf index de6f32455..fdc9e3729 100644 --- a/share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf +++ b/share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf @@ -1,9 +1,9 @@ locals { - pr_software_conflicts_env = local.image_matrix[8] + pr_software_conflicts_img = local.image_map["rocky8-py3-12-1-spack-v0-22-1"] } resource "google_cloudbuild_trigger" "pr_software_conflicts" { - name = "PR-Software-Conflicts-${local.pr_software_conflicts_env.base}${local.pr_software_conflicts_env.base_ver}-${replace(local.pr_software_conflicts_env.spack, ".", "-")}spack-${replace(local.pr_software_conflicts_env.python, ".", "-")}python" + name = "PR-Software-Conflicts-${local.pr_software_conflicts_img.base}${local.pr_software_conflicts_img.base_ver}-${replace(local.pr_software_conflicts_img.spack, ".", "-")}spack-${replace(local.pr_software_conflicts_img.python, ".", "-")}python" description = "Check for conflicts in application definitions on Ramble pull requests" github { @@ -26,9 +26,9 @@ resource "google_cloudbuild_trigger" "pr_software_conflicts" { ] substitutions = { - _BASE_IMG = local.pr_software_conflicts_env.base - _BASE_VER = local.pr_software_conflicts_env.base_ver - _PYTHON_VER = local.pr_software_conflicts_env.python - _SPACK_REF = local.pr_software_conflicts_env.spack + _BASE_IMG = local.pr_software_conflicts_img.base + _BASE_VER = local.pr_software_conflicts_img.base_ver + _PYTHON_VER = local.pr_software_conflicts_img.python + _SPACK_REF = local.pr_software_conflicts_img.spack } } diff --git a/share/ramble/cloud-build/terraform/triggers/pr_style.tf b/share/ramble/cloud-build/terraform/triggers/pr_style.tf index d2838deca..39ebd5e2a 100644 --- a/share/ramble/cloud-build/terraform/triggers/pr_style.tf +++ b/share/ramble/cloud-build/terraform/triggers/pr_style.tf @@ -1,9 +1,9 @@ locals { - pr_style_env = local.image_matrix[8] + pr_style_img = local.image_map["rocky8-py3-12-1-spack-v0-22-1"] } resource "google_cloudbuild_trigger" "pr_style" { - name = "PR-Style-${local.pr_style_env.base}${local.pr_style_env.base_ver}-${replace(local.pr_style_env.spack, ".", "-")}spack-${replace(local.pr_style_env.python, ".", "-")}python" + name = "PR-Style-${local.pr_style_img.base}${local.pr_style_img.base_ver}-${replace(local.pr_style_img.spack, ".", "-")}spack-${replace(local.pr_style_img.python, ".", "-")}python" description = "Run linting on Ramble pull requests" github { @@ -20,9 +20,9 @@ resource "google_cloudbuild_trigger" "pr_style" { filename = "share/ramble/cloud-build/ramble-pr-style.yaml" substitutions = { - _BASE_IMG = local.pr_style_env.base - _BASE_VER = local.pr_style_env.base_ver - _PYTHON_VER = local.pr_style_env.python - _SPACK_REF = local.pr_style_env.spack + _BASE_IMG = local.pr_style_img.base + _BASE_VER = local.pr_style_img.base_ver + _PYTHON_VER = local.pr_style_img.python + _SPACK_REF = local.pr_style_img.spack } } diff --git a/share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf b/share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf index 5ed5cb35c..3e477b574 100644 --- a/share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf +++ b/share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf @@ -1,5 +1,5 @@ resource "google_cloudbuild_trigger" "pr_unit_tests" { - for_each = { for img in local.image_matrix : "${img.base}${img.base_ver}-py${img.python}-spack${img.spack}" => img } + for_each = local.image_map name = "PR-Unit-Tests-${each.value.base}${replace(each.value.base_ver, ".", "-")}-${replace(each.value.spack, ".", "-")}spack-${replace(each.value.python, ".", "-")}python" description = "Run unit tests and linting on Ramble pull requests" From 69ace4ad53bbe5d7cdfacd350e478cd25e861b31 Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Sat, 28 Mar 2026 10:29:22 -0700 Subject: [PATCH 10/11] Set `INCLUDE_BUILD_LOGS_WITH_STATUS` for all PR-based triggers But don't set it for push-based triggers. --- share/ramble/cloud-build/terraform/triggers/perf_tests.tf | 2 -- share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/share/ramble/cloud-build/terraform/triggers/perf_tests.tf b/share/ramble/cloud-build/terraform/triggers/perf_tests.tf index 39ddc0585..f2d94fb05 100644 --- a/share/ramble/cloud-build/terraform/triggers/perf_tests.tf +++ b/share/ramble/cloud-build/terraform/triggers/perf_tests.tf @@ -43,8 +43,6 @@ resource "google_cloudbuild_trigger" "perf_test_push" { } } - include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS" - filename = "share/ramble/cloud-build/ramble-perf-tests.yaml" substitutions = { diff --git a/share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf b/share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf index 3e477b574..7f5f3134d 100644 --- a/share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf +++ b/share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf @@ -24,6 +24,8 @@ resource "google_cloudbuild_trigger" "pr_unit_tests" { "conftest.py" ] + include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS" + filename = "share/ramble/cloud-build/ramble-pr-unit-tests.yaml" substitutions = { From 87b91dd1840234a9501481f3786036675d3f966d Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Tue, 31 Mar 2026 13:20:45 -0700 Subject: [PATCH 11/11] Fix up invalid (dot) trigger names --- share/ramble/cloud-build/terraform/triggers/image_builds.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/ramble/cloud-build/terraform/triggers/image_builds.tf b/share/ramble/cloud-build/terraform/triggers/image_builds.tf index d0c283a04..1ba627160 100644 --- a/share/ramble/cloud-build/terraform/triggers/image_builds.tf +++ b/share/ramble/cloud-build/terraform/triggers/image_builds.tf @@ -1,7 +1,7 @@ resource "google_cloudbuild_trigger" "image_builders" { for_each = local.image_map - name = "ramble-image-builder-${each.value.base}${each.value.base_ver}-py${replace(each.value.python, ".", "-")}-spack${replace(each.value.spack, ".", "-")}" + name = "ramble-image-builder-${each.value.base}${replace(each.value.base_ver, ".", "-")}-py${replace(each.value.python, ".", "-")}-spack${replace(each.value.spack, ".", "-")}" description = "Build Ramble cloud build image for ${each.value.base} ${each.value.base_ver} with Python ${each.value.python} and Spack ${each.value.spack}" github {