Use Terraform to manage cloud-build triggers

share/ramble/cloud-build/terraform/triggers/.gitignore

@@ -0,0 +1,3 @@
+# We don't ignore any tfstate files, as those should reside on the GCS bucket instead
+.terraform/
+.terraform.lock.hcl

share/ramble/cloud-build/terraform/triggers/README.md

@@ -0,0 +1,20 @@
+# Ramble Cloud Build Image Triggers
+
+This directory contains Terraform configuration to deploy and manage Google Cloud Build Triggers used by the Ramble repository.
+
+## How to deploy
+
+The deployment states are stored in a GCS bucket, to allow for running Terraform from different locations. The bucket was created with:
+
+```bash
+gcloud storage buckets create gs://ramble-terraform-state --project=ramble-eng --location=us-central1
+```
+
+With the bucket created, the triggers can be deployed with:
+
+```bash
+terraform init
+# Optional to check the changes to be made
+terraform plan
+terraform apply --auto-approve
+```

share/ramble/cloud-build/terraform/triggers/backend.tf

@@ -0,0 +1,6 @@
+terraform {
+  backend "gcs" {
+    bucket = "ramble-terraform-state"
+    prefix = "terraform/state/image-triggers"
+  }
+}

share/ramble/cloud-build/terraform/triggers/image_builds.tf

@@ -0,0 +1,29 @@
+resource "google_cloudbuild_trigger" "image_builders" {
+  for_each = local.image_map
+
+  name        = "ramble-image-builder-${each.value.base}${replace(each.value.base_ver, ".", "-")}-py${replace(each.value.python, ".", "-")}-spack${replace(each.value.spack, ".", "-")}"
+  description = "Build Ramble cloud build image for ${each.value.base} ${each.value.base_ver} with Python ${each.value.python} and Spack ${each.value.spack}"
+
+  github {
+    owner = var.github_owner
+    name  = var.github_repo
+    push {
+      branch = "^develop$"
+    }
+  }
+
+  included_files = [
+    "share/ramble/cloud-build/ramble-image-builder.yaml",
+    "share/ramble/cloud-build/Dockerfile-${local.pm_map[each.value.base]}"
+  ]
+
+  filename = "share/ramble/cloud-build/ramble-image-builder.yaml"
+
+  substitutions = {
+    _PYTHON_VER  = each.value.python
+    _SPACK_REF   = each.value.spack
+    _PKG_MANAGER = local.pm_map[each.value.base]
+    _BASE_IMG    = each.value.base
+    _BASE_VER    = each.value.base_ver
+  }
+}

share/ramble/cloud-build/terraform/triggers/locals.tf

@@ -0,0 +1,23 @@
+locals {
+  # This map holds the full list of available images.
+  # A trigger should always reference these images instead of hard-coding new ones.
+  image_map = {
+    "debian12-5-py3-12-0-spack-v0-21-2" = { python = "3.12.0", spack = "v0.21.2", base = "debian", base_ver = "12.5" },
+    "debian12-5-py3-8-0-spack-v0-21-2"  = { python = "3.8.0", spack = "v0.21.2", base = "debian", base_ver = "12.5" },
+    "debian12-5-py3-12-1-spack-v0-22-1" = { python = "3.12.1", spack = "v0.22.1", base = "debian", base_ver = "12.5" },
+    "debian12-5-py3-8-0-spack-v0-22-1"  = { python = "3.8.0", spack = "v0.22.1", base = "debian", base_ver = "12.5" },
+    "debian12-5-py3-13-5-spack-v1-0-0"  = { python = "3.13.5", spack = "v1.0.0", base = "debian", base_ver = "12.5" },
+    "debian12-5-py3-7-17-spack-v1-0-0"  = { python = "3.7.17", spack = "v1.0.0", base = "debian", base_ver = "12.5" },
+    "rocky8-py3-12-0-spack-v0-21-2"     = { python = "3.12.0", spack = "v0.21.2", base = "rockylinux", base_ver = "8" },
+    "rocky8-py3-7-0-spack-v0-21-2"      = { python = "3.7.0", spack = "v0.21.2", base = "rockylinux", base_ver = "8" },
+    "rocky8-py3-12-1-spack-v0-22-1"     = { python = "3.12.1", spack = "v0.22.1", base = "rockylinux", base_ver = "8" },
+    "rocky8-py3-7-0-spack-v0-22-1"      = { python = "3.7.0", spack = "v0.22.1", base = "rockylinux", base_ver = "8" },
+    "rocky8-py3-13-5-spack-v1-0-0"      = { python = "3.13.5", spack = "v1.0.0", base = "rockylinux", base_ver = "8" },
+    "rocky8-py3-7-17-spack-v1-0-0"      = { python = "3.7.17", spack = "v1.0.0", base = "rockylinux", base_ver = "8" },
+  }
+
+  pm_map = {
+    "debian"     = "apt"
+    "rockylinux" = "yum"
+  }
+}

share/ramble/cloud-build/terraform/triggers/perf_tests.tf

@@ -0,0 +1,58 @@
+locals {
+  perf_test_img = local.image_map["rocky8-py3-13-5-spack-v1-0-0"]
+}
+
+resource "google_cloudbuild_trigger" "perf_test_pr" {
+  name        = "PerfTest-PR-${local.perf_test_img.base}${local.perf_test_img.base_ver}-${replace(local.perf_test_img.spack, ".", "-")}spack-${replace(local.perf_test_img.python, ".", "-")}python"
+  description = "Ramble perf tests for PR builds"
+
+  github {
+    owner = var.github_owner
+    name  = var.github_repo
+    pull_request {
+      branch          = "(?:main|develop)"
+      comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY"
+    }
+  }
+
+  include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS"
+
+  filename = "share/ramble/cloud-build/ramble-perf-tests.yaml"
+
+  substitutions = {
+    _SPACK_REF    = local.perf_test_img.spack
+    _PYTHON_VER   = local.perf_test_img.python
+    _BASE_IMG     = local.perf_test_img.base
+    _BASE_VER     = local.perf_test_img.base_ver
+    _DATASET_ID   = "ramble_metrics"
+    _PROJECT_ID   = var.project_id
+    _TABLE_ID     = "perf_test_durations"
+    _UPLOAD_TO_BQ = "false"
+  }
+}
+
+resource "google_cloudbuild_trigger" "perf_test_push" {
+  name        = "PerfTest-Push-${local.perf_test_img.base}${local.perf_test_img.base_ver}-${replace(local.perf_test_img.spack, ".", "-")}spack-${replace(local.perf_test_img.python, ".", "-")}python"
+  description = "Continuous monitoring of Ramble performance for develop push"
+
+  github {
+    owner = var.github_owner
+    name  = var.github_repo
+    push {
+      branch = "^develop$"
+    }
+  }
+
+  filename = "share/ramble/cloud-build/ramble-perf-tests.yaml"
+
+  substitutions = {
+    _SPACK_REF    = local.perf_test_img.spack
+    _PYTHON_VER   = local.perf_test_img.python
+    _BASE_IMG     = local.perf_test_img.base
+    _BASE_VER     = local.perf_test_img.base_ver
+    _DATASET_ID   = "ramble_metrics"
+    _PROJECT_ID   = var.project_id
+    _TABLE_ID     = "perf_test_durations"
+    _UPLOAD_TO_BQ = "true"
+  }
+}

share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf

@@ -0,0 +1,28 @@
+locals {
+  pr_doc_img = local.image_map["rocky8-py3-13-5-spack-v1-0-0"]
+}
+
+resource "google_cloudbuild_trigger" "pr_doc_build_tests" {
+  name        = "PR-Doc-Build-Tests"
+  description = "A presubmit check for building Ramble documentation"
+
+  github {
+    owner = var.github_owner
+    name  = var.github_repo
+    pull_request {
+      branch          = "(?:main|develop)"
+      comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY"
+    }
+  }
+
+  include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS"
+
+  filename = "share/ramble/cloud-build/ramble-pr-docs.yaml"
+
+  substitutions = {
+    _BASE_IMG   = local.pr_doc_img.base
+    _BASE_VER   = local.pr_doc_img.base_ver
+    _PYTHON_VER = local.pr_doc_img.python
+    _SPACK_REF  = local.pr_doc_img.spack
+  }
+}

share/ramble/cloud-build/terraform/triggers/pr_image_build.tf

@@ -0,0 +1,21 @@
+resource "google_cloudbuild_trigger" "pr_image_build_tests" {
+  name        = "PR-Image-Build-Tests"
+  description = "A presubmit check for building images used by other cloud build triggers"
+
+  github {
+    owner = var.github_owner
+    name  = var.github_repo
+    pull_request {
+      branch          = "(?:main|develop)"
+      comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY"
+    }
+  }
+
+  include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS"
+
+  filename = "share/ramble/cloud-build/ramble-pr-image-builds.yaml"
+
+  included_files = [
+    "share/ramble/cloud-build/**"
+  ]
+}

share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf

@@ -0,0 +1,34 @@
+locals {
+  pr_software_conflicts_img = local.image_map["rocky8-py3-12-1-spack-v0-22-1"]
+}
+
+resource "google_cloudbuild_trigger" "pr_software_conflicts" {
+  name        = "PR-Software-Conflicts-${local.pr_software_conflicts_img.base}${local.pr_software_conflicts_img.base_ver}-${replace(local.pr_software_conflicts_img.spack, ".", "-")}spack-${replace(local.pr_software_conflicts_img.python, ".", "-")}python"
+  description = "Check for conflicts in application definitions on Ramble pull requests"
+
+  github {
+    owner = var.github_owner
+    name  = var.github_repo
+    pull_request {
+      branch          = "(?:main|develop)"
+      comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY"
+    }
+  }
+
+  include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS"
+
+  filename = "share/ramble/cloud-build/ramble-pr-software-conflicts.yaml"
+
+  included_files = [
+    "var/ramble/repos/**",
+    "lib/ramble/ramble/**",
+    "share/ramble/cloud-build/**"
+  ]
+
+  substitutions = {
+    _BASE_IMG   = local.pr_software_conflicts_img.base
+    _BASE_VER   = local.pr_software_conflicts_img.base_ver
+    _PYTHON_VER = local.pr_software_conflicts_img.python
+    _SPACK_REF  = local.pr_software_conflicts_img.spack
+  }
+}

share/ramble/cloud-build/terraform/triggers/pr_style.tf

@@ -0,0 +1,28 @@
+locals {
+  pr_style_img = local.image_map["rocky8-py3-12-1-spack-v0-22-1"]
+}
+
+resource "google_cloudbuild_trigger" "pr_style" {
+  name        = "PR-Style-${local.pr_style_img.base}${local.pr_style_img.base_ver}-${replace(local.pr_style_img.spack, ".", "-")}spack-${replace(local.pr_style_img.python, ".", "-")}python"
+  description = "Run linting on Ramble pull requests"
+
+  github {
+    owner = var.github_owner
+    name  = var.github_repo
+    pull_request {
+      branch          = "(?:main|develop)"
+      comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY"
+    }
+  }
+
+  include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS"
+
+  filename = "share/ramble/cloud-build/ramble-pr-style.yaml"
+
+  substitutions = {
+    _BASE_IMG   = local.pr_style_img.base
+    _BASE_VER   = local.pr_style_img.base_ver
+    _PYTHON_VER = local.pr_style_img.python
+    _SPACK_REF  = local.pr_style_img.spack
+  }
+}

share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf

@@ -0,0 +1,37 @@
+resource "google_cloudbuild_trigger" "pr_unit_tests" {
+  for_each = local.image_map
+
+  name        = "PR-Unit-Tests-${each.value.base}${replace(each.value.base_ver, ".", "-")}-${replace(each.value.spack, ".", "-")}spack-${replace(each.value.python, ".", "-")}python"
+  description = "Run unit tests and linting on Ramble pull requests"
+
+  github {
+    owner = var.github_owner
+    name  = var.github_repo
+    pull_request {
+      branch          = "(?:main|develop)"
+      comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY"
+    }
+  }
+
+  ignored_files = [
+    "lib/ramble/docs/**"
+  ]
+
+  included_files = [
+    "lib/ramble/**",
+    "var/ramble/repos/**",
+    "share/ramble/cloud-build/**",
+    "conftest.py"
+  ]
+
+  include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS"
+
+  filename = "share/ramble/cloud-build/ramble-pr-unit-tests.yaml"
+
+  substitutions = {
+    _BASE_IMG   = each.value.base
+    _BASE_VER   = each.value.base_ver
+    _PYTHON_VER = each.value.python
+    _SPACK_REF  = each.value.spack
+  }
+}

share/ramble/cloud-build/terraform/triggers/providers.tf

@@ -0,0 +1,13 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 7.0"
+    }
+  }
+}
+
+provider "google" {
+  project = var.project_id
+  region  = var.region
+}

share/ramble/cloud-build/terraform/triggers/terraform.tfvars

@@ -0,0 +1,4 @@
+project_id   = "ramble-eng"
+region       = "us-central1"
+github_owner = "GoogleCloudPlatform"
+github_repo  = "ramble"

share/ramble/cloud-build/terraform/triggers/variables.tf

@@ -0,0 +1,22 @@
+variable "project_id" {
+  type        = string
+  description = "The GCP Project ID to host the Cloud Build triggers"
+}
+
+variable "region" {
+  type        = string
+  description = "The GCP Region to deploy the triggers into"
+  default     = "us-central1"
+}
+
+variable "github_owner" {
+  type        = string
+  description = "The GitHub organization or user hosting the repository"
+  default     = "GoogleCloudPlatform"
+}
+
+variable "github_repo" {
+  type        = string
+  description = "The GitHub repository name"
+  default     = "ramble"
+}