diff --git a/share/ramble/cloud-build/terraform/triggers/.gitignore b/share/ramble/cloud-build/terraform/triggers/.gitignore new file mode 100644 index 000000000..048ba8a6e --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/.gitignore @@ -0,0 +1,3 @@ +# We don't ignore any tfstate files, as those should reside on the GCS bucket instead +.terraform/ +.terraform.lock.hcl \ No newline at end of file diff --git a/share/ramble/cloud-build/terraform/triggers/README.md b/share/ramble/cloud-build/terraform/triggers/README.md new file mode 100644 index 000000000..a9f7e3504 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/README.md @@ -0,0 +1,20 @@ +# Ramble Cloud Build Image Triggers + +This directory contains Terraform configuration to deploy and manage Google Cloud Build Triggers used by the Ramble repository. + +## How to deploy + +The deployment states are stored in a GCS bucket, to allow for running Terraform from different locations. The bucket was created with: + +```bash +gcloud storage buckets create gs://ramble-terraform-state --project=ramble-eng --location=us-central1 +``` + +With the bucket created, the triggers can be deployed with: + +```bash +terraform init +# Optional to check the changes to be made +terraform plan +terraform apply --auto-approve +``` diff --git a/share/ramble/cloud-build/terraform/triggers/backend.tf b/share/ramble/cloud-build/terraform/triggers/backend.tf new file mode 100644 index 000000000..37d6fbb20 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/backend.tf @@ -0,0 +1,6 @@ +terraform { + backend "gcs" { + bucket = "ramble-terraform-state" + prefix = "terraform/state/image-triggers" + } +} diff --git a/share/ramble/cloud-build/terraform/triggers/image_builds.tf b/share/ramble/cloud-build/terraform/triggers/image_builds.tf new file mode 100644 index 000000000..1ba627160 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/image_builds.tf @@ -0,0 +1,29 @@ +resource "google_cloudbuild_trigger" "image_builders" { + for_each = local.image_map + + name = "ramble-image-builder-${each.value.base}${replace(each.value.base_ver, ".", "-")}-py${replace(each.value.python, ".", "-")}-spack${replace(each.value.spack, ".", "-")}" + description = "Build Ramble cloud build image for ${each.value.base} ${each.value.base_ver} with Python ${each.value.python} and Spack ${each.value.spack}" + + github { + owner = var.github_owner + name = var.github_repo + push { + branch = "^develop$" + } + } + + included_files = [ + "share/ramble/cloud-build/ramble-image-builder.yaml", + "share/ramble/cloud-build/Dockerfile-${local.pm_map[each.value.base]}" + ] + + filename = "share/ramble/cloud-build/ramble-image-builder.yaml" + + substitutions = { + _PYTHON_VER = each.value.python + _SPACK_REF = each.value.spack + _PKG_MANAGER = local.pm_map[each.value.base] + _BASE_IMG = each.value.base + _BASE_VER = each.value.base_ver + } +} diff --git a/share/ramble/cloud-build/terraform/triggers/locals.tf b/share/ramble/cloud-build/terraform/triggers/locals.tf new file mode 100644 index 000000000..9ce73960c --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/locals.tf @@ -0,0 +1,23 @@ +locals { + # This map holds the full list of available images. + # A trigger should always reference these images instead of hard-coding new ones. + image_map = { + "debian12-5-py3-12-0-spack-v0-21-2" = { python = "3.12.0", spack = "v0.21.2", base = "debian", base_ver = "12.5" }, + "debian12-5-py3-8-0-spack-v0-21-2" = { python = "3.8.0", spack = "v0.21.2", base = "debian", base_ver = "12.5" }, + "debian12-5-py3-12-1-spack-v0-22-1" = { python = "3.12.1", spack = "v0.22.1", base = "debian", base_ver = "12.5" }, + "debian12-5-py3-8-0-spack-v0-22-1" = { python = "3.8.0", spack = "v0.22.1", base = "debian", base_ver = "12.5" }, + "debian12-5-py3-13-5-spack-v1-0-0" = { python = "3.13.5", spack = "v1.0.0", base = "debian", base_ver = "12.5" }, + "debian12-5-py3-7-17-spack-v1-0-0" = { python = "3.7.17", spack = "v1.0.0", base = "debian", base_ver = "12.5" }, + "rocky8-py3-12-0-spack-v0-21-2" = { python = "3.12.0", spack = "v0.21.2", base = "rockylinux", base_ver = "8" }, + "rocky8-py3-7-0-spack-v0-21-2" = { python = "3.7.0", spack = "v0.21.2", base = "rockylinux", base_ver = "8" }, + "rocky8-py3-12-1-spack-v0-22-1" = { python = "3.12.1", spack = "v0.22.1", base = "rockylinux", base_ver = "8" }, + "rocky8-py3-7-0-spack-v0-22-1" = { python = "3.7.0", spack = "v0.22.1", base = "rockylinux", base_ver = "8" }, + "rocky8-py3-13-5-spack-v1-0-0" = { python = "3.13.5", spack = "v1.0.0", base = "rockylinux", base_ver = "8" }, + "rocky8-py3-7-17-spack-v1-0-0" = { python = "3.7.17", spack = "v1.0.0", base = "rockylinux", base_ver = "8" }, + } + + pm_map = { + "debian" = "apt" + "rockylinux" = "yum" + } +} diff --git a/share/ramble/cloud-build/terraform/triggers/perf_tests.tf b/share/ramble/cloud-build/terraform/triggers/perf_tests.tf new file mode 100644 index 000000000..f2d94fb05 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/perf_tests.tf @@ -0,0 +1,58 @@ +locals { + perf_test_img = local.image_map["rocky8-py3-13-5-spack-v1-0-0"] +} + +resource "google_cloudbuild_trigger" "perf_test_pr" { + name = "PerfTest-PR-${local.perf_test_img.base}${local.perf_test_img.base_ver}-${replace(local.perf_test_img.spack, ".", "-")}spack-${replace(local.perf_test_img.python, ".", "-")}python" + description = "Ramble perf tests for PR builds" + + github { + owner = var.github_owner + name = var.github_repo + pull_request { + branch = "(?:main|develop)" + comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY" + } + } + + include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS" + + filename = "share/ramble/cloud-build/ramble-perf-tests.yaml" + + substitutions = { + _SPACK_REF = local.perf_test_img.spack + _PYTHON_VER = local.perf_test_img.python + _BASE_IMG = local.perf_test_img.base + _BASE_VER = local.perf_test_img.base_ver + _DATASET_ID = "ramble_metrics" + _PROJECT_ID = var.project_id + _TABLE_ID = "perf_test_durations" + _UPLOAD_TO_BQ = "false" + } +} + +resource "google_cloudbuild_trigger" "perf_test_push" { + name = "PerfTest-Push-${local.perf_test_img.base}${local.perf_test_img.base_ver}-${replace(local.perf_test_img.spack, ".", "-")}spack-${replace(local.perf_test_img.python, ".", "-")}python" + description = "Continuous monitoring of Ramble performance for develop push" + + github { + owner = var.github_owner + name = var.github_repo + push { + branch = "^develop$" + } + } + + filename = "share/ramble/cloud-build/ramble-perf-tests.yaml" + + substitutions = { + _SPACK_REF = local.perf_test_img.spack + _PYTHON_VER = local.perf_test_img.python + _BASE_IMG = local.perf_test_img.base + _BASE_VER = local.perf_test_img.base_ver + _DATASET_ID = "ramble_metrics" + _PROJECT_ID = var.project_id + _TABLE_ID = "perf_test_durations" + _UPLOAD_TO_BQ = "true" + } +} diff --git a/share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf b/share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf new file mode 100644 index 000000000..6d499de97 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/pr_doc_build.tf @@ -0,0 +1,28 @@ +locals { + pr_doc_img = local.image_map["rocky8-py3-13-5-spack-v1-0-0"] +} + +resource "google_cloudbuild_trigger" "pr_doc_build_tests" { + name = "PR-Doc-Build-Tests" + description = "A presubmit check for building Ramble documentation" + + github { + owner = var.github_owner + name = var.github_repo + pull_request { + branch = "(?:main|develop)" + comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY" + } + } + + include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS" + + filename = "share/ramble/cloud-build/ramble-pr-docs.yaml" + + substitutions = { + _BASE_IMG = local.pr_doc_img.base + _BASE_VER = local.pr_doc_img.base_ver + _PYTHON_VER = local.pr_doc_img.python + _SPACK_REF = local.pr_doc_img.spack + } +} diff --git a/share/ramble/cloud-build/terraform/triggers/pr_image_build.tf b/share/ramble/cloud-build/terraform/triggers/pr_image_build.tf new file mode 100644 index 000000000..b1c30d979 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/pr_image_build.tf @@ -0,0 +1,21 @@ +resource "google_cloudbuild_trigger" "pr_image_build_tests" { + name = "PR-Image-Build-Tests" + description = "A presubmit check for building images used by other cloud build triggers" + + github { + owner = var.github_owner + name = var.github_repo + pull_request { + branch = "(?:main|develop)" + comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY" + } + } + + include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS" + + filename = "share/ramble/cloud-build/ramble-pr-image-builds.yaml" + + included_files = [ + "share/ramble/cloud-build/**" + ] +} diff --git a/share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf b/share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf new file mode 100644 index 000000000..fdc9e3729 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/pr_software_conflicts.tf @@ -0,0 +1,34 @@ +locals { + pr_software_conflicts_img = local.image_map["rocky8-py3-12-1-spack-v0-22-1"] +} + +resource "google_cloudbuild_trigger" "pr_software_conflicts" { + name = "PR-Software-Conflicts-${local.pr_software_conflicts_img.base}${local.pr_software_conflicts_img.base_ver}-${replace(local.pr_software_conflicts_img.spack, ".", "-")}spack-${replace(local.pr_software_conflicts_img.python, ".", "-")}python" + description = "Check for conflicts in application definitions on Ramble pull requests" + + github { + owner = var.github_owner + name = var.github_repo + pull_request { + branch = "(?:main|develop)" + comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY" + } + } + + include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS" + + filename = "share/ramble/cloud-build/ramble-pr-software-conflicts.yaml" + + included_files = [ + "var/ramble/repos/**", + "lib/ramble/ramble/**", + "share/ramble/cloud-build/**" + ] + + substitutions = { + _BASE_IMG = local.pr_software_conflicts_img.base + _BASE_VER = local.pr_software_conflicts_img.base_ver + _PYTHON_VER = local.pr_software_conflicts_img.python + _SPACK_REF = local.pr_software_conflicts_img.spack + } +} diff --git a/share/ramble/cloud-build/terraform/triggers/pr_style.tf b/share/ramble/cloud-build/terraform/triggers/pr_style.tf new file mode 100644 index 000000000..39ebd5e2a --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/pr_style.tf @@ -0,0 +1,28 @@ +locals { + pr_style_img = local.image_map["rocky8-py3-12-1-spack-v0-22-1"] +} + +resource "google_cloudbuild_trigger" "pr_style" { + name = "PR-Style-${local.pr_style_img.base}${local.pr_style_img.base_ver}-${replace(local.pr_style_img.spack, ".", "-")}spack-${replace(local.pr_style_img.python, ".", "-")}python" + description = "Run linting on Ramble pull requests" + + github { + owner = var.github_owner + name = var.github_repo + pull_request { + branch = "(?:main|develop)" + comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY" + } + } + + include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS" + + filename = "share/ramble/cloud-build/ramble-pr-style.yaml" + + substitutions = { + _BASE_IMG = local.pr_style_img.base + _BASE_VER = local.pr_style_img.base_ver + _PYTHON_VER = local.pr_style_img.python + _SPACK_REF = local.pr_style_img.spack + } +} diff --git a/share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf b/share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf new file mode 100644 index 000000000..7f5f3134d --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/pr_unit_tests.tf @@ -0,0 +1,37 @@ +resource "google_cloudbuild_trigger" "pr_unit_tests" { + for_each = local.image_map + + name = "PR-Unit-Tests-${each.value.base}${replace(each.value.base_ver, ".", "-")}-${replace(each.value.spack, ".", "-")}spack-${replace(each.value.python, ".", "-")}python" + description = "Run unit tests and linting on Ramble pull requests" + + github { + owner = var.github_owner + name = var.github_repo + pull_request { + branch = "(?:main|develop)" + comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY" + } + } + + ignored_files = [ + "lib/ramble/docs/**" + ] + + included_files = [ + "lib/ramble/**", + "var/ramble/repos/**", + "share/ramble/cloud-build/**", + "conftest.py" + ] + + include_build_logs = "INCLUDE_BUILD_LOGS_WITH_STATUS" + + filename = "share/ramble/cloud-build/ramble-pr-unit-tests.yaml" + + substitutions = { + _BASE_IMG = each.value.base + _BASE_VER = each.value.base_ver + _PYTHON_VER = each.value.python + _SPACK_REF = each.value.spack + } +} diff --git a/share/ramble/cloud-build/terraform/triggers/providers.tf b/share/ramble/cloud-build/terraform/triggers/providers.tf new file mode 100644 index 000000000..b404ad86d --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/providers.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 7.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} diff --git a/share/ramble/cloud-build/terraform/triggers/terraform.tfvars b/share/ramble/cloud-build/terraform/triggers/terraform.tfvars new file mode 100644 index 000000000..1c92804de --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/terraform.tfvars @@ -0,0 +1,4 @@ +project_id = "ramble-eng" +region = "us-central1" +github_owner = "GoogleCloudPlatform" +github_repo = "ramble" diff --git a/share/ramble/cloud-build/terraform/triggers/variables.tf b/share/ramble/cloud-build/terraform/triggers/variables.tf new file mode 100644 index 000000000..393526d48 --- /dev/null +++ b/share/ramble/cloud-build/terraform/triggers/variables.tf @@ -0,0 +1,22 @@ +variable "project_id" { + type = string + description = "The GCP Project ID to host the Cloud Build triggers" +} + +variable "region" { + type = string + description = "The GCP Region to deploy the triggers into" + default = "us-central1" +} + +variable "github_owner" { + type = string + description = "The GitHub organization or user hosting the repository" + default = "GoogleCloudPlatform" +} + +variable "github_repo" { + type = string + description = "The GitHub repository name" + default = "ramble" +}