Add Ubuntu build automation via PNNL GitLab

.gitlab-ci.yml

@@ -0,0 +1,141 @@
+default:
+  tags: [basic, gridpack, ikp, k8s]
+  artifacts:
+    when: always
+    untracked: true
+  image: ubuntu:22.04
+
+stages:
+- check-container
+- build-container
+- build-gridpack
+- test-gridpack
+
+variables:
+  MAKE_JOBS: "2"
+  UBUNTU_BASE_IMAGE: "ubuntu:22.04"
+  UBUNTU_ENV_IMAGE: "ubuntu-gridpack-env"
+  ROCKY_BASE_IMAGE: "rockylinux:9"
+  ROCKY_ENV_IMAGE: "rocky-gridpack-env"
+
+check-container-definition-changed:
+  stage: check-container
+  script:
+    - touch container-definition-changed
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "push"
+      changes:
+        - "build-system/dockerfile"
+        - "build-system/install_package_deps_lib.sh"
+        - "build-system/install_boost.sh"
+        - "build-system/install_ga.sh"
+        - "build-system/install_petsc.sh"
+
+.check-container-needs-built:
+  stage: check-container
+  needs:
+    - job: check-container-definition-changed
+      optional: true
+  script:
+    - set -o xtrace -o errexit -o nounset -o pipefail
+    - apt-get update && apt-get install -y curl jq
+    - source ./build-system/container_lib.sh
+    - check_container_needs_built "${ENV_IMAGE}"
+
+check-container-needs-built:ubuntu:
+  extends: .check-container-needs-built
+  variables:
+    ENV_IMAGE: ${UBUNTU_ENV_IMAGE}
+
+check-container-needs-built:rocky:
+  extends: .check-container-needs-built
+  variables:
+    ENV_IMAGE: ${ROCKY_ENV_IMAGE}
+  when: manual # todo remove when ready to test with rockylinux
+
+# https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy
+.build-container:
+  stage: build-container
+  timeout: 5 hours
+  image:
+    name: gcr.io/kaniko-project/executor:v1.9.0-debug
+    entrypoint: [""]
+  script:
+    - set -o errexit -o pipefail
+    - test -f container-needs-built || test "$FORCE_CONTAINER_BUILD" = "true" || exit 0
+    - >-
+      /kaniko/executor
+      --context build-system
+      --build-arg "BASE_IMAGE=${BASE_IMAGE}"
+      --build-arg "http_proxy=${HTTP_PROXY}"
+      --build-arg "https_proxy=${HTTPS_PROXY}"
+      --dockerfile build-system/dockerfile
+      --destination "${CI_REGISTRY_IMAGE}:${ENV_IMAGE}" 2>&1 | tee build.log
+
+build-container:ubuntu:
+  extends: .build-container
+  needs: [check-container-needs-built:ubuntu]
+  variables:
+    BASE_IMAGE: ${UBUNTU_BASE_IMAGE}
+    ENV_IMAGE: ${UBUNTU_ENV_IMAGE}
+
+build-container:rocky:
+  extends: .build-container
+  needs: [check-container-needs-built:rocky]
+  variables:
+    BASE_IMAGE: ${ROCKY_BASE_IMAGE}
+    ENV_IMAGE: ${ROCKY_ENV_IMAGE}
+  when: manual # todo remove when ready to test with rockylinux
+
+.build-gridpack:
+  stage: build-gridpack
+  script:
+    - /bin/bash ./build-system/install_gridpack.sh
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "push"
+      changes:
+        - "build-system/**/*"
+        - "src/**/*"
+        - "python/**/*"
+    - if: $CI_PIPELINE_SOURCE == "web"
+
+build-gridpack:ubuntu:
+  extends: .build-gridpack
+  image: ${CI_REGISTRY_IMAGE}:${UBUNTU_ENV_IMAGE}
+  needs: [build-container:ubuntu]
+
+build-gridpack:rocky:
+  extends: .build-gridpack
+  image: ${CI_REGISTRY_IMAGE}:${ROCKY_ENV_IMAGE}
+  needs: [build-container:rocky]
+  when: manual # todo remove when ready to test with rockylinux
+
+.test-gridpack:
+  stage: test-gridpack
+  script:
+    # load mpi module if on RHEL
+    - source ./build-system/install_package_deps_lib.sh
+    - load_mpi_module
+
+    # run tests
+    - ctest --test-dir src/build --output-on-failure
+  artifacts:
+    paths: [src/build/Testing/Temporary]
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "push"
+      changes:
+        - "build-system/**/*"
+        - "src/**/*"
+        - "python/**/*"
+    - if: $CI_PIPELINE_SOURCE == "web"
+
+test-gridpack:ubuntu:
+  extends: .test-gridpack
+  image: ${CI_REGISTRY_IMAGE}:${UBUNTU_ENV_IMAGE}
+  needs: [build-gridpack:ubuntu]
+
+test-gridpack:rocky:
+  extends: .test-gridpack
+  image: ${CI_REGISTRY_IMAGE}:${ROCKY_ENV_IMAGE}
+  needs: [build-gridpack:rocky]
+  when: manual # todo remove when ready to test with rockylinux

README.md

@@ -3,6 +3,9 @@
 # GridPACK<sup>TM</sup>-->
 # GridPACK: High-Performance Electric Grid Simulation
 
+![PNNL Build Status](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fapi.github.com%2Frepos%2FGridOPTICS%2FGridPACK%2Fcommits%2Fdevelop%2Fstatus&query=%24.state&logo=gitlab&label=build%20status&color=white
+)
+
 GridPACK is an open-source high-performance (HPC) package for simulation of large-scale electrical grids. Powered by distributed (parallel) computing and high-performance numerical solvers, GridPACK offers several applications forfast simulation of electrical transmission systems. GridPACK includes a number of prebuilt applications that can be directly used. The most commonly used and well-developed are:
 - AC Power Flow
 - Dynamics Simulation
@@ -18,7 +21,7 @@ In addition, GridPACK is also a framework to simplify the development of new app
 See the [instructions](docs/markdown/BASIC_INSTALL.md) for installing GridPACK, prerequisite software, and installation notes for different platforms.
 
 ## Usage
-See [User manual](docs/user_manual/GridPACK.pdf) for a deep dive on GridPACK internals and/or refer to the [tutorials](docs/markdown/TUTORIALS.md) for more info. 
+See [User manual](docs/user_manual/GridPACK.pdf) for a deep dive on GridPACK internals and/or refer to the [tutorials](docs/markdown/TUTORIALS.md) for more info.
 
 - Quick Guide (To do)
 
@@ -36,16 +39,16 @@ The best (and fastest) way to reach us for any technical questions is by posting
 
 ## Citing GridPACK
 ```
-@article{doi:10.1177/1094342015607609, 
-author = {Bruce Palmer and William Perkins and Yousu Chen and Shuangshuang Jin and David C allahan and Kevin Glass and Ruisheng Diao and Mark Rice and Stephen Elbert and Mallikarjun a Vallem and Zhenyu Huang}, 
-title ={GridPACKTM: A framework for developing power grid simulations on high-performance computing platforms}, 
-journal = {The International Journal of High Performance Computing Applications}, 
-volume = {30}, 
-number = {2}, 
-pages = {223-240}, 
-year = {2016}, 
-doi = {10.1177/1094342015607609}, 
-URL = {https://doi.org/10.1177/1094342015607609}, 
+@article{doi:10.1177/1094342015607609,
+author = {Bruce Palmer and William Perkins and Yousu Chen and Shuangshuang Jin and David C allahan and Kevin Glass and Ruisheng Diao and Mark Rice and Stephen Elbert and Mallikarjun a Vallem and Zhenyu Huang},
+title ={GridPACKTM: A framework for developing power grid simulations on high-performance computing platforms},
+journal = {The International Journal of High Performance Computing Applications},
+volume = {30},
+number = {2},
+pages = {223-240},
+year = {2016},
+doi = {10.1177/1094342015607609},
+URL = {https://doi.org/10.1177/1094342015607609},
 eprint = {https://doi.org/10.1177/1094342015607609}
 ```
 
@@ -69,7 +72,7 @@ GridPACK has been developed through funding from various sources over the years.
 ## Copyright
 Copyright &copy; 2013, Battelle Memorial Institute.
 
-GridPACK<sup>TM</sup> is a free software distributed under a BSD 2-clause license. You may reuse, modify, and redistribute the software. 
+GridPACK<sup>TM</sup> is a free software distributed under a BSD 2-clause license. You may reuse, modify, and redistribute the software.
 
 See the [license](src/LICENSE.md) file for details.
 

build-system/README.md

@@ -0,0 +1,25 @@
+The shell scripts in this directory are used to automate the installation of GridPACK on Debian/RHEL based systems.
+
+# GitLab Build Automation
+
+## Repository Mirror
+
+A [PNNL GitLab instance](https://devops.pnnl.gov/gridpack-code/GridPACK) hosting a [pull-mirror](https://docs.gitlab.com/ee/user/project/repository/mirror/pull.html) of this [GitHub repo](https://github.com/GridOPTICS/GridPACK) uses these scripts to execute tests whenever commits are pushed to GitHub. GitLab polls the GitHub repo to check for new commits every 30 minutes or every 5 minutes when triggered by the GitLab UI/API. An [integration](https://docs.gitlab.com/ee/ci/ci_cd_for_external_repos/github_integration.html) is configured to report a pass/fail status back to GitHub for any commit associated with a build ["pipeline"](https://docs.gitlab.com/ee/ci/pipelines/).
+
+## Build Jobs
+
+The build/test automation is defined by the following set of ["jobs"](https://docs.gitlab.com/ee/ci/jobs/) in `/.gitlab-ci.yml`, which are conditionally added to a pipeline by GitLab when a build is triggered:
+
+1. `check-container-definition-changed`: Creates a file called `container-definition-changed` if the scripts or `./dockerfile` used to build the container image have been modified. The file is passed to subsequent jobs as an ["artifact"](https://docs.gitlab.com/ee/ci/jobs/job_artifacts.html).
+2. `.check-container-needs-built`: A template (note the leading `.`) to define jobs for multiple distributions via [extension](https://docs.gitlab.com/ee/ci/yaml/#extends). Runs `check_container_needs_built`, a function in `./container_lib.sh` which creates another file called `container-needs-built` if `container-definition-changed` exists or the container image tag does not exist in the GitLab project [container registry](https://docs.gitlab.com/ee/user/packages/container_registry/). Additionally, if the container image does need to be rebuilt, this function [deletes](https://docs.gitlab.com/ee/api/container_registry.html#delete-a-registry-repository-tag) the image tag from the registry to avoid unintentional use by subsequent pipelines in the event that the container image build fails and does not replace the old image.
+3. `.build-container`: A template job for multiple distributions. Uses [kaniko](https://github.com/GoogleContainerTools/kaniko) to build a container image based on a given distribution if the file `container-needs-built` exists or an environment variable called `FORCE_CONTAINER_BUILD` is set to `true`. The environment variable is intended to be used when [manually triggering a pipeline from the GitLab UI](https://docs.gitlab.com/ee/ci/pipelines/#run-a-pipeline-manually).
+4. `.build-gridpack`: A template job for multiple distributions. Builds GridPACK in a container using an image from the GitLab project container registry based on a given distribution whenever files in `/src`, `/python`, or `/build` are modified using `./install_gridpack.sh`. All files created by the build are saved as artifacts.
+5. `.test-gridpack`: A template job for multiple distributions. Tests GridPACK in a new container using the artifacts from the associated `.build-container` job and with the same container image that was used in that job. These tests could have been performed at the end of the `.build-container` job but were separated to provide a distinct status for each of these concerns and to maintain organized logs/artifacts.
+
+`check-container-definition-changed` and `.check-container-needs-built` are very slim, taking about a minute each to run. Considering the source code and GitLab runner resources as of writing, `.build_container` takes about 35 minutes, `.build-gridpack` about 25 minutes, and `.test-gridpack` about 15 minutes for the `ubuntu:22.04` base image. Note again that the `.build_container` job only needs to run if the container image definition has changed. This mainly means changes to package dependencies, from-source dependencies, or environment variables.
+
+## Variables
+
+From the [project settings](https://devops.pnnl.gov/gridpack-code/GridPACK/-/settings/ci_cd) in GitLab a few variables are defined that are injected into each build pipeline. The difference between this type of variable and those in `/.gitlab-ci.yml` is that they can contain secrets and do not require source code modification to change. `API_TOKEN` is a [project access token](https://devops.pnnl.gov/gridpack-code/GridPACK/-/settings/access_tokens) which allows jobs to interact with the container registry API. When PNNL's GitLab instance is upgraded to `v16.8`, this variable should no longer be necessary. At that point it can be removed and curl can use the `JOB-TOKEN: ${CI_JOB_TOKEN}` header instead. That is desirable because the token will not need to be rotated.
+
+Another set of variables are used to manage resources available to the [GitLab runner](https://docs.gitlab.com/runner/executors/kubernetes/) executing each job in Kubernetes: `KUBERNETES_CPU_LIMIT`, `KUBERNETES_CPU_REQUEST`, `KUBERNETES_MEMORY_LIMIT`, and `KUBERNETES_MEMORY_REQUEST`. These mainly need to be considered if tests are performed for multiple distributions because that makes jobs available for concurrent processing. In this situation active jobs must share the resources from the namespace quota. Currently the runner is configured to allow two concurrent jobs, but the performance of this hasn't been compared to a serial approach with full resources.

build-system/container_lib.sh

@@ -0,0 +1,146 @@
+#! /bin/bash
+
+# library of functions related to building the container image
+
+# print a set of args as a comma-space seperated list
+# usage:
+#   print_array item1 item2 item3
+# usage:
+#   array=('item1' 'item2' 'item3')
+#   print_array "${array[@]}"
+# https://www.gnu.org/software/bash/manual/html_node/Arrays.html
+print_array() {
+  local list
+  list=$(printf "'%s', " "${@}")
+  echo "${list%, }"
+}
+
+# check that a command is available
+# usage:
+#   check_installed jq curl
+check_installed() {
+  # gather a list of missing commands
+  missing_commands=()
+  for cmd in "${@}"; do
+    if ! command -v "$cmd" &>/dev/null; then
+      missing_commands+=("$cmd")
+    fi
+  done
+
+  # if any commands are missing, print an error and exit
+  if [[ ${#missing_commands[@]} -gt 0 ]]; then
+    local list
+    list=$(print_array "${missing_commands[@]}")
+    echo "Please install $list" >&2
+    exit 1
+  fi
+}
+
+# make a gitlab api call
+# usage:
+#   glab_api GET /projects/123/registry/repositories
+#   glab_api POST /projects/123/registry/repositories "name=foo" "path=bar"
+# needs: curl, $CI_API_V4_URL, $API_TOKEN
+function glab_api {
+  local method=${1:?}
+  local endpoint=${2:?}
+  local form_items=("${@:3}")
+
+  local api_url=${CI_API_V4_URL:?}
+  local api_token=${API_TOKEN:?}
+
+  check_installed curl
+
+  # construct the form data
+  local form_data=""
+  for item in "${form_items[@]}"; do
+    form_data+="--form ${item} "
+  done
+
+  # curl options:
+  # - no-progress-meter: don't show progress bar but show errors
+  # - location: follow redirects
+  # - fail: return 22 on server errors
+  # - header: send private token for authentication
+  # - request: specify the HTTP method
+  # https://docs.gitlab.com/ee/api/container_registry.html
+  curl \
+    --no-progress-meter \
+    --fail \
+    --location \
+    --header "PRIVATE-TOKEN: ${api_token}" \
+    --request "${method}" \
+    "${form_data}${api_url}${endpoint}"
+}
+
+# get the id of a container registry repo for a project by path
+# usage:
+#   get_container_registry_repo_id
+# needs: jq, curl, $CI_API_V4_URL, $API_TOKEN, $CI_PROJECT_ID, $CI_PROJECT_PATH
+function get_container_registry_repo_id {
+  local project_id=${CI_PROJECT_ID:?}
+  : "${CI_PROJECT_PATH:?}"
+  local project_path=${CI_PROJECT_PATH,,}
+
+  check_installed jq
+
+  # jq query:
+  # - raw-output: output raw strings instead of json
+  # - exit-status: return 1 if no results
+  # - for each elements of the array
+  # - where the registry repo path is the one we care about
+  # - select the id prop
+  # https://jqlang.github.io/jq/manual/
+  glab_api GET "/projects/${project_id}/registry/repositories" \
+  | jq --raw-output --exit-status ".[] | select(.path == \"${project_path}\") | .id"
+}
+
+# check if a tag exists in a project's container registry
+# usage:
+#   container_tag_exists 123 "my-tag"
+# needs: curl, $CI_API_V4_URL, $API_TOKEN, $CI_PROJECT_ID
+function container_tag_exists {
+  local reg_repo_id=${1:?}
+  local image_tag=${2:?}
+
+  local project_id=${CI_PROJECT_ID:?}
+
+  glab_api GET "/projects/${project_id}/registry/repositories/${reg_repo_id}/tags/${image_tag}" >/dev/null
+}
+
+# delete tag from a project's container registry
+# usage:
+#  delete_container_tag 123 "my-tag"
+# needs: curl, $CI_API_V4_URL, $API_TOKEN, $CI_PROJECT_ID
+function delete_container_tag {
+  local reg_repo_id=${1:?}
+  local image_tag=${2:?}
+
+  local project_id=${CI_PROJECT_ID:?}
+
+  glab_api DELETE "/projects/${project_id}/registry/repositories/${reg_repo_id}/tags/${image_tag}" >/dev/null
+}
+
+# build container image if not available in registry or force_rebuild = 'true'
+# usage:
+#   check_container_needs_built "ubuntu:22.04"
+# needs: jq, curl, $CI_API_V4_URL, $API_TOKEN, $CI_PROJECT_ID, $CI_PROJECT_PATH_SLUG
+function check_container_needs_built {
+  local tag=${1:?}
+
+  local repo_id
+  repo_id=$(get_container_registry_repo_id)
+
+  # if we see a file in the current directory called "container-definition-changed"
+  # or the tag does not exist in the registry, then the container needs rebuilt
+  if test -f ./container-definition-changed || ! container_tag_exists "$repo_id" "$tag"; then
+    # in case the build fails, delete the image associated with this tag from the registry
+    # this will signal to potential subsequent pipelines that the container image still needs
+    # rebuilt even if relevant files have not changed
+    # this provents unknowingly using an outdated image in subsequent pipelines
+    delete_container_tag "$repo_id" "$tag"
+
+    # signal to the next job via an empty file artifact that the container needs built
+    touch ./container-needs-built
+  fi
+}