diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..e95e8e818 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,141 @@ +default: + tags: [basic, gridpack, ikp, k8s] + artifacts: + when: always + untracked: true + image: ubuntu:22.04 + +stages: +- check-container +- build-container +- build-gridpack +- test-gridpack + +variables: + MAKE_JOBS: "2" + UBUNTU_BASE_IMAGE: "ubuntu:22.04" + UBUNTU_ENV_IMAGE: "ubuntu-gridpack-env" + ROCKY_BASE_IMAGE: "rockylinux:9" + ROCKY_ENV_IMAGE: "rocky-gridpack-env" + +check-container-definition-changed: + stage: check-container + script: + - touch container-definition-changed + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: + - "build-system/dockerfile" + - "build-system/install_package_deps_lib.sh" + - "build-system/install_boost.sh" + - "build-system/install_ga.sh" + - "build-system/install_petsc.sh" + +.check-container-needs-built: + stage: check-container + needs: + - job: check-container-definition-changed + optional: true + script: + - set -o xtrace -o errexit -o nounset -o pipefail + - apt-get update && apt-get install -y curl jq + - source ./build-system/container_lib.sh + - check_container_needs_built "${ENV_IMAGE}" + +check-container-needs-built:ubuntu: + extends: .check-container-needs-built + variables: + ENV_IMAGE: ${UBUNTU_ENV_IMAGE} + +check-container-needs-built:rocky: + extends: .check-container-needs-built + variables: + ENV_IMAGE: ${ROCKY_ENV_IMAGE} + when: manual # todo remove when ready to test with rockylinux + +# https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy +.build-container: + stage: build-container + timeout: 5 hours + image: + name: gcr.io/kaniko-project/executor:v1.9.0-debug + entrypoint: [""] + script: + - set -o errexit -o pipefail + - test -f container-needs-built || test "$FORCE_CONTAINER_BUILD" = "true" || exit 0 + - >- + /kaniko/executor + --context build-system + --build-arg "BASE_IMAGE=${BASE_IMAGE}" + --build-arg "http_proxy=${HTTP_PROXY}" + --build-arg "https_proxy=${HTTPS_PROXY}" + --dockerfile build-system/dockerfile + --destination "${CI_REGISTRY_IMAGE}:${ENV_IMAGE}" 2>&1 | tee build.log + +build-container:ubuntu: + extends: .build-container + needs: [check-container-needs-built:ubuntu] + variables: + BASE_IMAGE: ${UBUNTU_BASE_IMAGE} + ENV_IMAGE: ${UBUNTU_ENV_IMAGE} + +build-container:rocky: + extends: .build-container + needs: [check-container-needs-built:rocky] + variables: + BASE_IMAGE: ${ROCKY_BASE_IMAGE} + ENV_IMAGE: ${ROCKY_ENV_IMAGE} + when: manual # todo remove when ready to test with rockylinux + +.build-gridpack: + stage: build-gridpack + script: + - /bin/bash ./build-system/install_gridpack.sh + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: + - "build-system/**/*" + - "src/**/*" + - "python/**/*" + - if: $CI_PIPELINE_SOURCE == "web" + +build-gridpack:ubuntu: + extends: .build-gridpack + image: ${CI_REGISTRY_IMAGE}:${UBUNTU_ENV_IMAGE} + needs: [build-container:ubuntu] + +build-gridpack:rocky: + extends: .build-gridpack + image: ${CI_REGISTRY_IMAGE}:${ROCKY_ENV_IMAGE} + needs: [build-container:rocky] + when: manual # todo remove when ready to test with rockylinux + +.test-gridpack: + stage: test-gridpack + script: + # load mpi module if on RHEL + - source ./build-system/install_package_deps_lib.sh + - load_mpi_module + + # run tests + - ctest --test-dir src/build --output-on-failure + artifacts: + paths: [src/build/Testing/Temporary] + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: + - "build-system/**/*" + - "src/**/*" + - "python/**/*" + - if: $CI_PIPELINE_SOURCE == "web" + +test-gridpack:ubuntu: + extends: .test-gridpack + image: ${CI_REGISTRY_IMAGE}:${UBUNTU_ENV_IMAGE} + needs: [build-gridpack:ubuntu] + +test-gridpack:rocky: + extends: .test-gridpack + image: ${CI_REGISTRY_IMAGE}:${ROCKY_ENV_IMAGE} + needs: [build-gridpack:rocky] + when: manual # todo remove when ready to test with rockylinux diff --git a/README.md b/README.md index 08d243154..5b7422099 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,9 @@ # GridPACKTM--> # GridPACK: High-Performance Electric Grid Simulation +![PNNL Build Status](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fapi.github.com%2Frepos%2FGridOPTICS%2FGridPACK%2Fcommits%2Fdevelop%2Fstatus&query=%24.state&logo=gitlab&label=build%20status&color=white +) + GridPACK is an open-source high-performance (HPC) package for simulation of large-scale electrical grids. Powered by distributed (parallel) computing and high-performance numerical solvers, GridPACK offers several applications forfast simulation of electrical transmission systems. GridPACK includes a number of prebuilt applications that can be directly used. The most commonly used and well-developed are: - AC Power Flow - Dynamics Simulation @@ -18,7 +21,7 @@ In addition, GridPACK is also a framework to simplify the development of new app See the [instructions](docs/markdown/BASIC_INSTALL.md) for installing GridPACK, prerequisite software, and installation notes for different platforms. ## Usage -See [User manual](docs/user_manual/GridPACK.pdf) for a deep dive on GridPACK internals and/or refer to the [tutorials](docs/markdown/TUTORIALS.md) for more info. +See [User manual](docs/user_manual/GridPACK.pdf) for a deep dive on GridPACK internals and/or refer to the [tutorials](docs/markdown/TUTORIALS.md) for more info. - Quick Guide (To do) @@ -36,16 +39,16 @@ The best (and fastest) way to reach us for any technical questions is by posting ## Citing GridPACK ``` -@article{doi:10.1177/1094342015607609, -author = {Bruce Palmer and William Perkins and Yousu Chen and Shuangshuang Jin and David C allahan and Kevin Glass and Ruisheng Diao and Mark Rice and Stephen Elbert and Mallikarjun a Vallem and Zhenyu Huang}, -title ={GridPACKTM: A framework for developing power grid simulations on high-performance computing platforms}, -journal = {The International Journal of High Performance Computing Applications}, -volume = {30}, -number = {2}, -pages = {223-240}, -year = {2016}, -doi = {10.1177/1094342015607609}, -URL = {https://doi.org/10.1177/1094342015607609}, +@article{doi:10.1177/1094342015607609, +author = {Bruce Palmer and William Perkins and Yousu Chen and Shuangshuang Jin and David C allahan and Kevin Glass and Ruisheng Diao and Mark Rice and Stephen Elbert and Mallikarjun a Vallem and Zhenyu Huang}, +title ={GridPACKTM: A framework for developing power grid simulations on high-performance computing platforms}, +journal = {The International Journal of High Performance Computing Applications}, +volume = {30}, +number = {2}, +pages = {223-240}, +year = {2016}, +doi = {10.1177/1094342015607609}, +URL = {https://doi.org/10.1177/1094342015607609}, eprint = {https://doi.org/10.1177/1094342015607609} ``` @@ -69,7 +72,7 @@ GridPACK has been developed through funding from various sources over the years. ## Copyright Copyright © 2013, Battelle Memorial Institute. -GridPACKTM is a free software distributed under a BSD 2-clause license. You may reuse, modify, and redistribute the software. +GridPACKTM is a free software distributed under a BSD 2-clause license. You may reuse, modify, and redistribute the software. See the [license](src/LICENSE.md) file for details. diff --git a/build-system/README.md b/build-system/README.md new file mode 100644 index 000000000..7228ce051 --- /dev/null +++ b/build-system/README.md @@ -0,0 +1,25 @@ +The shell scripts in this directory are used to automate the installation of GridPACK on Debian/RHEL based systems. + +# GitLab Build Automation + +## Repository Mirror + +A [PNNL GitLab instance](https://devops.pnnl.gov/gridpack-code/GridPACK) hosting a [pull-mirror](https://docs.gitlab.com/ee/user/project/repository/mirror/pull.html) of this [GitHub repo](https://github.com/GridOPTICS/GridPACK) uses these scripts to execute tests whenever commits are pushed to GitHub. GitLab polls the GitHub repo to check for new commits every 30 minutes or every 5 minutes when triggered by the GitLab UI/API. An [integration](https://docs.gitlab.com/ee/ci/ci_cd_for_external_repos/github_integration.html) is configured to report a pass/fail status back to GitHub for any commit associated with a build ["pipeline"](https://docs.gitlab.com/ee/ci/pipelines/). + +## Build Jobs + +The build/test automation is defined by the following set of ["jobs"](https://docs.gitlab.com/ee/ci/jobs/) in `/.gitlab-ci.yml`, which are conditionally added to a pipeline by GitLab when a build is triggered: + +1. `check-container-definition-changed`: Creates a file called `container-definition-changed` if the scripts or `./dockerfile` used to build the container image have been modified. The file is passed to subsequent jobs as an ["artifact"](https://docs.gitlab.com/ee/ci/jobs/job_artifacts.html). +2. `.check-container-needs-built`: A template (note the leading `.`) to define jobs for multiple distributions via [extension](https://docs.gitlab.com/ee/ci/yaml/#extends). Runs `check_container_needs_built`, a function in `./container_lib.sh` which creates another file called `container-needs-built` if `container-definition-changed` exists or the container image tag does not exist in the GitLab project [container registry](https://docs.gitlab.com/ee/user/packages/container_registry/). Additionally, if the container image does need to be rebuilt, this function [deletes](https://docs.gitlab.com/ee/api/container_registry.html#delete-a-registry-repository-tag) the image tag from the registry to avoid unintentional use by subsequent pipelines in the event that the container image build fails and does not replace the old image. +3. `.build-container`: A template job for multiple distributions. Uses [kaniko](https://github.com/GoogleContainerTools/kaniko) to build a container image based on a given distribution if the file `container-needs-built` exists or an environment variable called `FORCE_CONTAINER_BUILD` is set to `true`. The environment variable is intended to be used when [manually triggering a pipeline from the GitLab UI](https://docs.gitlab.com/ee/ci/pipelines/#run-a-pipeline-manually). +4. `.build-gridpack`: A template job for multiple distributions. Builds GridPACK in a container using an image from the GitLab project container registry based on a given distribution whenever files in `/src`, `/python`, or `/build` are modified using `./install_gridpack.sh`. All files created by the build are saved as artifacts. +5. `.test-gridpack`: A template job for multiple distributions. Tests GridPACK in a new container using the artifacts from the associated `.build-container` job and with the same container image that was used in that job. These tests could have been performed at the end of the `.build-container` job but were separated to provide a distinct status for each of these concerns and to maintain organized logs/artifacts. + +`check-container-definition-changed` and `.check-container-needs-built` are very slim, taking about a minute each to run. Considering the source code and GitLab runner resources as of writing, `.build_container` takes about 35 minutes, `.build-gridpack` about 25 minutes, and `.test-gridpack` about 15 minutes for the `ubuntu:22.04` base image. Note again that the `.build_container` job only needs to run if the container image definition has changed. This mainly means changes to package dependencies, from-source dependencies, or environment variables. + +## Variables + +From the [project settings](https://devops.pnnl.gov/gridpack-code/GridPACK/-/settings/ci_cd) in GitLab a few variables are defined that are injected into each build pipeline. The difference between this type of variable and those in `/.gitlab-ci.yml` is that they can contain secrets and do not require source code modification to change. `API_TOKEN` is a [project access token](https://devops.pnnl.gov/gridpack-code/GridPACK/-/settings/access_tokens) which allows jobs to interact with the container registry API. When PNNL's GitLab instance is upgraded to `v16.8`, this variable should no longer be necessary. At that point it can be removed and curl can use the `JOB-TOKEN: ${CI_JOB_TOKEN}` header instead. That is desirable because the token will not need to be rotated. + +Another set of variables are used to manage resources available to the [GitLab runner](https://docs.gitlab.com/runner/executors/kubernetes/) executing each job in Kubernetes: `KUBERNETES_CPU_LIMIT`, `KUBERNETES_CPU_REQUEST`, `KUBERNETES_MEMORY_LIMIT`, and `KUBERNETES_MEMORY_REQUEST`. These mainly need to be considered if tests are performed for multiple distributions because that makes jobs available for concurrent processing. In this situation active jobs must share the resources from the namespace quota. Currently the runner is configured to allow two concurrent jobs, but the performance of this hasn't been compared to a serial approach with full resources. diff --git a/build-system/container_lib.sh b/build-system/container_lib.sh new file mode 100644 index 000000000..febdf0a63 --- /dev/null +++ b/build-system/container_lib.sh @@ -0,0 +1,146 @@ +#! /bin/bash + +# library of functions related to building the container image + +# print a set of args as a comma-space seperated list +# usage: +# print_array item1 item2 item3 +# usage: +# array=('item1' 'item2' 'item3') +# print_array "${array[@]}" +# https://www.gnu.org/software/bash/manual/html_node/Arrays.html +print_array() { + local list + list=$(printf "'%s', " "${@}") + echo "${list%, }" +} + +# check that a command is available +# usage: +# check_installed jq curl +check_installed() { + # gather a list of missing commands + missing_commands=() + for cmd in "${@}"; do + if ! command -v "$cmd" &>/dev/null; then + missing_commands+=("$cmd") + fi + done + + # if any commands are missing, print an error and exit + if [[ ${#missing_commands[@]} -gt 0 ]]; then + local list + list=$(print_array "${missing_commands[@]}") + echo "Please install $list" >&2 + exit 1 + fi +} + +# make a gitlab api call +# usage: +# glab_api GET /projects/123/registry/repositories +# glab_api POST /projects/123/registry/repositories "name=foo" "path=bar" +# needs: curl, $CI_API_V4_URL, $API_TOKEN +function glab_api { + local method=${1:?} + local endpoint=${2:?} + local form_items=("${@:3}") + + local api_url=${CI_API_V4_URL:?} + local api_token=${API_TOKEN:?} + + check_installed curl + + # construct the form data + local form_data="" + for item in "${form_items[@]}"; do + form_data+="--form ${item} " + done + + # curl options: + # - no-progress-meter: don't show progress bar but show errors + # - location: follow redirects + # - fail: return 22 on server errors + # - header: send private token for authentication + # - request: specify the HTTP method + # https://docs.gitlab.com/ee/api/container_registry.html + curl \ + --no-progress-meter \ + --fail \ + --location \ + --header "PRIVATE-TOKEN: ${api_token}" \ + --request "${method}" \ + "${form_data}${api_url}${endpoint}" +} + +# get the id of a container registry repo for a project by path +# usage: +# get_container_registry_repo_id +# needs: jq, curl, $CI_API_V4_URL, $API_TOKEN, $CI_PROJECT_ID, $CI_PROJECT_PATH +function get_container_registry_repo_id { + local project_id=${CI_PROJECT_ID:?} + : "${CI_PROJECT_PATH:?}" + local project_path=${CI_PROJECT_PATH,,} + + check_installed jq + + # jq query: + # - raw-output: output raw strings instead of json + # - exit-status: return 1 if no results + # - for each elements of the array + # - where the registry repo path is the one we care about + # - select the id prop + # https://jqlang.github.io/jq/manual/ + glab_api GET "/projects/${project_id}/registry/repositories" \ + | jq --raw-output --exit-status ".[] | select(.path == \"${project_path}\") | .id" +} + +# check if a tag exists in a project's container registry +# usage: +# container_tag_exists 123 "my-tag" +# needs: curl, $CI_API_V4_URL, $API_TOKEN, $CI_PROJECT_ID +function container_tag_exists { + local reg_repo_id=${1:?} + local image_tag=${2:?} + + local project_id=${CI_PROJECT_ID:?} + + glab_api GET "/projects/${project_id}/registry/repositories/${reg_repo_id}/tags/${image_tag}" >/dev/null +} + +# delete tag from a project's container registry +# usage: +# delete_container_tag 123 "my-tag" +# needs: curl, $CI_API_V4_URL, $API_TOKEN, $CI_PROJECT_ID +function delete_container_tag { + local reg_repo_id=${1:?} + local image_tag=${2:?} + + local project_id=${CI_PROJECT_ID:?} + + glab_api DELETE "/projects/${project_id}/registry/repositories/${reg_repo_id}/tags/${image_tag}" >/dev/null +} + +# build container image if not available in registry or force_rebuild = 'true' +# usage: +# check_container_needs_built "ubuntu:22.04" +# needs: jq, curl, $CI_API_V4_URL, $API_TOKEN, $CI_PROJECT_ID, $CI_PROJECT_PATH_SLUG +function check_container_needs_built { + local tag=${1:?} + + local repo_id + repo_id=$(get_container_registry_repo_id) + + # if we see a file in the current directory called "container-definition-changed" + # or the tag does not exist in the registry, then the container needs rebuilt + if test -f ./container-definition-changed || ! container_tag_exists "$repo_id" "$tag"; then + # in case the build fails, delete the image associated with this tag from the registry + # this will signal to potential subsequent pipelines that the container image still needs + # rebuilt even if relevant files have not changed + # this provents unknowingly using an outdated image in subsequent pipelines + delete_container_tag "$repo_id" "$tag" + + # signal to the next job via an empty file artifact that the container needs built + touch ./container-needs-built + fi +} diff --git a/build-system/dockerfile b/build-system/dockerfile new file mode 100644 index 000000000..a55ff6330 --- /dev/null +++ b/build-system/dockerfile @@ -0,0 +1,33 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +SHELL ["/bin/bash", "-c"] + +ARG http_proxy +ARG https_proxy + +ENV GP_EXT_DEPS=/gridpack-dependencies +ENV HTTP_PROXY=${http_proxy} \ + HTTPS_PROXY=${https_proxy} \ + OMPI_ALLOW_RUN_AS_ROOT="1" \ + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM="1" \ + LD_LIBRARY_PATH="${GP_EXT_DEPS}/boost/install_for_gridpack/lib:${GP_EXT_DEPS}/ga/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib" + +WORKDIR ${GP_EXT_DEPS} +RUN mkdir logs + +COPY install_package_deps_lib.sh ./ +RUN set -xeuo pipefail \ + && source ./install_package_deps_lib.sh \ + && install_packages >logs/install_packages.log 2>&1 + +COPY install_boost.sh ./ +RUN bash -o xtrace -o errexit -o nounset -o pipefail ./install_boost.sh >logs/install_boost.log 2>&1 + +COPY install_ga.sh ./ +RUN bash -o xtrace -o errexit -o nounset -o pipefail ./install_ga.sh >logs/install_ga.log 2>&1 + +COPY install_petsc.sh ./ +RUN bash -o xtrace -o errexit -o nounset -o pipefail ./install_petsc.sh >logs/install_petsc.log 2>&1 + +RUN rm *.sh diff --git a/build-system/install_boost.sh b/build-system/install_boost.sh new file mode 100644 index 000000000..8ae3190c0 --- /dev/null +++ b/build-system/install_boost.sh @@ -0,0 +1,51 @@ +#! /bin/bash + +# get the parent directory of this script +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + + +boost_version="${BOOST_VERSION:-1.78.0}" + +echo "--- Installing Boost ${boost_version} ---" + +# remove existing +rm -rf boost* + +# download +echo "Downloading Boost" +wget \ + "https://boostorg.jfrog.io/artifactory/main/release/${boost_version}/source/boost_${boost_version//./_}.tar.gz" \ + -O boost.tar.gz \ + --quiet + +# unpack +echo "Unpacking Boost" +tar -xf boost.tar.gz && rm -f boost.tar.gz + +# remove version from dir name +mv "boost_${boost_version//./_}" boost + +pushd boost || exit + +# load mpi module for RHEL +source "$script_dir/install_package_deps_lib.sh" +load_mpi_module + +# bootstrap +echo "Bootstrapping Boost" +./bootstrap.sh \ + --prefix=install_for_gridpack \ + --with-libraries=mpi,serialization,random,filesystem,system +echo 'using mpi ;' >>project-config.jam + +# build +echo "Building Boost" +./b2 -a -d+2 link=shared stage + +# install +echo "Installing Boost" +./b2 -a -d+2 link=shared install + +popd || exit + +echo "Boost installation complete" diff --git a/build-system/install_from_source_deps.sh b/build-system/install_from_source_deps.sh new file mode 100644 index 000000000..f160c0ddb --- /dev/null +++ b/build-system/install_from_source_deps.sh @@ -0,0 +1,23 @@ +#! /bin/bash + +# installs GridPACK dependencies to the current directory + +# bash options: +# - xtrace: print each command before executing it +# - errexit: exit on error +# - nounset: treat unset variables as errors +# - pipefail: treat whole pipeline as errored if any commands within error +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +set -o xtrace -o errexit -o nounset -o pipefail + +# get the parent directory of this script +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +echo "Installing GridPACK dependencies" +date + +bash "${script_dir}/install_boost.sh" +bash "${script_dir}/install_ga.sh" +bash "${script_dir}/install_petsc.sh" + +echo "GridPACK dependency installation complete" diff --git a/build-system/install_ga.sh b/build-system/install_ga.sh new file mode 100644 index 000000000..5b5d1721d --- /dev/null +++ b/build-system/install_ga.sh @@ -0,0 +1,47 @@ +#! /bin/bash + +# get the parent directory of this script +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +ga_version="${GA_VERSION:-5.8}" + +echo "--- Installing Global Arrays ${ga_version} ---" + +# download +echo "Downloading Global Arrays" +wget \ + "https://github.com/GlobalArrays/ga/releases/download/v${ga_version}/ga-${ga_version}.tar.gz" \ + -O ga.tar.gz \ + --quiet + +# unpack +echo "Unpacking Global Arrays" +tar -xf ga.tar.gz && rm -f ga.tar.gz + +# remove version from dir name +mv "ga-${ga_version}" ga + +pushd ga || exit + +# load mpi module for RHEL +source "$script_dir/install_package_deps_lib.sh" +load_mpi_module + +# build +echo "Configuring Global Arrays" +./configure \ + --with-mpi-ts \ + --disable-f77 \ + --without-blas \ + --enable-cxx \ + --enable-i4 \ + --prefix="${PWD}/install_for_gridpack" \ + --enable-shared + +# install +echo "Installing Global Arrays" +make -j "${MAKE_JOBS:-$(nproc)}" install + +popd || exit + +echo "Global Arrays installation complete" diff --git a/build-system/install_gridpack.sh b/build-system/install_gridpack.sh new file mode 100644 index 000000000..5eb106b2f --- /dev/null +++ b/build-system/install_gridpack.sh @@ -0,0 +1,136 @@ +#! /bin/bash + +# installs GridPACK and its python wrapper +# gridPACK is built in src/build and installed to src/install +# run this from the top level GridPACK directory + +# bash options: +# - xtrace: print each command before executing it +# - errexit: exit on error +# - nounset: treat unset variables as errors +# - pipefail: treat whole pipeline as errored if any commands within error +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +set -o xtrace -o errexit -o nounset -o pipefail + +function install_gridpack { + echo "--- GridPACK ---" + + # args + local gridpack_deps_dir=${1:?} + local gridpack_build_dir=${2:?} + local gridpack_install_dir=${3:?} + + # remove existing build and install dir + rm -rf "$gridpack_build_dir" + rm -rf "$gridpack_install_dir" + + # create the build dir + mkdir -p "$gridpack_build_dir" + + pushd "$gridpack_build_dir" || exit + + # todo? git checkout develop + + # remove existing cmake output + rm -rf CMake* + + # load mpi module for RHEL + load_mpi_module + + # generate make files + echo "Generating GridPACK make files" + cmake \ + -D GA_DIR:STRING="${gridpack_deps_dir}/ga/install_for_gridpack" \ + -D BOOST_ROOT:STRING="${gridpack_deps_dir}/boost/install_for_gridpack" \ + -D Boost_DIR:STRING="${gridpack_deps_dir}/boost/install_for_gridpack/lib/cmake/Boost" \ + -D Boost_LIBRARIES:STRING="${gridpack_deps_dir}/boost/install_for_gridpack/lib" \ + -D Boost_INCLUDE_DIRS:STRING="${gridpack_deps_dir}/boost/install_for_gridpack/include" \ + -D PETSC_DIR:PATH="${gridpack_deps_dir}/petsc/install_for_gridpack" \ + -D MPI_CXX_COMPILER:STRING='mpicxx' \ + -D MPI_C_COMPILER:STRING='mpicc' \ + -D MPIEXEC:STRING='mpiexec' \ + -D GRIDPACK_TEST_TIMEOUT:STRING=120 \ + -D CMAKE_INSTALL_PREFIX:PATH="${gridpack_install_dir}" \ + -D CMAKE_BUILD_TYPE:STRING=Debug \ + -D BUILD_SHARED_LIBS=YES \ + -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ + -D Boost_NO_BOOST_CMAKE:BOOL=TRUE \ + -D MPIEXEC_MAX_NUMPROCS:STRING="2" \ + .. + + # install + echo "Installing GridPACK" + make -j "${MAKE_JOBS:-$(nproc)}" install + + popd || exit + + echo "GridPACK installation complete" +} + +function install_gridpack_python { + echo "--- GridPACK python wrapper ---" + + # args + local gridpack_build_dir=${1:?} + local gridpack_install_dir=${2:?} + + # update submodules + echo "Updating GridPACK submodules" + git submodule update --init + + pushd python || exit + + # export RHEL_OPENMPI_HACK for RHEL + distribution=$(get_base_distro) + if test "$distribution" = "rhel"; then + export RHEL_OPENMPI_HACK=yes + fi + + # set python executable path + python_exe=$(which python || which python3) + + # remove existing build dir + rm -rf build + + # export GRIDPACK_DIR + export GRIDPACK_DIR="${gridpack_install_dir}" + + # build + echo "Building GridPACK python wrapper" + ${python_exe} setup.py build + + # remove existing install dir + local py_lib="${gridpack_install_dir}/lib/python" + rm -rf "${py_lib}" + mkdir -p "${py_lib}" + + # add lib to python path + export PYTHONPATH="${py_lib}${PYTHONPATH:+:$PYTHONPATH}" + + # install + echo "Installing GridPACK python wrapper" + ${python_exe} setup.py install --home="$gridpack_install_dir" + + popd || exit + + echo "GridPACK python wrapper installation complete" +} + +# get the parent directory of this script +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +# load the `load_mpi_module`, `get_base_distro` functions +source "$script_dir/install_package_deps_lib.sh" + +echo "Installing GridPACK" +date + +build_dir=${PWD}/src/build +install_dir=${PWD}/src/install + +gp_ext_deps=${GP_EXT_DEPS:-/gridpack-dependencies} + +install_gridpack "$gp_ext_deps" "$build_dir" "$install_dir" +install_gridpack_python "$build_dir" "$install_dir" + +echo "Completed GridPACK installation" diff --git a/build-system/install_package_deps_lib.sh b/build-system/install_package_deps_lib.sh new file mode 100644 index 000000000..29d3e02f9 --- /dev/null +++ b/build-system/install_package_deps_lib.sh @@ -0,0 +1,92 @@ +#! /bin/bash + +# library to install packages required to build GridPACK on debian or RHEL based distros +# meant to be sourced + +# load the mpi module on RHEL distributions +function load_mpi_module { + local base_distro + base_distro=$(get_base_distro) + + if ! test "$base_distro" = "rhel"; then + return + fi + + echo "Loading mpi module" + + # clear unbound variable check temporarily + option_state=$([[ $- == *u* ]] && echo "-u" || echo "+u") + set +u + + # source the script which contains the module command + # shellcheck disable=SC1091 + source /etc/profile.d/modules.sh + + # reset the unbound variable check setting + set "$option_state" + + # load the mpi module + module load "mpi/openmpi-$(arch)" +} + +# detect if this is a "debian" or "rhel" based linux distribution +# usage: +# if test "$(get_base_distro)" = "debian"; then +# apt-get update +# apt-get upgrade --yes +# fi +function get_base_distro { + distribution=$( + # shellcheck disable=SC1091 + source /etc/os-release + echo "$ID" + ) + + case $distribution in + debian | ubuntu) + echo "debian" + ;; + fedora | rhel | centos | rocky) + echo "rhel" + ;; + *) + echo "unknown" + ;; + esac +} + +# install packages for RHEL +function install_rhel_packages { + dnf upgrade --assumeyes --verbose + + # use epel and crb repos + # https://wiki.rockylinux.org/rocky/repo/#notes-on-epel + dnf install epel-release --assumeyes + crb enable + + # install required packages + dnf install --assumeyes \ + wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ + python3-mpi4py-openmpi +} + +# install packages for debian +function install_debian_packages { + apt-get update + apt-get upgrade --yes + + # install required packages + apt-get install --yes \ + wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py +} + +function install_packages { + local base_distro + base_distro=$(get_base_distro) + + if test "$base_distro" = "debian"; then + install_debian_packages + elif test "$base_distro" = "rhel"; then + install_rhel_packages + fi +} diff --git a/build-system/install_petsc.sh b/build-system/install_petsc.sh new file mode 100644 index 000000000..4edddeba2 --- /dev/null +++ b/build-system/install_petsc.sh @@ -0,0 +1,52 @@ +#! /bin/bash + +# get the parent directory of this script +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +petsc_version="${PETSC_VERSION:-3.16.4}" + +echo "--- Installing PETSc ${petsc_version} ---" + +# clone +echo "Cloning PETSc repository" +git clone https://gitlab.com/petsc/petsc.git + +pushd petsc || exit + +git checkout "tags/v${petsc_version}" -b "v${petsc_version}" + +export PETSC_DIR=${PWD} +export PETSC_ARCH=build-dir + +# load mpi module for RHEL +source "$script_dir/install_package_deps_lib.sh" +load_mpi_module + +# install +echo "Configuring PETSc" +./configure \ + --download-mumps \ + --download-scalapack \ + --download-metis \ + --download-parmetis \ + --download-suitesparse \ + --download-f2cblaslapack \ + --prefix="${PWD}"/install_for_gridpack \ + --scalar-type=complex \ + --with-shared-libraries=1 + +# build +echo "Building PETSc" +make + +# install +echo "Installing PETSc" +make install + +# check +echo "Checking PETSc" +make check + +popd || exit + +echo "PETSc installation complete" diff --git a/install_gridpack.sh b/install_gridpack.sh deleted file mode 100755 index 933636dae..000000000 --- a/install_gridpack.sh +++ /dev/null @@ -1,103 +0,0 @@ -# This script installs GridPACK and python wrappter.GridPACK is built in src/build directory and installed in src/install directory. - -# This script should be run from the top-level GridPACK directory. - -# Flag for install GridPACK and GridPACK python wrapper -install_gridpack=true -install_gridpack_python=true - -# Set your python executable here -python_exe=`which python` -if test -z ${python_exe} -then - python_exe=`which python3` -fi - -if test -z ${GRIDPACK_ROOT_DIR} -then - export GRIDPACK_ROOT_DIR=${PWD} - echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" -fi - -# Create directory for installing external packages -if test -z ${GP_EXT_DEPS} -then - export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies -fi - -cd ${GRIDPACK_ROOT_DIR} - -# Set environment variable GRIDPACK_BUILD_DIR and create a build directory -export GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build - -GRIDPACK_INSTALL_DIR=${GRIDPACK_ROOT_DIR}/src/install -export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} - -if ${install_gridpack} -then - - rm -rf $GRIDPACK_BUILD_DIR - mkdir $GRIDPACK_BUILD_DIR - - rm -rf ${GRIDPACK_INSTALL_DIR} - - cd ${GRIDPACK_BUILD_DIR} - - ## GridPACK installation - echo "Building GridPACK" - -# git checkout develop - - rm -rf CMake* - - cmake_args="-D GA_DIR:STRING=${GP_EXT_DEPS}/ga-5.8/install_for_gridpack \ - -D BOOST_ROOT:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack \ - -D Boost_DIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib/cmake/Boost-1.78.0 \ - -D Boost_LIBRARIES:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib \ - -D Boost_INCLUDE_DIRS:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/include \ - -D PETSC_DIR:PATH=${GP_EXT_DEPS}/petsc/install_for_gridpack \ - -D MPI_CXX_COMPILER:STRING='mpicxx' \ - -D MPI_C_COMPILER:STRING='mpicc' \ - -D MPIEXEC:STRING='mpiexec' \ - -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ - -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_INSTALL_DIR} \ - -D CMAKE_BUILD_TYPE:STRING=Debug \ - -D BUILD_SHARED_LIBS=YES \ - -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ - .. " - - cmake ${cmake_args} - - echo "Installing GridPACK develop branch" - make -j 10 install -fi - -if ${install_gridpack_python} -then - echo "Installing GridPACK python wrapper" - - cd ${GRIDPACK_ROOT_DIR} - - git submodule update --init - - echo ${GRIDPACK_DIR} - cd python - - export RHEL_OPENMPI_HACK=yes - - rm -rf build - - ${python_exe} setup.py build - - rm -rf ${GRIDPACK_INSTALL_DIR}/lib/python - mkdir ${GRIDPACK_INSTALL_DIR}/lib/python - - PYTHONPATH="${GRIDPACK_DIR}/lib/python:${PYTHONPATH}" - export PYTHONPATH - ${python_exe} setup.py install --home="$GRIDPACK_DIR" - -fi - -cd ${GRIDPACK_ROOT_DIR} - -echo "Completed GridPACK installation" diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh deleted file mode 100755 index 87897593e..000000000 --- a/install_gridpack_deps.sh +++ /dev/null @@ -1,133 +0,0 @@ -# This script installs all GridPACK dependencies.The dependencies are installed in external-dependencies directory. - -# This script should be run from the top-level GridPACK directory. - -# Flags for installing/not installing different dependency packages -install_boost=true -install_ga=true -install_petsc=true - -echo $(date) - -if test -z ${GRIDPACK_ROOT_DIR} -then - export GRIDPACK_ROOT_DIR=${PWD} - echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" -fi - -cd ${GRIDPACK_ROOT_DIR} - -# Create directory for installing external packages -if test -z ${GP_EXT_DEPS} -then - export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies - rm -rf ${GP_EXT_DEPS} - mkdir ${GP_EXT_DEPS} - echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" -else - if test -d ${GP_EXT_DEPS} - then - echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" - else - mkdir ${GP_EXT_DEPS} - fi -fi - -cd ${GP_EXT_DEPS} - -if ${install_boost} -then - - rm -rf boost* - - # Download and install Boost - echo "Downloading Boost-1.78.0" - - # Download Boost - wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz - - # Untar - tar -xf boost_1_78_0.tar.gz - - cd boost_1_78_0 - - # Install boost - echo "Building Boost-1.78.0" - - ./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem,system - - echo 'using mpi ;' >> project-config.jam - - ./b2 -a -d+2 link=shared stage - - echo "Installing Boost-1.78.0" - ./b2 -a -d+2 link=shared install - - echo "Building and Installing Boost libraries complete" -fi - -if ${install_ga} -then - # Download, build, and install GA - cd ${GP_EXT_DEPS} - - echo "Downloading GA-5.8" - - wget https://github.com/GlobalArrays/ga/releases/download/v5.8/ga-5.8.tar.gz - - tar -xf ga-5.8.tar.gz - - cd ga-5.8 - - # Build GA - echo "Building GA-5.8" - ./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix=${PWD}/install_for_gridpack --enable-shared - - # Install GA - echo "Installing GA-5.8" - make -j 10 install - - echo "GA-5.8 installation complete" -fi - -if ${install_petsc} -then - - # Install PETSc 3.16.4 - cd ${GP_EXT_DEPS} - - # Download - echo "Downloading PETSc 3.16.4" - - git clone https://gitlab.com/petsc/petsc.git - - cd petsc - - git checkout tags/v3.16.4 -b v3.16.4 - - export PETSC_DIR=${PWD} - export PETSC_ARCH=build-dir - - # Install PETSc - echo "Installing PETSc 3.16.4" - - ./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix=${PWD}/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 - - # Build PETSc - echo "Building PETSc 3.16.4" - - make - - # Install PETSc - echo "Installing PETSc 3.16.4" - - make install - make check -fi - -# update LD_LIBRARY_PATH so that boost,ga, and petsc are on it -export LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-5.8/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - -cd ${GRIDPACK_ROOT_DIR} - -echo "Completed installing GridPACK dependencies in ${GP_EXT_DEPS}"