From bb9bd551aedbb4910604c946d183596c44c7f184 Mon Sep 17 00:00:00 2001 From: Shri Abhyankar Date: Sat, 31 Dec 2022 16:44:44 -0600 Subject: [PATCH 001/195] Added script to build GridPACK in one-go, including installing all dependencies. --- install_gridpack.sh | 123 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100755 install_gridpack.sh diff --git a/install_gridpack.sh b/install_gridpack.sh new file mode 100755 index 000000000..ffeee710b --- /dev/null +++ b/install_gridpack.sh @@ -0,0 +1,123 @@ +# This script installs GridPACK and all its dependencies.The dependencies are installed in external-dependencies directory. GridPACK is built in src/build directory and installed in src/install directory. + +# This script should be run from the top-level GridPACK directory. + +# Set environment variable GRIDPACK_DIR +export GRIDPACK_DIR=${PWD} + +# Create directory for installing external packages +rm -rf external-dependencies +mkdir external-dependencies + +cd external-dependencies + +export GP_EXT_DEPS=${PWD} + +# Download and install Boost +echo "Downloading Boost-1.78.0" + +# Download Boost +wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz + +# Untar +tar -xvf boost_1_78_0.tar.gz + +cd boost_1_78_0 + +# Install boost +echo "Building Boost-1.78.0" + +./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem + +cat project-config.jam >> `using mpi ;` + +./b2 -a -d+2 link=shared stage + +echo "Installing Boost-1.78.0" +./b2 -a -d+2 link=shared install + +echo "Building and Installing Boost libraries complete" + +# Download, build, and install GA +cd .. + +echo "Downloading GA-5.8" + +wget https://github.com/GlobalArrays/ga/releases/download/v5.8/ga-5.8.tar.gz + +tar -xvf ga-5.8.tar.gz + +cd ga-5.8 + +# Build GA +echo "Building GA-5.8" +./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix=${PWD}/install_for_gridpack --enable-shared + +# Install GA +echo "Installing GA-5.8" +make -j 10 install + +echo "GA-5.8 installation complete" + + +# Install PETSc 3.16.4 +cd .. + +# Download +echo "Downloading PETSc 3.16.4" + +git clone https://gitlab.com/petsc/petsc.githttps://gitlab.com/petsc/petsc.git + +git checkout v3.16.4 + +cd petsc + +export PETSC_DIR=${PWD} +export PETSC_ARCH=build-dir + +# Install PETSc +echo "Installing PETSc 3.16.4" + +./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix=${PWD}/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 --download-cmake=1 + +# Build PETSc +echo "Building PETSc 3.16.4" + +make + +# Install PETSc +echo "Installing PETSc 3.16.4" + +make install +make check + +## GridPACK installation +echo "Building GridPACK develop branch" + +git checkout develop + +cd ${GRIDPACK_DIR}/src + +rm -rf build +mkdir build + +cd build + +rm -rf CMake* +cmake \ + -D GA_DIR:STRING=${GP_EXT_DEPS}/ga-5.8/install_for_gridpack \ + -D BOOST_ROOT:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack \ + -D Boost_DIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib/cmake/Boost-1.78.0 \ + -D BOOST_LIBRARYDIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/build/lib \ + -D PETSC_DIR:PATH=${GP_EXT_DEPS}/petsc/install_for_gridpack \ + -D MPI_CXX_COMPILER:STRING='mpicxx' \ + -D MPI_C_COMPILER:STRING='mpicc' \ + -D MPIEXEC:STRING='mpiexec' \ + -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ + -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_DIR}/src/install \ + -D CMAKE_BUILD_TYPE:STRING=Debug \ + -D BUILD_SHARED_LIBS=YES \ + .. + +echo "Installing GridPACK develop branch" +make -j 10 install From eb24be962688f233c74eb6b24196e961e6bd279d Mon Sep 17 00:00:00 2001 From: Shri Date: Thu, 16 Mar 2023 13:41:13 -0700 Subject: [PATCH 002/195] Updated installation script tested on EIOC-Ubuntu8 --- install_gridpack.sh | 211 +++++++++++++++++++++++++------------- src/lib/GridPACK.cmake.in | 8 +- 2 files changed, 144 insertions(+), 75 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index ffeee710b..9cc6ca4e8 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -1,123 +1,192 @@ -# This script installs GridPACK and all its dependencies.The dependencies are installed in external-dependencies directory. GridPACK is built in src/build directory and installed in src/install directory. +# This script installs GridPACK and all its dependencies.The dependencies are installed in src/build/external-dependencies directory. GridPACK is built in src/build directory and installed in src/install directory. # This script should be run from the top-level GridPACK directory. -# Set environment variable GRIDPACK_DIR -export GRIDPACK_DIR=${PWD} +# Flags for installing/not installing different dependency packages +install_boost=true +install_ga=true +install_petsc=true -# Create directory for installing external packages -rm -rf external-dependencies -mkdir external-dependencies +# Flag for install GridPACK and GridPACK python wrapper +install_gridpack=true +install_gridpack_python=true -cd external-dependencies +# Set your python executable here +python_exe=`which python3` -export GP_EXT_DEPS=${PWD} +# Install log file +install_log_file=${PWD}/install.log -# Download and install Boost -echo "Downloading Boost-1.78.0" +echo 'GRIDPACK installation log file' > ${install_log_file} +echo $(date) >> install_log_file -# Download Boost -wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz +export GRIDPACK_ROOT_DIR=${PWD} +# Set environment variable GRIDPACK_BUILD_DIR and create a build directory +export GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build -# Untar -tar -xvf boost_1_78_0.tar.gz +rm -rf $GRIDPACK_BUILD_DIR +mkdir $GRIDPACK_BUILD_DIR +cd $GRIDPACK_BUILD_DIR -cd boost_1_78_0 +# Create directory for installing external packages +mkdir external-dependencies -# Install boost -echo "Building Boost-1.78.0" +export GP_EXT_DEPS=${GRIDPACK_BUILD_DIR}/external-dependencies -./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem +cd external-dependencies + +if ${install_boost} +then -cat project-config.jam >> `using mpi ;` + rm -rf boost* + + # Download and install Boost + echo "Downloading Boost-1.78.0" -./b2 -a -d+2 link=shared stage + # Download Boost + wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz &>> ${install_log_file} -echo "Installing Boost-1.78.0" -./b2 -a -d+2 link=shared install + # Untar + tar -xf boost_1_78_0.tar.gz -echo "Building and Installing Boost libraries complete" + cd boost_1_78_0 -# Download, build, and install GA -cd .. + # Install boost + echo "Building Boost-1.78.0" -echo "Downloading GA-5.8" + ./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem,system &>> ${install_log_file} -wget https://github.com/GlobalArrays/ga/releases/download/v5.8/ga-5.8.tar.gz + echo 'using mpi ;' >> project-config.jam -tar -xvf ga-5.8.tar.gz + ./b2 -a -d+2 link=shared stage &>> ${install_log_file} -cd ga-5.8 + echo "Installing Boost-1.78.0" + ./b2 -a -d+2 link=shared install &>> ${install_log_file} -# Build GA -echo "Building GA-5.8" -./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix=${PWD}/install_for_gridpack --enable-shared + echo "Building and Installing Boost libraries complete" +fi -# Install GA -echo "Installing GA-5.8" -make -j 10 install +if ${install_ga} +then + # Download, build, and install GA + cd ${GP_EXT_DEPS} -echo "GA-5.8 installation complete" + echo "Downloading GA-5.8" + wget https://github.com/GlobalArrays/ga/releases/download/v5.8/ga-5.8.tar.gz &>> ${install_log_file} -# Install PETSc 3.16.4 -cd .. + tar -xf ga-5.8.tar.gz &>> ${install_log_file} -# Download -echo "Downloading PETSc 3.16.4" + cd ga-5.8 -git clone https://gitlab.com/petsc/petsc.githttps://gitlab.com/petsc/petsc.git + # Build GA + echo "Building GA-5.8" + ./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix=${PWD}/install_for_gridpack --enable-shared &>> ${install_log_file} + + # Install GA + echo "Installing GA-5.8" + make -j 10 install &>> ${install_log_file} + + echo "GA-5.8 installation complete" +fi -git checkout v3.16.4 +if ${install_petsc} +then + + # Install PETSc 3.16.4 + cd ${GP_EXT_DEPS} -cd petsc + # Download + echo "Downloading PETSc 3.16.4" -export PETSC_DIR=${PWD} -export PETSC_ARCH=build-dir + git clone https://gitlab.com/petsc/petsc.git &>> ${install_log_file} + + cd petsc -# Install PETSc -echo "Installing PETSc 3.16.4" + git checkout tags/v3.16.4 -b v3.16.4 &>> ${install_log_file} -./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix=${PWD}/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 --download-cmake=1 + export PETSC_DIR=${PWD} + export PETSC_ARCH=build-dir -# Build PETSc -echo "Building PETSc 3.16.4" + # Install PETSc + echo "Installing PETSc 3.16.4" + + ./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix=${PWD}/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 &>> ${install_log_file} -make + # Build PETSc + echo "Building PETSc 3.16.4" -# Install PETSc -echo "Installing PETSc 3.16.4" + make &>> ${install_log_file} -make install -make check + # Install PETSc + echo "Installing PETSc 3.16.4" -## GridPACK installation -echo "Building GridPACK develop branch" + make install &>> ${install_log_file} + make check &>> ${install_log_file} +fi -git checkout develop +GRIDPACK_INSTALL_DIR=${GRIDPACK_ROOT_DIR}/src/install -cd ${GRIDPACK_DIR}/src +if ${install_gridpack} +then -rm -rf build -mkdir build + cd ${GRIDPACK_BUILD_DIR} + + ## GridPACK installation + echo "Building GridPACK develop branch" -cd build + git checkout develop -rm -rf CMake* -cmake \ - -D GA_DIR:STRING=${GP_EXT_DEPS}/ga-5.8/install_for_gridpack \ + rm -rf CMake* + + cmake_args="-D GA_DIR:STRING=${GP_EXT_DEPS}/ga-5.8/install_for_gridpack \ -D BOOST_ROOT:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack \ -D Boost_DIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib/cmake/Boost-1.78.0 \ - -D BOOST_LIBRARYDIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/build/lib \ + -D BOOST_LIBRARIES:STRING=${GP_EXT_DEPS}/boost_1_78_0/build/lib \ -D PETSC_DIR:PATH=${GP_EXT_DEPS}/petsc/install_for_gridpack \ -D MPI_CXX_COMPILER:STRING='mpicxx' \ -D MPI_C_COMPILER:STRING='mpicc' \ -D MPIEXEC:STRING='mpiexec' \ -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ - -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_DIR}/src/install \ + -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_INSTALL_DIR} \ -D CMAKE_BUILD_TYPE:STRING=Debug \ -D BUILD_SHARED_LIBS=YES \ - .. - -echo "Installing GridPACK develop branch" -make -j 10 install + -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ + .. " + + cmake ${cmake_args} + + echo "Installing GridPACK develop branch" + make -j 10 install +fi + +if ${install_gridpack_python} +then + echo "Installing GridPACK python wrapper" + + cd ${GRIDPACK_ROOT_DIR} + + git submodule update --init + + export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} + echo ${GRIDPACK_DIR} + cd python + + export RHEL_OPENMPI_HACK=yes + + ${python_exe} setup.py build + + rm -rf ${GRIDPACK_INSTALL_DIR}/lib/python + mkdir ${GRIDPACK_INSTALL_DIR}/lib/python + + PYTHONPATH="${GRIDPACK_DIR}/lib/python:${PYTHONPATH}" + export PYTHONPATH + ${python_exe} setup.py install --home="$GRIDPACK_DIR" + +fi + +# update LD_LIBRARY_PATH so that boost,ga, and petsc are on it +export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-5.8/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib +cd ${GRIDPACK_ROOT_DIR} + +echo "Completed GridPACK installation" diff --git a/src/lib/GridPACK.cmake.in b/src/lib/GridPACK.cmake.in index d351cd3f9..1e2516125 100644 --- a/src/lib/GridPACK.cmake.in +++ b/src/lib/GridPACK.cmake.in @@ -233,10 +233,10 @@ function(gridpack_setup) list(APPEND gp_libs @GLPK_LIBRARY@) endif() - list(APPEND gp_include - @Boost_INCLUDE_DIRS@ - @MPI_INCLUDE_PATH@ - ) +# list(APPEND gp_include +# @Boost_INCLUDE_DIR@ +# @MPI_INCLUDE_PATH@ +# ) list(APPEND gp_libs @Boost_LIBRARIES@ From f29a18fc9d8e59be65f6b56f6c11d51cf1eb711e Mon Sep 17 00:00:00 2001 From: Shri Date: Thu, 16 Mar 2023 20:44:38 -0700 Subject: [PATCH 003/195] Minor modifications to install script --- install_gridpack.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 9cc6ca4e8..d4cd3482c 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -12,7 +12,11 @@ install_gridpack=true install_gridpack_python=true # Set your python executable here -python_exe=`which python3` +python_exe=`which python` +if[ -z ${python_exe}] +then + python_exe=`which python3` +fi # Install log file install_log_file=${PWD}/install.log @@ -126,6 +130,7 @@ then fi GRIDPACK_INSTALL_DIR=${GRIDPACK_ROOT_DIR}/src/install +export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} if ${install_gridpack} then @@ -168,7 +173,6 @@ then git submodule update --init - export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} echo ${GRIDPACK_DIR} cd python From 42d480756d8d8b370bfbc2f322e40429e6aadcde Mon Sep 17 00:00:00 2001 From: Shri Abhyankar Date: Sat, 18 Mar 2023 10:16:34 -0700 Subject: [PATCH 004/195] Small modification to check python --- install_gridpack.sh | 2 +- src/ga | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index d4cd3482c..62665fa83 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -13,7 +13,7 @@ install_gridpack_python=true # Set your python executable here python_exe=`which python` -if[ -z ${python_exe}] +if test -z ${python_exe} then python_exe=`which python3` fi diff --git a/src/ga b/src/ga index cd638ca3b..fcdde0274 160000 --- a/src/ga +++ b/src/ga @@ -1 +1 @@ -Subproject commit cd638ca3b431566b29e55813095ae16a2da0bc47 +Subproject commit fcdde027422c256f42b852cd3d937242103da49d From 0e6b2cee6e7d05e879860129873d7be8c84c257c Mon Sep 17 00:00:00 2001 From: Shri Date: Fri, 7 Apr 2023 14:32:05 -0700 Subject: [PATCH 005/195] Updated gridpack install script to only install GridPACK and python wrapper. --- install_gridpack.sh | 126 +++++--------------------------------------- src/ga | 2 +- 2 files changed, 14 insertions(+), 114 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 62665fa83..e0f6fc42e 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -1,12 +1,7 @@ -# This script installs GridPACK and all its dependencies.The dependencies are installed in src/build/external-dependencies directory. GridPACK is built in src/build directory and installed in src/install directory. +# This script installs GridPACK and python wrappter.GridPACK is built in src/build directory and installed in src/install directory. # This script should be run from the top-level GridPACK directory. -# Flags for installing/not installing different dependency packages -install_boost=true -install_ga=true -install_petsc=true - # Flag for install GridPACK and GridPACK python wrapper install_gridpack=true install_gridpack_python=true @@ -18,116 +13,22 @@ then python_exe=`which python3` fi -# Install log file -install_log_file=${PWD}/install.log - -echo 'GRIDPACK installation log file' > ${install_log_file} -echo $(date) >> install_log_file - -export GRIDPACK_ROOT_DIR=${PWD} -# Set environment variable GRIDPACK_BUILD_DIR and create a build directory -export GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build - -rm -rf $GRIDPACK_BUILD_DIR -mkdir $GRIDPACK_BUILD_DIR -cd $GRIDPACK_BUILD_DIR - -# Create directory for installing external packages -mkdir external-dependencies - -export GP_EXT_DEPS=${GRIDPACK_BUILD_DIR}/external-dependencies - -cd external-dependencies - -if ${install_boost} +if test -z ${GRIDPACK_ROOT_DIR} then - - rm -rf boost* - - # Download and install Boost - echo "Downloading Boost-1.78.0" - - # Download Boost - wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz &>> ${install_log_file} - - # Untar - tar -xf boost_1_78_0.tar.gz - - cd boost_1_78_0 - - # Install boost - echo "Building Boost-1.78.0" - - ./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem,system &>> ${install_log_file} - - echo 'using mpi ;' >> project-config.jam - - ./b2 -a -d+2 link=shared stage &>> ${install_log_file} - - echo "Installing Boost-1.78.0" - ./b2 -a -d+2 link=shared install &>> ${install_log_file} - - echo "Building and Installing Boost libraries complete" + export GRIDPACK_ROOT_DIR=${PWD} + echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" fi -if ${install_ga} +# Create directory for installing external packages +if test -z ${GP_EXT_DEPS} then - # Download, build, and install GA - cd ${GP_EXT_DEPS} - - echo "Downloading GA-5.8" - - wget https://github.com/GlobalArrays/ga/releases/download/v5.8/ga-5.8.tar.gz &>> ${install_log_file} - - tar -xf ga-5.8.tar.gz &>> ${install_log_file} - - cd ga-5.8 - - # Build GA - echo "Building GA-5.8" - ./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix=${PWD}/install_for_gridpack --enable-shared &>> ${install_log_file} - - # Install GA - echo "Installing GA-5.8" - make -j 10 install &>> ${install_log_file} - - echo "GA-5.8 installation complete" + export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies fi -if ${install_petsc} -then - - # Install PETSc 3.16.4 - cd ${GP_EXT_DEPS} - - # Download - echo "Downloading PETSc 3.16.4" - - git clone https://gitlab.com/petsc/petsc.git &>> ${install_log_file} - - cd petsc - - git checkout tags/v3.16.4 -b v3.16.4 &>> ${install_log_file} - - export PETSC_DIR=${PWD} - export PETSC_ARCH=build-dir - - # Install PETSc - echo "Installing PETSc 3.16.4" - - ./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix=${PWD}/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 &>> ${install_log_file} - - # Build PETSc - echo "Building PETSc 3.16.4" - - make &>> ${install_log_file} - - # Install PETSc - echo "Installing PETSc 3.16.4" +cd ${GRIDPACK_ROOT_DIR} - make install &>> ${install_log_file} - make check &>> ${install_log_file} -fi +# Set environment variable GRIDPACK_BUILD_DIR and create a build directory +export GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build GRIDPACK_INSTALL_DIR=${GRIDPACK_ROOT_DIR}/src/install export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} @@ -135,6 +36,9 @@ export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} if ${install_gridpack} then + rm -rf $GRIDPACK_BUILD_DIR + mkdir $GRIDPACK_BUILD_DIR + cd ${GRIDPACK_BUILD_DIR} ## GridPACK installation @@ -189,8 +93,4 @@ then fi -# update LD_LIBRARY_PATH so that boost,ga, and petsc are on it -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-5.8/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib -cd ${GRIDPACK_ROOT_DIR} - echo "Completed GridPACK installation" diff --git a/src/ga b/src/ga index fcdde0274..cd638ca3b 160000 --- a/src/ga +++ b/src/ga @@ -1 +1 @@ -Subproject commit fcdde027422c256f42b852cd3d937242103da49d +Subproject commit cd638ca3b431566b29e55813095ae16a2da0bc47 From 2d9d2f18ed9a73e392bfe5de4f858a8147d32b86 Mon Sep 17 00:00:00 2001 From: Shri Date: Fri, 7 Apr 2023 15:37:21 -0700 Subject: [PATCH 006/195] More work on installation script - Split the script into two (one for installing dependencies and other for GridPACK) - Ignore boost includes and boost libraries in GridPACK.cmake.in. Otherwise python installation fails. - Update GridPACK installation script with addition CMake flags for Boost. --- install_gridpack.sh | 15 +++-- install_gridpack_deps.sh | 126 ++++++++++++++++++++++++++++++++++++++ src/lib/GridPACK.cmake.in | 8 +-- 3 files changed, 141 insertions(+), 8 deletions(-) create mode 100755 install_gridpack_deps.sh diff --git a/install_gridpack.sh b/install_gridpack.sh index e0f6fc42e..933636dae 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -39,19 +39,22 @@ then rm -rf $GRIDPACK_BUILD_DIR mkdir $GRIDPACK_BUILD_DIR + rm -rf ${GRIDPACK_INSTALL_DIR} + cd ${GRIDPACK_BUILD_DIR} ## GridPACK installation - echo "Building GridPACK develop branch" + echo "Building GridPACK" - git checkout develop +# git checkout develop rm -rf CMake* cmake_args="-D GA_DIR:STRING=${GP_EXT_DEPS}/ga-5.8/install_for_gridpack \ -D BOOST_ROOT:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack \ - -D Boost_DIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib/cmake/Boost-1.78.0 \ - -D BOOST_LIBRARIES:STRING=${GP_EXT_DEPS}/boost_1_78_0/build/lib \ + -D Boost_DIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib/cmake/Boost-1.78.0 \ + -D Boost_LIBRARIES:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib \ + -D Boost_INCLUDE_DIRS:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/include \ -D PETSC_DIR:PATH=${GP_EXT_DEPS}/petsc/install_for_gridpack \ -D MPI_CXX_COMPILER:STRING='mpicxx' \ -D MPI_C_COMPILER:STRING='mpicc' \ @@ -82,6 +85,8 @@ then export RHEL_OPENMPI_HACK=yes + rm -rf build + ${python_exe} setup.py build rm -rf ${GRIDPACK_INSTALL_DIR}/lib/python @@ -93,4 +98,6 @@ then fi +cd ${GRIDPACK_ROOT_DIR} + echo "Completed GridPACK installation" diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh new file mode 100755 index 000000000..e6e054e69 --- /dev/null +++ b/install_gridpack_deps.sh @@ -0,0 +1,126 @@ +# This script installs all GridPACK dependencies.The dependencies are installed in external-dependencies directory. + +# This script should be run from the top-level GridPACK directory. + +# Flags for installing/not installing different dependency packages +install_boost=true +install_ga=true +install_petsc=true + +echo $(date) + +if test -z ${GRIDPACK_ROOT_DIR} +then + export GRIDPACK_ROOT_DIR=${PWD} + echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" +fi + +cd ${GRIDPACK_ROOT_DIR} + +# Create directory for installing external packages +if test -z ${GP_EXT_DEPS} +then + export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies + rm -rf ${GP_EXT_DEPS} + mkdir ${GP_EXT_DEPS} + echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" +fi + +cd ${GP_EXT_DEPS} + +if ${install_boost} +then + + rm -rf boost* + + # Download and install Boost + echo "Downloading Boost-1.78.0" + + # Download Boost + wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz + + # Untar + tar -xf boost_1_78_0.tar.gz + + cd boost_1_78_0 + + # Install boost + echo "Building Boost-1.78.0" + + ./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem,system + + echo 'using mpi ;' >> project-config.jam + + ./b2 -a -d+2 link=shared stage + + echo "Installing Boost-1.78.0" + ./b2 -a -d+2 link=shared install + + echo "Building and Installing Boost libraries complete" +fi + +if ${install_ga} +then + # Download, build, and install GA + cd ${GP_EXT_DEPS} + + echo "Downloading GA-5.8" + + wget https://github.com/GlobalArrays/ga/releases/download/v5.8/ga-5.8.tar.gz + + tar -xf ga-5.8.tar.gz + + cd ga-5.8 + + # Build GA + echo "Building GA-5.8" + ./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix=${PWD}/install_for_gridpack --enable-shared + + # Install GA + echo "Installing GA-5.8" + make -j 10 install + + echo "GA-5.8 installation complete" +fi + +if ${install_petsc} +then + + # Install PETSc 3.16.4 + cd ${GP_EXT_DEPS} + + # Download + echo "Downloading PETSc 3.16.4" + + git clone https://gitlab.com/petsc/petsc.git + + cd petsc + + git checkout tags/v3.16.4 -b v3.16.4 + + export PETSC_DIR=${PWD} + export PETSC_ARCH=build-dir + + # Install PETSc + echo "Installing PETSc 3.16.4" + + ./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix=${PWD}/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 + + # Build PETSc + echo "Building PETSc 3.16.4" + + make + + # Install PETSc + echo "Installing PETSc 3.16.4" + + make install + make check +fi + +# update LD_LIBRARY_PATH so that boost,ga, and petsc are on it +export LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-5.8/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} + +cd ${GRIDPACK_ROOT_DIR} + +echo "Completed installing GridPACK dependencies in ${GP_EXT_DEPS}" diff --git a/src/lib/GridPACK.cmake.in b/src/lib/GridPACK.cmake.in index 1e2516125..eaa59dc56 100644 --- a/src/lib/GridPACK.cmake.in +++ b/src/lib/GridPACK.cmake.in @@ -238,10 +238,10 @@ function(gridpack_setup) # @MPI_INCLUDE_PATH@ # ) - list(APPEND gp_libs - @Boost_LIBRARIES@ - @MPI_CXX_LIBRARIES@ - ) +# list(APPEND gp_libs +# @Boost_LIBRARIES@ +# @MPI_CXX_LIBRARIES@ +# ) set(GRIDPACK_INCLUDE_DIRS ${gp_include} From 177881703a06763a8b711345d5b587b8546e1612 Mon Sep 17 00:00:00 2001 From: Shri Date: Fri, 7 Apr 2023 16:27:07 -0700 Subject: [PATCH 007/195] Create external-dependencies directory, if not present. --- install_gridpack_deps.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index e6e054e69..87897593e 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -23,7 +23,14 @@ then export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies rm -rf ${GP_EXT_DEPS} mkdir ${GP_EXT_DEPS} - echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" + echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" +else + if test -d ${GP_EXT_DEPS} + then + echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" + else + mkdir ${GP_EXT_DEPS} + fi fi cd ${GP_EXT_DEPS} From 37c8dd77b4e0aa3c344b575f45fba2deb9e60c30 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 5 Sep 2023 21:44:01 -0500 Subject: [PATCH 008/195] Apply shellcheck suggestions and run formatter --- install_gridpack.sh | 111 +++++++++++++++++++-------------------- install_gridpack_deps.sh | 82 ++++++++++++++--------------- 2 files changed, 93 insertions(+), 100 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 933636dae..523adef1c 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -1,3 +1,5 @@ +#! /bin/bash + # This script installs GridPACK and python wrappter.GridPACK is built in src/build directory and installed in src/install directory. # This script should be run from the top-level GridPACK directory. @@ -7,25 +9,22 @@ install_gridpack=true install_gridpack_python=true # Set your python executable here -python_exe=`which python` -if test -z ${python_exe} -then - python_exe=`which python3` +python_exe=$(which python) +if test -z "${python_exe}"; then + python_exe=$(which python3) fi -if test -z ${GRIDPACK_ROOT_DIR} -then - export GRIDPACK_ROOT_DIR=${PWD} - echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" +if test -z "${GRIDPACK_ROOT_DIR}"; then + export GRIDPACK_ROOT_DIR=${PWD} + echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" fi # Create directory for installing external packages -if test -z ${GP_EXT_DEPS} -then - export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies +if test -z "${GP_EXT_DEPS}"; then + export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies fi -cd ${GRIDPACK_ROOT_DIR} +cd "${GRIDPACK_ROOT_DIR}" || exit # Set environment variable GRIDPACK_BUILD_DIR and create a build directory export GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build @@ -33,71 +32,69 @@ export GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build GRIDPACK_INSTALL_DIR=${GRIDPACK_ROOT_DIR}/src/install export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} -if ${install_gridpack} -then +if ${install_gridpack}; then + + rm -rf "$GRIDPACK_BUILD_DIR" + mkdir "$GRIDPACK_BUILD_DIR" - rm -rf $GRIDPACK_BUILD_DIR - mkdir $GRIDPACK_BUILD_DIR + rm -rf "${GRIDPACK_INSTALL_DIR}" - rm -rf ${GRIDPACK_INSTALL_DIR} + cd "${GRIDPACK_BUILD_DIR}" || exit - cd ${GRIDPACK_BUILD_DIR} - ## GridPACK installation echo "Building GridPACK" -# git checkout develop + # git checkout develop rm -rf CMake* cmake_args="-D GA_DIR:STRING=${GP_EXT_DEPS}/ga-5.8/install_for_gridpack \ - -D BOOST_ROOT:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack \ + -D BOOST_ROOT:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack \ -D Boost_DIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib/cmake/Boost-1.78.0 \ - -D Boost_LIBRARIES:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib \ + -D Boost_LIBRARIES:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib \ -D Boost_INCLUDE_DIRS:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/include \ - -D PETSC_DIR:PATH=${GP_EXT_DEPS}/petsc/install_for_gridpack \ - -D MPI_CXX_COMPILER:STRING='mpicxx' \ - -D MPI_C_COMPILER:STRING='mpicc' \ - -D MPIEXEC:STRING='mpiexec' \ - -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ - -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_INSTALL_DIR} \ + -D PETSC_DIR:PATH=${GP_EXT_DEPS}/petsc/install_for_gridpack \ + -D MPI_CXX_COMPILER:STRING='mpicxx' \ + -D MPI_C_COMPILER:STRING='mpicc' \ + -D MPIEXEC:STRING='mpiexec' \ + -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ + -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_INSTALL_DIR} \ -D CMAKE_BUILD_TYPE:STRING=Debug \ -D BUILD_SHARED_LIBS=YES \ -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ .. " - - cmake ${cmake_args} - echo "Installing GridPACK develop branch" - make -j 10 install + cmake "${cmake_args}" + + echo "Installing GridPACK develop branch" + make -j 10 install fi -if ${install_gridpack_python} -then - echo "Installing GridPACK python wrapper" - - cd ${GRIDPACK_ROOT_DIR} - - git submodule update --init - - echo ${GRIDPACK_DIR} - cd python - - export RHEL_OPENMPI_HACK=yes - - rm -rf build - - ${python_exe} setup.py build - - rm -rf ${GRIDPACK_INSTALL_DIR}/lib/python - mkdir ${GRIDPACK_INSTALL_DIR}/lib/python - - PYTHONPATH="${GRIDPACK_DIR}/lib/python:${PYTHONPATH}" - export PYTHONPATH - ${python_exe} setup.py install --home="$GRIDPACK_DIR" - +if ${install_gridpack_python}; then + echo "Installing GridPACK python wrapper" + + cd "${GRIDPACK_ROOT_DIR}" || exit + + git submodule update --init + + echo "${GRIDPACK_DIR}" + cd python || exit + + export RHEL_OPENMPI_HACK=yes + + rm -rf build + + ${python_exe} setup.py build + + rm -rf "${GRIDPACK_INSTALL_DIR}"/lib/python + mkdir "${GRIDPACK_INSTALL_DIR}"/lib/python + + PYTHONPATH="${GRIDPACK_DIR}/lib/python:${PYTHONPATH}" + export PYTHONPATH + ${python_exe} setup.py install --home="$GRIDPACK_DIR" + fi -cd ${GRIDPACK_ROOT_DIR} +cd "${GRIDPACK_ROOT_DIR}" || exit echo "Completed GridPACK installation" diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 87897593e..ed54e5d3d 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -1,3 +1,5 @@ +#! /bin/bash + # This script installs all GridPACK dependencies.The dependencies are installed in external-dependencies directory. # This script should be run from the top-level GridPACK directory. @@ -7,39 +9,35 @@ install_boost=true install_ga=true install_petsc=true -echo $(date) +date -if test -z ${GRIDPACK_ROOT_DIR} -then - export GRIDPACK_ROOT_DIR=${PWD} - echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" +if test -z "${GRIDPACK_ROOT_DIR}"; then + export GRIDPACK_ROOT_DIR=${PWD} + echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" fi -cd ${GRIDPACK_ROOT_DIR} +cd "${GRIDPACK_ROOT_DIR}" || exit # Create directory for installing external packages -if test -z ${GP_EXT_DEPS} -then - export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies - rm -rf ${GP_EXT_DEPS} - mkdir ${GP_EXT_DEPS} - echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" +if test -z "${GP_EXT_DEPS}"; then + export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies + rm -rf "${GP_EXT_DEPS}" + mkdir "${GP_EXT_DEPS}" + echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" else - if test -d ${GP_EXT_DEPS} - then - echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" - else - mkdir ${GP_EXT_DEPS} - fi + if test -d "${GP_EXT_DEPS}"; then + echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" + else + mkdir "${GP_EXT_DEPS}" + fi fi -cd ${GP_EXT_DEPS} +cd "${GP_EXT_DEPS}" || exit -if ${install_boost} -then +if ${install_boost}; then rm -rf boost* - + # Download and install Boost echo "Downloading Boost-1.78.0" @@ -49,14 +47,14 @@ then # Untar tar -xf boost_1_78_0.tar.gz - cd boost_1_78_0 + cd boost_1_78_0 || exit # Install boost echo "Building Boost-1.78.0" ./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem,system - echo 'using mpi ;' >> project-config.jam + echo 'using mpi ;' >>project-config.jam ./b2 -a -d+2 link=shared stage @@ -66,10 +64,9 @@ then echo "Building and Installing Boost libraries complete" fi -if ${install_ga} -then +if ${install_ga}; then # Download, build, and install GA - cd ${GP_EXT_DEPS} + cd "${GP_EXT_DEPS}" || exit echo "Downloading GA-5.8" @@ -77,31 +74,30 @@ then tar -xf ga-5.8.tar.gz - cd ga-5.8 + cd ga-5.8 || exit # Build GA echo "Building GA-5.8" - ./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix=${PWD}/install_for_gridpack --enable-shared - + ./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix="${PWD}"/install_for_gridpack --enable-shared + # Install GA echo "Installing GA-5.8" make -j 10 install - + echo "GA-5.8 installation complete" fi -if ${install_petsc} -then - +if ${install_petsc}; then + # Install PETSc 3.16.4 - cd ${GP_EXT_DEPS} + cd "${GP_EXT_DEPS}" || exit # Download echo "Downloading PETSc 3.16.4" git clone https://gitlab.com/petsc/petsc.git - - cd petsc + + cd petsc || exit git checkout tags/v3.16.4 -b v3.16.4 @@ -110,24 +106,24 @@ then # Install PETSc echo "Installing PETSc 3.16.4" - - ./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix=${PWD}/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 + + ./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix="${PWD}"/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 # Build PETSc echo "Building PETSc 3.16.4" - make + make # Install PETSc echo "Installing PETSc 3.16.4" make install - make check -fi + make check +fi # update LD_LIBRARY_PATH so that boost,ga, and petsc are on it export LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-5.8/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} -cd ${GRIDPACK_ROOT_DIR} +cd "${GRIDPACK_ROOT_DIR}" || exit echo "Completed installing GridPACK dependencies in ${GP_EXT_DEPS}" From 24cdd96196a09350e63e202b34e28cbd519bf63e Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 5 Sep 2023 22:41:14 -0500 Subject: [PATCH 009/195] Move blocks into functions and extract versions - Remove directory changes and assume current directory - Use local variables in functions - Capture logs --- install_gridpack.sh | 153 ++++++++++++++++------------- install_gridpack_deps.sh | 207 ++++++++++++++++++++++----------------- 2 files changed, 203 insertions(+), 157 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 523adef1c..21d676270 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -1,100 +1,115 @@ #! /bin/bash -# This script installs GridPACK and python wrappter.GridPACK is built in src/build directory and installed in src/install directory. +# installs GridPACK and its python wrappter +# gridPACK is built in src/build and installed to src/install +# run this from the top level GridPACK directory -# This script should be run from the top-level GridPACK directory. +set -xeuo pipefail -# Flag for install GridPACK and GridPACK python wrapper -install_gridpack=true -install_gridpack_python=true +function install_gridpack { + echo "--- GridPACK ---" -# Set your python executable here -python_exe=$(which python) -if test -z "${python_exe}"; then - python_exe=$(which python3) -fi + # args + local gridpack_deps_dir=$1 + local gridpack_build_dir=$2 + local gridpack_install_dir=$3 + ${gridpack_deps_dir:?} + ${gridpack_build_dir:?} + ${gridpack_install_dir:?} -if test -z "${GRIDPACK_ROOT_DIR}"; then - export GRIDPACK_ROOT_DIR=${PWD} - echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" -fi + # remove existing build and install dir + rm -rf "$gridpack_build_dir" + rm -rf "$gridpack_install_dir" -# Create directory for installing external packages -if test -z "${GP_EXT_DEPS}"; then - export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies -fi + # create the build dir + mkdir -p "$gridpack_build_dir" -cd "${GRIDPACK_ROOT_DIR}" || exit + pushd "$gridpack_build_dir" || exit -# Set environment variable GRIDPACK_BUILD_DIR and create a build directory -export GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build - -GRIDPACK_INSTALL_DIR=${GRIDPACK_ROOT_DIR}/src/install -export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} - -if ${install_gridpack}; then - - rm -rf "$GRIDPACK_BUILD_DIR" - mkdir "$GRIDPACK_BUILD_DIR" - - rm -rf "${GRIDPACK_INSTALL_DIR}" - - cd "${GRIDPACK_BUILD_DIR}" || exit - - ## GridPACK installation - echo "Building GridPACK" - - # git checkout develop + # todo? git checkout develop + # remove existing cmake output rm -rf CMake* - cmake_args="-D GA_DIR:STRING=${GP_EXT_DEPS}/ga-5.8/install_for_gridpack \ - -D BOOST_ROOT:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack \ - -D Boost_DIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib/cmake/Boost-1.78.0 \ - -D Boost_LIBRARIES:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib \ - -D Boost_INCLUDE_DIRS:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/include \ - -D PETSC_DIR:PATH=${GP_EXT_DEPS}/petsc/install_for_gridpack \ - -D MPI_CXX_COMPILER:STRING='mpicxx' \ - -D MPI_C_COMPILER:STRING='mpicc' \ - -D MPIEXEC:STRING='mpiexec' \ - -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ - -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_INSTALL_DIR} \ - -D CMAKE_BUILD_TYPE:STRING=Debug \ - -D BUILD_SHARED_LIBS=YES \ - -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ - .. " - - cmake "${cmake_args}" - - echo "Installing GridPACK develop branch" + # generate make files + echo "Generating GridPACK make files" + cmake \ + -D GA_DIR:STRING="${gridpack_deps_dir}/ga/install_for_gridpack" \ + -D BOOST_ROOT:STRING="${gridpack_deps_dir}/boost/install_for_gridpack" \ + -D Boost_DIR:STRING="${gridpack_deps_dir}/boost/install_for_gridpack/lib/cmake/Boost" \ + -D Boost_LIBRARIES:STRING="${gridpack_deps_dir}/boost/install_for_gridpack/lib" \ + -D Boost_INCLUDE_DIRS:STRING="${gridpack_deps_dir}/boost/install_for_gridpack/include" \ + -D PETSC_DIR:PATH="${gridpack_deps_dir}/petsc/install_for_gridpack" \ + -D MPI_CXX_COMPILER:STRING='mpicxx' \ + -D MPI_C_COMPILER:STRING='mpicc' \ + -D MPIEXEC:STRING='mpiexec' \ + -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ + -D CMAKE_INSTALL_PREFIX:PATH="${gridpack_install_dir}" \ + -D CMAKE_BUILD_TYPE:STRING=Debug \ + -D BUILD_SHARED_LIBS=YES \ + -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ + .. + + # install + echo "Installing GridPACK" make -j 10 install -fi -if ${install_gridpack_python}; then - echo "Installing GridPACK python wrapper" + popd || exit - cd "${GRIDPACK_ROOT_DIR}" || exit + echo "GridPACK installation complete" +} +function install_gridpack_python { + echo "--- GridPACK python wrapper ---" + + # args + local gridpack_build_dir=$1 + local gridpack_install_dir=$2 + ${gridpack_build_dir:?} + ${gridpack_install_dir:?} + + # update submodules + echo "Updating GridPACK submodules" git submodule update --init - echo "${GRIDPACK_DIR}" - cd python || exit + pushd python || exit export RHEL_OPENMPI_HACK=yes + # set python executable path + python_exe=$(which python || which python3) + + # remove existing build dir rm -rf build + # build + echo "Building GridPACK python wrapper" ${python_exe} setup.py build - rm -rf "${GRIDPACK_INSTALL_DIR}"/lib/python - mkdir "${GRIDPACK_INSTALL_DIR}"/lib/python + # remove existing install dir + local py_lib="${gridpack_install_dir}/lib/python" + rm -rf "${py_lib}" + mkdir -p "${py_lib}" + + # add lib to python path + export PYTHONPATH="${py_lib}:${PYTHONPATH}" - PYTHONPATH="${GRIDPACK_DIR}/lib/python:${PYTHONPATH}" - export PYTHONPATH + # install + echo "Installing GridPACK python wrapper" ${python_exe} setup.py install --home="$GRIDPACK_DIR" -fi + popd || exit + + echo "GridPACK python wrapper installation complete" +} + +echo "Installing GridPACK" +date + +build_dir=${PWD}/src/build +install_dir=${PWD}/src/install -cd "${GRIDPACK_ROOT_DIR}" || exit +install_gridpack "${GRIDPACK_EXT_DEPS-?}" "$build_dir" "$install_dir" +install_gridpack_python "$build_dir" "$install_dir" echo "Completed GridPACK installation" diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index ed54e5d3d..94b55000f 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -1,129 +1,160 @@ #! /bin/bash -# This script installs all GridPACK dependencies.The dependencies are installed in external-dependencies directory. +# installs GridPACK dependencies to the current directory -# This script should be run from the top-level GridPACK directory. +set -xeuo pipefail -# Flags for installing/not installing different dependency packages -install_boost=true -install_ga=true -install_petsc=true +function install_boost { + local boost_version=$1 -date - -if test -z "${GRIDPACK_ROOT_DIR}"; then - export GRIDPACK_ROOT_DIR=${PWD} - echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" -fi + # check args + : "${boost_version:?}" -cd "${GRIDPACK_ROOT_DIR}" || exit + echo "--- Installing Boost ${boost_version} ---" -# Create directory for installing external packages -if test -z "${GP_EXT_DEPS}"; then - export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies - rm -rf "${GP_EXT_DEPS}" - mkdir "${GP_EXT_DEPS}" - echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" -else - if test -d "${GP_EXT_DEPS}"; then - echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" - else - mkdir "${GP_EXT_DEPS}" - fi -fi + # remove existing + rm -rf boost* -cd "${GP_EXT_DEPS}" || exit + # download + echo "Downloading Boost" + wget \ + "https://boostorg.jfrog.io/artifactory/main/release/${boost_version}/source/boost_${boost_version//./_}.tar.gz" \ + -O boost.tar.gz \ + --quiet -if ${install_boost}; then + # unpack + echo "Unpacking Boost" + tar -xf boost.tar.gz && rm boost.tar.gz - rm -rf boost* + # remove version from dir name + mv "boost_${boost_version//./_}" boost - # Download and install Boost - echo "Downloading Boost-1.78.0" + pushd boost || exit - # Download Boost - wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz - - # Untar - tar -xf boost_1_78_0.tar.gz + # bootstrap + echo "Bootstrapping Boost" + ./bootstrap.sh \ + --prefix=install_for_gridpack \ + --with-libraries=mpi,serialization,random,filesystem,system \ + >../log/boost_bootstrap.log 2>&1 + echo 'using mpi ;' >>project-config.jam - cd boost_1_78_0 || exit + # build + echo "Building Boost" + ./b2 -a -d+2 link=shared stage >../log/boost_build.log 2>&1 - # Install boost - echo "Building Boost-1.78.0" + # install + echo "Installing Boost" + ./b2 -a -d+2 link=shared install >../log/boost_install.log 2>&1 - ./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem,system + popd || exit - echo 'using mpi ;' >>project-config.jam + echo "Boost installation complete" +} - ./b2 -a -d+2 link=shared stage +function install_ga { + local ga_version=$1 - echo "Installing Boost-1.78.0" - ./b2 -a -d+2 link=shared install + # check args + : "${ga_version:?}" - echo "Building and Installing Boost libraries complete" -fi + echo "--- Installing Global Arrays ${ga_version} ---" -if ${install_ga}; then - # Download, build, and install GA - cd "${GP_EXT_DEPS}" || exit + # download + echo "Downloading Global Arrays" + wget \ + "https://github.com/GlobalArrays/ga/releases/download/v${ga_version}/ga-${ga_version}.tar.gz" \ + -O ga.tar.gz \ + --quiet - echo "Downloading GA-5.8" + # unpack + echo "Unpacking Global Arrays" + tar -xf ga.tar.gz && rm ga.tar.gz - wget https://github.com/GlobalArrays/ga/releases/download/v5.8/ga-5.8.tar.gz + # remove version from dir name + mv "ga-${ga_version}" ga - tar -xf ga-5.8.tar.gz + pushd ga || exit - cd ga-5.8 || exit + # build + echo "Configuring Global Arrays" + ./configure \ + --with-mpi-ts \ + --disable-f77 \ + --without-blas \ + --enable-cxx \ + --enable-i4 \ + --prefix="${PWD}/install_for_gridpack" \ + --enable-shared \ + >../log/ga_configure.log 2>&1 - # Build GA - echo "Building GA-5.8" - ./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix="${PWD}"/install_for_gridpack --enable-shared + # install + echo "Installing Global Arrays" + make -j 10 install >../log/ga_install.log 2>&1 - # Install GA - echo "Installing GA-5.8" - make -j 10 install + popd || exit - echo "GA-5.8 installation complete" -fi + echo "Global Arrays installation complete" +} -if ${install_petsc}; then +function install_petsc { + local petsc_version=$1 - # Install PETSc 3.16.4 - cd "${GP_EXT_DEPS}" || exit + # check args + : "${petsc_version:?}" - # Download - echo "Downloading PETSc 3.16.4" + echo "--- Installing PETSc ${petsc_version} ---" + # clone + echo "Cloning PETSc repository" git clone https://gitlab.com/petsc/petsc.git - cd petsc || exit + pushd petsc || exit - git checkout tags/v3.16.4 -b v3.16.4 + git checkout "tags/v${petsc_version}" -b "v${petsc_version}" export PETSC_DIR=${PWD} export PETSC_ARCH=build-dir - # Install PETSc - echo "Installing PETSc 3.16.4" - - ./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix="${PWD}"/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 - - # Build PETSc - echo "Building PETSc 3.16.4" - - make - - # Install PETSc - echo "Installing PETSc 3.16.4" - - make install - make check -fi + # install + echo "Configuring PETSc" + ./configure \ + --download-superlu_dist \ + --download-metis \ + --download-parmetis \ + --download-suitesparse \ + --download-f2cblaslapack \ + --download-cmake \ + --prefix="${PWD}"/install_for_gridpack \ + --scalar-type=complex \ + --with-shared-libraries=1 \ + --download-f2cblaslapack=1 \ + >../log/petsc_configure.log 2>&1 + + # build + echo "Building PETSc" + make >../log/petsc_build.log 2>&1 + + # install + echo "Installing PETSc" + make install >../log/petsc_install.log 2>&1 + + # check + echo "Checking PETSc" + make check >../log/petsc_check.log 2>&1 + + popd || exit + + echo "PETSc installation complete" +} + +echo "Installing GridPACK dependencies" +date -# update LD_LIBRARY_PATH so that boost,ga, and petsc are on it -export LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-5.8/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} +mkdir -p log -cd "${GRIDPACK_ROOT_DIR}" || exit +install_boost "1.78.0" +install_ga "5.8" +install_petsc "3.16.4" -echo "Completed installing GridPACK dependencies in ${GP_EXT_DEPS}" +echo "GridPACK dependency installation complete" From af2952c568eec49a1c6dab61c1f1de3f851d6a6b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 6 Sep 2023 01:39:00 -0500 Subject: [PATCH 010/195] Correct how variables are checked --- install_gridpack.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 21d676270..fdadabf24 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -13,9 +13,7 @@ function install_gridpack { local gridpack_deps_dir=$1 local gridpack_build_dir=$2 local gridpack_install_dir=$3 - ${gridpack_deps_dir:?} - ${gridpack_build_dir:?} - ${gridpack_install_dir:?} + : "${gridpack_deps_dir:?}" "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" # remove existing build and install dir rm -rf "$gridpack_build_dir" @@ -65,8 +63,7 @@ function install_gridpack_python { # args local gridpack_build_dir=$1 local gridpack_install_dir=$2 - ${gridpack_build_dir:?} - ${gridpack_install_dir:?} + : "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" # update submodules echo "Updating GridPACK submodules" @@ -109,7 +106,7 @@ date build_dir=${PWD}/src/build install_dir=${PWD}/src/install -install_gridpack "${GRIDPACK_EXT_DEPS-?}" "$build_dir" "$install_dir" +install_gridpack "${GRIDPACK_EXT_DEPS:?}" "$build_dir" "$install_dir" install_gridpack_python "$build_dir" "$install_dir" echo "Completed GridPACK installation" From 1e9392431984a8399759ee6546fcf38ea16bda3c Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 6 Sep 2023 01:39:37 -0500 Subject: [PATCH 011/195] Bump versions out of script with env vars --- install_gridpack_deps.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 94b55000f..74a5dd15a 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -153,8 +153,8 @@ date mkdir -p log -install_boost "1.78.0" -install_ga "5.8" -install_petsc "3.16.4" +install_boost "${BOOST_VERSION:?}" +install_ga "${GA_VERSION:?}" +install_petsc "${PETSC_VERSION:?}" echo "GridPACK dependency installation complete" From 1d509f05da29b5a10b451e5616e7d1a903630d5d Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 23 Aug 2023 14:24:13 -0700 Subject: [PATCH 012/195] Remove GA-based dense matrix; adjust matrix transpose --- src/example_configuration.sh | 21 +- src/math/CMakeLists.txt | 12 +- src/math/petsc/ga_matrix.cpp | 903 --------------------- src/math/petsc/ga_matrix.hpp | 69 -- src/math/petsc/petsc_matrix_operations.cpp | 58 +- src/math/test/petsc_ga_matrix.cpp | 358 -------- 6 files changed, 16 insertions(+), 1405 deletions(-) delete mode 100644 src/math/petsc/ga_matrix.cpp delete mode 100644 src/math/petsc/ga_matrix.hpp delete mode 100644 src/math/test/petsc_ga_matrix.cpp diff --git a/src/example_configuration.sh b/src/example_configuration.sh index 0e0cb3cc9..ee870d678 100755 --- a/src/example_configuration.sh +++ b/src/example_configuration.sh @@ -265,30 +265,19 @@ elif [ $host == "tlaloc" ]; then # -D PETSC_ARCH:STRING="ubuntu-real-shared-3.16.6" \ # -D USE_OLD_PETSC:BOOL=OFF \ - # System PETSc package: - # -D PETSC_DIR:STRING="/usr/lib/petsc" \ - # -D PETSC_ARCH:STRING="" \ + # Custom built 3.14, complex: + # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.14.6" \ + # -D PETSC_ARCH:STRING="ubuntu-complex-shared" \ # -D USE_OLD_PETSC:BOOL=OFF \ - + prefix="$HOME/Projects/GridPakLDRD/gridpack-install" cmake -Wdev --debug-trycompile \ - # Custom built 3.12.5, real: - # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.12.5" \ - # -D PETSC_ARCH:STRING="ubuntu-real-shared-3.12" \ - # -D USE_OLD_PETSC:BOOL=OFF \ - - # Custom built 3.10.5, real: - # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.10.5" \ - # -D PETSC_ARCH:STRING="ubuntu-real-shared-3.10" \ - # -D USE_OLD_PETSC:BOOL=OFF \ - - prefix="$HOME/Projects/GridPakLDRD/gridpack-install" cmake -Wdev --debug-trycompile \ --graphviz=GridPACK.dot \ -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc.gitlab" \ - -D PETSC_ARCH:STRING="ubuntu-real-shared-3.16.6" \ + -D PETSC_ARCH:STRING="ubuntu-complex-shared-3.16.6" \ -D USE_OLD_PETSC:BOOL=OFF \ -D BOOST_ROOT:PATH="/usr" \ -D Boost_NO_BOOST_CMAKE:BOOL=TRUE \ diff --git a/src/math/CMakeLists.txt b/src/math/CMakeLists.txt index 5146dc957..b7220c312 100644 --- a/src/math/CMakeLists.txt +++ b/src/math/CMakeLists.txt @@ -7,7 +7,7 @@ # distribution. # ------------------------------------------------------------- # Created May 7, 2013 by William A. Perkins -# Last Change: 2022-10-05 08:15:59 d3g096 +# Last Change: 2023-08-23 12:35:13 d3g096 # ------------------------------------------------------------- set(gridpack_math_sources) set(target_libraries @@ -80,7 +80,6 @@ if (PETSC_FOUND) petsc/petsc_vector.cpp petsc/petsc_vector_wrapper.cpp petsc/petsc_matrix.cpp - petsc/ga_matrix.cpp petsc/petsc_matrix_operations.cpp petsc/petsc_matrix_wrapper.cpp petsc/petsc_matrix.cpp @@ -359,15 +358,6 @@ add_executable(bouncing_ball_test test/bouncing_ball_test.cpp) target_link_libraries(bouncing_ball_test gridpack_math ${target_libraries}) -# ------------------------------------------------------------- -# Test GA-based Dense Matrix -# ------------------------------------------------------------- -if (PETSC_FOUND) - add_executable(petsc_ga_matrix_test test/petsc_ga_matrix.cpp) - target_link_libraries(petsc_ga_matrix_test gridpack_math ${target_libraries}) - gridpack_add_unit_test(petsc_ga_matrix petsc_ga_matrix_test) -endif() - # ------------------------------------------------------------- # installation # ------------------------------------------------------------- diff --git a/src/math/petsc/ga_matrix.cpp b/src/math/petsc/ga_matrix.cpp deleted file mode 100644 index 7e4cf6e6b..000000000 --- a/src/math/petsc/ga_matrix.cpp +++ /dev/null @@ -1,903 +0,0 @@ -/* ------------------------------------------------------------- - * Copyright (c) 2013 Battelle Memorial Institute - * Licensed under modified BSD License. A copy of this license can be found - * in the LICENSE file in the top level directory of this distribution. - * ------------------------------------------------------------- */ -/** - * @file ga_matrix.c - * @author William A. Perkins - * @date 2019-07-29 12:16:09 d3g096 - * - * @brief - * - * - */ - - -#include -#include -#include - -#include -#include "ga_matrix.hpp" - - -// ------------------------------------------------------------- -// Determine the Global Arrays data type from PETSc configuration -// ------------------------------------------------------------- -#if defined(PETSC_USE_REAL_DOUBLE) - -#if defined(PETSC_USE_COMPLEX) -typedef DoubleComplex PetscScalarGA; -#define MT_PETSC_SCALAR MT_C_DCPL - -#else -typedef double PetscScalarGA; -#define MT_PETSC_SCALAR MT_C_DBL -#endif - -#else - -#if defined(PETSC_USE_COMPLEX) -typedef SingleComplex PetscScalarGA; -#define MT_PETSC_SCALAR MT_C_SCPL; -#else -typedef float PetscScalarGA; -#define MT_PETSC_SCALAR MT_C_FLOAT -#endif - -#endif - -static const PetscScalarGA one = -#if defined(PETSC_USE_COMPLEX) - { 1.0, 0.0 } -#else - 1.0 -#endif - ; - -static const PetscScalarGA zero = -#if defined(PETSC_USE_COMPLEX) - { 0.0, 0.0 } -#else - 0.0 -#endif - ; - - - -// ------------------------------------------------------------- -// struct MatGACtx -// ------------------------------------------------------------- -struct MatGACtx { - int gaGroup; /**< GA process group handle */ - int ga; /**< GA handle */ - int lo[2]; /**< Lower limits of "local" matrix ownership */ - int hi[2]; /**< Upper limits of "local" matrix ownership */ -}; - -static PetscErrorCode MatSetOperations_DenseGA(Mat A); - -// ------------------------------------------------------------- -// CreateMatGA -// ------------------------------------------------------------- -static -PetscErrorCode -CreateMatGA(int pgroup, int lrows, int lcols, int grows, int gcols, int *ga) -{ - PetscErrorCode ierr = 0; - - /* Try to honor local ownership request (of rows). */ - - int nprocs = GA_Pgroup_nnodes(pgroup); - int me = GA_Pgroup_nodeid(pgroup); - std::vector tmapc(nprocs+1); - std::vector mapc(nprocs+1); - int i; - - for (i = 0; i < nprocs+1; i++) tmapc[i] = 0; - tmapc[me] = lrows; - GA_Pgroup_igop(pgroup, &tmapc[0], nprocs+1, "+"); - mapc[0] = 0; - for (i = 1; i < nprocs; i++) mapc[i] = mapc[i-1]+tmapc[i-1]; - mapc[nprocs] = 0; - - int dims[2] = {grows, gcols}; - int blocks[2] = { nprocs, 1 }; - - - *ga = NGA_Create_irreg_config(MT_PETSC_SCALAR, 2, &dims[0], - "PETSc Matrix in GA", - &blocks[0], &mapc[0], - pgroup); - - PetscScalar z(0.0); - GA_Fill(*ga, &z); - - return ierr; -} - -// ------------------------------------------------------------- -// MPIComm2GApgroup -// ------------------------------------------------------------- -static -PetscErrorCode -MPIComm2GApgroup(MPI_Comm comm, int *pGrpHandle) -{ - PetscErrorCode ierr = 0; - int nproc; - int me, myGlobalRank; - int *proclist; - int p; - - ierr = MPI_Comm_size(comm, &nproc); CHKERRQ(ierr); - ierr = MPI_Comm_rank(comm, &me); CHKERRQ(ierr); - myGlobalRank = GA_Nodeid(); - ierr = PetscMalloc(nproc*sizeof(int), &proclist); CHKERRQ(ierr); - for (p = 0; p < nproc; ++p) { - proclist[p] = 0; - } - proclist[me] = myGlobalRank; - ierr = MPI_Allreduce(MPI_IN_PLACE, &proclist[0], nproc, MPI_INT, MPI_SUM, comm); CHKERRQ(ierr); - *pGrpHandle = GA_Pgroup_create(&proclist[0], nproc); - ierr = PetscFree(proclist); CHKERRQ(ierr); - return ierr; -} - -// ------------------------------------------------------------- -// Mat2GA -// This should only be called by MatMultbyGA(). It assumes the Mat is -// dense and the GA has the same ownership distribution. -// ------------------------------------------------------------- -static -PetscErrorCode -Mat2GA(const Mat& A, int Aga) -{ - PetscErrorCode ierr(0); - int me(GA_Nodeid()); - int r, rmin, rmax; - int lo[2], hi[2], ld[2]; - int i, j; - PetscInt grows, gcols, lrows, lcols; - PetscScalar *matdata; - ierr = MatGetSize(A, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetOwnershipRange(A, &rmin, &rmax); CHKERRQ(ierr); - - NGA_Distribution(Aga, me, &lo[0], &hi[0]); - - std::vector cidx(gcols); - for (j = 0; j < gcols; ++j) cidx[j] = j; - std::vector rdata(gcols); - PetscScalarGA *gadata; - - for (r = rmin; r < rmax; ++r) { - ierr = MatGetValues(A, 1, &r, gcols, &cidx[0], &rdata[0]); CHKERRQ(ierr); - lo[0] = r; hi[0] = r; - lo[1] = 0; hi[1] = gcols - 1; - ld[0] = 1; - NGA_Access(Aga, &lo[0], &hi[0], &gadata, &ld[0]); - for (j = 0; j < gcols; ++j) { -#if defined(PETSC_USE_COMPLEX) - gadata[j].real = rdata[j].real(); - gadata[j].imag = rdata[j].imag(); -#else - gadata[j] = rdata[j]; -#endif - } - NGA_Release_update(Aga, &lo[0], &hi[0]); - } - GA_Sync(); - // GA_Print(Aga); - return ierr; -} - -// ------------------------------------------------------------- -// GA2Mat -// This should only be called by MatMultbyGA(). It assumes the Mat is -// dense and the GA has the same ownership distribution. -// ------------------------------------------------------------- -static -PetscErrorCode -GA2Mat(const int Aga, Mat& A) -{ - PetscErrorCode ierr(0); - int me(GA_Nodeid()); - int r, rmin, rmax; - int lo[2], hi[2], ld[2]; - int i, j; - PetscInt grows, gcols, lrows, lcols; - PetscScalar *matdata; - ierr = MatGetSize(A, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetOwnershipRange(A, &rmin, &rmax); CHKERRQ(ierr); - - NGA_Distribution(Aga, me, &lo[0], &hi[0]); - - std::vector cidx(gcols); - for (j = 0; j < gcols; ++j) cidx[j] = j; - std::vector rdata(gcols); - PetscScalarGA *gadata; - - for (r = rmin; r < rmax; ++r) { - lo[0] = r; hi[0] = r; - lo[1] = 0; hi[1] = gcols - 1; - ld[0] = 1; - NGA_Access(Aga, &lo[0], &hi[0], &gadata, &ld[0]); - for (j = 0; j < gcols; ++j) { -#if defined(PETSC_USE_COMPLEX) - rdata[j].real(gadata[j].real); - rdata[j].imag(gadata[j].imag); -#else - rdata[j] = gadata[j]; -#endif - } - ierr = MatSetValues(A, 1, &r, gcols, &cidx[0], &rdata[0], INSERT_VALUES); CHKERRQ(ierr); - NGA_Release(Aga, &lo[0], &hi[0]); - } - ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - - return ierr; -} - -// ------------------------------------------------------------- -// MatMultbyGA -// ------------------------------------------------------------- -PetscErrorCode -MatMultbyGA(const Mat& A, const Mat& B, Mat& C) -{ - PetscErrorCode ierr(0); - MPI_Comm comm; - int lrows, grows, lcols, gcols; - int pgp, opgp; - int Aga, Bga, Cga; - int m, n, k; - - ierr = PetscObjectGetComm((PetscObject)A, &comm); CHKERRQ(ierr); - opgp = GA_Pgroup_get_default(); - - ierr = MPIComm2GApgroup(comm, &pgp); - GA_Pgroup_set_default(pgp); - - ierr = MatGetSize(A, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(A, &lrows, &lcols); CHKERRQ(ierr); - ierr = CreateMatGA(pgp, lrows, lcols, grows, gcols, &Aga); CHKERRQ(ierr); - ierr = Mat2GA(A, Aga); CHKERRQ(ierr); - m = grows; - k = gcols; - - ierr = MatGetSize(B, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(B, &lrows, &lcols); CHKERRQ(ierr); - ierr = CreateMatGA(pgp, lrows, lcols, grows, gcols, &Bga); CHKERRQ(ierr); - ierr = Mat2GA(B, Bga); CHKERRQ(ierr); - n = gcols; - - ierr = MatGetSize(C, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(C, &lrows, &lcols); CHKERRQ(ierr); - ierr = CreateMatGA(pgp, lrows, lcols, grows, gcols, &Cga); CHKERRQ(ierr); - - char no('n'); - PetscScalarGA alpha, beta; - -#if defined(PETSC_USE_REAL_DOUBLE) -# if defined(PETSC_USE_COMPLEX) - - alpha.real = 1.0; - alpha.imag = 0.0; - beta.real = 0.0; - beta.imag = 0.0; - - GA_Zgemm(no, no, m, n, k, one, Aga, Bga, beta, Cga); - -# else - - alpha = 1.0; - beta = 0.0; - - GA_Dgemm(no, no, m, n, k, one, Aga, Bga, beta, Cga); - -# endif -#else -# if defined(PETSC_USE_COMPLEX) -# error "no single precision complex" -# else - - alpha = 1.0; - beta = 0.0; - - GA_Sgemm(no, no, m, n, k, one, Aga, Bga, beta, Cga); - -# endif -#endif - - ierr = GA2Mat(Cga, C); CHKERRQ(ierr); - - GA_Pgroup_sync(pgp); - - GA_Destroy(Aga); - GA_Destroy(Bga); - GA_Destroy(Cga); - - GA_Pgroup_set_default(opgp); - GA_Pgroup_destroy(pgp); - - return ierr; -} - -// ------------------------------------------------------------- -// MatMultbyGA_new -// ------------------------------------------------------------- -PetscErrorCode -MatMultbyGA_new(const Mat& A, const Mat& B, Mat& C) -{ - PetscErrorCode ierr(0); - MPI_Comm comm; - int nproc; - PetscInt grows_a, grows_b, grows_c; - PetscInt lrows_a, lrows_b, lrows_c; - PetscInt gcols_a, gcols_b, gcols_c; - PetscInt lcols_a, lcols_b, lcols_c; - - ierr = MatGetSize(A, &grows_a, &gcols_a); CHKERRQ(ierr); - ierr = MatGetLocalSize(A, &lrows_a, &lcols_a); CHKERRQ(ierr); - - ierr = MatGetSize(B, &grows_b, &gcols_b); CHKERRQ(ierr); - ierr = MatGetLocalSize(B, &lrows_b, &lcols_b); CHKERRQ(ierr); - - grows_c = grows_a; - lrows_c = lrows_a; - gcols_c = gcols_b; - lcols_c = lcols_b; - - ierr = PetscObjectGetComm((PetscObject)A, &comm); CHKERRQ(ierr); - ierr = MPI_Comm_size(comm, &nproc); - - ierr = MatCreate(comm, &C); CHKERRQ(ierr); - ierr = MatSetSizes(C, lrows_c, lcols_c, grows_c, gcols_c); CHKERRQ(ierr); - if (nproc == 1) { - ierr = MatSetType(C, MATSEQDENSE); CHKERRQ(ierr); - ierr = MatSeqDenseSetPreallocation(C, PETSC_NULL); CHKERRQ(ierr); - } else { - ierr = MatSetType(C, MATDENSE); CHKERRQ(ierr); - ierr = MatMPIDenseSetPreallocation(C, PETSC_NULL); CHKERRQ(ierr); - } - - ierr = MatAssemblyBegin(C, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - ierr = MatAssemblyEnd(C, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - ierr = MatMultbyGA(A, B, C); CHKERRQ(ierr); - return ierr; -} - -// ------------------------------------------------------------- -// Vec2GA -// ------------------------------------------------------------- -static -PetscErrorCode -Vec2GA(Vec x, int pgroup, int *ga, bool trans = false) -{ - int lrows, rows; - PetscErrorCode ierr = 0; - - ierr = VecGetLocalSize(x, &lrows); CHKERRQ(ierr); - ierr = VecGetSize(x, &rows); CHKERRQ(ierr); - - PetscInt vlo, vhi; - ierr = VecGetOwnershipRange(x, &vlo, &vhi); CHKERRQ(ierr); - - const PetscScalar *v; - ierr = VecGetArrayRead(x, &v); CHKERRQ(ierr); - - int lo[2] = {0,0}, hi[2] = {0,0}, ld[2] = {1,1}; - if (!trans) { - ierr = CreateMatGA(pgroup, lrows, 1, rows, 1, ga); CHKERRQ(ierr); - lo[0] = vlo; - hi[0] = vhi-1; - } else { - ierr = CreateMatGA(pgroup, 1, lrows, 1, rows, ga); CHKERRQ(ierr); - lo[1] = vlo; - hi[1] = vhi-1; - } - // bad cast to get rid of const - NGA_Put(*ga, lo, hi, (void *)v, ld); - // GA_Print(*ga); - ierr = VecRestoreArrayRead(x, &v); CHKERRQ(ierr); - - GA_Pgroup_sync(pgroup); - return ierr; -} - -// ------------------------------------------------------------- -// GA2Vec -// ------------------------------------------------------------- -static -int -GA2Vec(int ga, Vec x) -{ - int lrows, rows; - PetscErrorCode ierr = 0; - - ierr = VecGetLocalSize(x, &lrows); CHKERRQ(ierr); - ierr = VecGetSize(x, &rows); CHKERRQ(ierr); - - PetscInt vlo, vhi; - ierr = VecGetOwnershipRange(x, &vlo, &vhi); CHKERRQ(ierr); - - PetscScalar *v; - ierr = VecGetArray(x, &v); CHKERRQ(ierr); - int lo[2] = {vlo,0}, hi[2] = {vhi-1,0}, ld[2] = {1,1}; - NGA_Get(ga, lo, hi, v, ld); - ierr = VecRestoreArray(x, &v); CHKERRQ(ierr); - - return ierr; -} - -// ------------------------------------------------------------- -// MatSetValues_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatSetValues_DenseGA(Mat mat, - PetscInt m, const PetscInt idxm[], PetscInt n, const PetscInt idxn[], - const PetscScalar v[],InsertMode addv) -{ - PetscErrorCode ierr = 0; - struct MatGACtx *ctx; - int i, j, idx; - PetscScalar vij, one(1.0); - int lo[2], hi[2], ld[2] = {1, 1}; - ierr = MatShellGetContext(mat, (void *)&ctx); CHKERRQ(ierr); - - idx = 0; - for (i = 0; i < m; ++i) { - for (j = 0; j < n; ++j, ++idx) { - lo[0] = idxm[i]; - hi[0] = idxm[i]; - lo[1] = idxn[j]; - hi[1] = idxn[j]; - vij = v[idx]; - switch (addv) { - case INSERT_VALUES: - NGA_Put(ctx->ga, lo, hi, (void *)&vij, ld); - break; - case ADD_VALUES: - NGA_Acc(ctx->ga, lo, hi, (void *)&vij, ld, &one); - break; - default: - BOOST_ASSERT_MSG(false, "Unknown set operation"); - } - } - } - return ierr; -} - -// ------------------------------------------------------------- -// MatGetValues_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatGetValues_DenseGA(Mat mat, - PetscInt m, const PetscInt idxm[], PetscInt n, const PetscInt idxn[], - PetscScalar v[]) -{ - PetscErrorCode ierr = 0; - struct MatGACtx *ctx; - int i, j, idx; - PetscScalar vij; - int lo[2], hi[2], ld[2] = {1, 1}; - ierr = MatShellGetContext(mat, (void *)&ctx); CHKERRQ(ierr); - - idx = 0; - for (i = 0; i < m; ++i) { - for (j = 0; j < n; ++j, ++idx) { - lo[0] = idxm[i]; - hi[0] = idxm[i]; - lo[1] = idxn[j]; - hi[1] = idxn[j]; - NGA_Get(ctx->ga, lo, hi, (void *)&vij, ld); - v[idx] = vij; - } - } - return ierr; -} - -// ------------------------------------------------------------- -// MatAssemmblyBegin_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatAssemmblyBegin_DenseGA(Mat mat, MatAssemblyType type) -{ - PetscErrorCode ierr = 0; - struct MatGACtx *ctx; - ierr = MatShellGetContext(mat, &ctx); CHKERRQ(ierr); - return ierr; -} - -// ------------------------------------------------------------- -// MatAssemmblyEnd_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatAssemmblyEnd_DenseGA(Mat mat, MatAssemblyType type) -{ - PetscErrorCode ierr = 0; - struct MatGACtx *ctx; - ierr = MatShellGetContext(mat, &ctx); CHKERRQ(ierr); - GA_Pgroup_sync(ctx->gaGroup); - MPI_Comm comm; - ierr = PetscObjectGetComm((PetscObject)mat,&comm); CHKERRQ(ierr); - ierr = MPI_Barrier(comm); - // GA_Print(ctx->ga); - return ierr; -} - - -// ------------------------------------------------------------- -// MatMult_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatMult_DenseGA(Mat mat, Vec x, Vec y) -{ - // FIXME: I'm assuming the Mat and Vec's are compatible and that's - // been checked somewhere else. Probably a mistake. - - PetscErrorCode ierr = 0; - struct MatGACtx *ctx; - ierr = MatShellGetContext(mat, &ctx); CHKERRQ(ierr); - - PetscInt Arows, Acols; - ierr = MatGetSize(mat, &Arows, &Acols); CHKERRQ(ierr); - - int g_x, g_y; - ierr = Vec2GA(x, ctx->gaGroup, &g_x, false); CHKERRQ(ierr); - ierr = Vec2GA(y, ctx->gaGroup, &g_y, false); CHKERRQ(ierr); - - PetscScalarGA alpha(one), beta(zero); - int ndim, itype, lo[2] = {0,0}, ahi[2], xhi[2], yhi[2]; - NGA_Inquire(ctx->ga, &itype, &ndim, ahi); - ahi[0] -= 1; ahi[1] -= 1; - NGA_Inquire(g_x, &itype, &ndim, xhi); - xhi[0] -= 1; xhi[1] -= 1; - NGA_Inquire(g_y, &itype, &ndim, yhi); - yhi[0] -= 1; yhi[1] -= 1; - // GA_Print(ctx->ga); - // GA_Print(g_x); - NGA_Matmul_patch('N', 'N', &alpha, &beta, - ctx->ga, lo, ahi, - g_x, lo, xhi, - g_y, lo, yhi); - - GA_Pgroup_sync(ctx->gaGroup); - // GA_Print(g_y); - - ierr = GA2Vec(g_y, y); CHKERRQ(ierr); - - GA_Destroy(g_y); - GA_Destroy(g_x); - - MPI_Comm comm; - ierr = PetscObjectGetComm((PetscObject)mat,&comm); CHKERRQ(ierr); - ierr = MPI_Barrier(comm); - - return ierr; -} - -// ------------------------------------------------------------- -// MatMatMult_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatMatMult_DenseGA(Mat A, Mat B, MatReuse scall, PetscReal fill, Mat *C) -{ - - // matrix sizes appear to be checked before here, so we won't do it again - - PetscErrorCode ierr = 0; - - MPI_Comm comm; - ierr = PetscObjectGetComm((PetscObject)A, &comm); CHKERRQ(ierr); - - MatType atype, btype; - ierr = MatGetType(A, &atype); - ierr = MatGetType(B, &btype); - - PetscBool issame; - PetscStrcmp(atype, btype, &issame); - - Mat Bga, Cga; - struct MatGACtx *Actx, *Bctx, *Cctx; - - ierr = MatShellGetContext(A, &Actx); CHKERRQ(ierr); - - if (issame) { - Bga = B; - } else { - ierr = MatConvertToDenseGA(B, &Bga); CHKERRQ(ierr); - } - ierr = MatShellGetContext(Bga, &Bctx); CHKERRQ(ierr); - - PetscInt lrows, lcols, grows, gcols, junk; - - ierr = MatGetSize(A, &grows, &junk); CHKERRQ(ierr); - ierr = MatGetSize(B, &junk, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(A, &lrows, &junk); CHKERRQ(ierr); - ierr = MatGetLocalSize(B, &junk, &lcols); CHKERRQ(ierr); - - ierr = MatCreateDenseGA(comm, lrows, lcols, grows, gcols, &Cga); CHKERRQ(ierr); - ierr = MatShellGetContext(Cga, &Cctx); CHKERRQ(ierr); - - PetscScalarGA alpha(one), beta(zero); - int ndim, itype, lo[2] = {0,0}, ahi[2], bhi[2], chi[2]; - NGA_Inquire(Actx->ga, &itype, &ndim, ahi); - ahi[0] -= 1; ahi[1] -= 1; - NGA_Inquire(Bctx->ga, &itype, &ndim, bhi); - bhi[0] -= 1; bhi[1] -= 1; - NGA_Inquire(Cctx->ga, &itype, &ndim, chi); - chi[0] -= 1; chi[1] -= 1; - // GA_Print(Actx->ga); - // GA_Print(Bctx->ga); - NGA_Matmul_patch('N', 'N', &alpha, &beta, - Actx->ga, lo, ahi, - Bctx->ga, lo, bhi, - Cctx->ga, lo, chi); - // GA_Print(Cctx->ga); - - switch (scall) { - case MAT_REUSE_MATRIX: - ierr = MatCopy(Cga, *C, SAME_NONZERO_PATTERN); CHKERRQ(ierr); - case MAT_INITIAL_MATRIX: - default: - ierr = MatDuplicate(Cga, MAT_COPY_VALUES, C); CHKERRQ(ierr); - break; - } - - if (!issame) ierr = MatDestroy(&Bga); CHKERRQ(ierr); - ierr = MatDestroy(&Cga); CHKERRQ(ierr); - - return ierr; -} - -// ------------------------------------------------------------- -// MatView_DenseGA -// ------------------------------------------------------------- - - -// ------------------------------------------------------------- -// MatZeroEntries_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatZeroEntries_DenseGA(Mat A) -{ - PetscErrorCode ierr = 0; - - struct MatGACtx *ctx; - ierr = MatShellGetContext(A, &ctx); CHKERRQ(ierr); - GA_Zero(ctx->ga); - - return ierr; -} - - -// ------------------------------------------------------------- -// MatDestroy_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatDestroy_DenseGA(Mat A) -{ - PetscErrorCode ierr = 0; - - MPI_Comm comm; - ierr = PetscObjectGetComm((PetscObject)A,&comm); CHKERRQ(ierr); - ierr = MPI_Barrier(comm); - - struct MatGACtx *ctx; - ierr = MatShellGetContext(A, &ctx); CHKERRQ(ierr); - GA_Pgroup_sync(ctx->gaGroup); - GA_Destroy(ctx->ga); - GA_Pgroup_destroy(ctx->gaGroup); - ierr = PetscFree(ctx); - return ierr; -} - -// ------------------------------------------------------------- -// MatTranspose_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatTranspose_DenseGA(Mat mat, MatReuse reuse, Mat *B) -{ - PetscErrorCode ierr = 0; - - MPI_Comm comm; - ierr = PetscObjectGetComm((PetscObject)mat, &comm); CHKERRQ(ierr); - - struct MatGACtx *ctx, *newctx; - ierr = MatShellGetContext(mat, &ctx); CHKERRQ(ierr); - - PetscInt lrows, grows, lcols, gcols; - ierr = MatGetSize(mat, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(mat, &lrows, &lcols); CHKERRQ(ierr); - - ierr = PetscMalloc(sizeof(struct MatGACtx), &newctx); CHKERRQ(ierr); - ierr = MPIComm2GApgroup(comm, &(newctx->gaGroup)); - - ierr = CreateMatGA(newctx->gaGroup, lcols, lrows, gcols, grows, &(newctx->ga)); CHKERRQ(ierr); - GA_Transpose(ctx->ga, newctx->ga); - - ierr = MatCreateShell(comm, lcols, lrows, gcols, grows, newctx, B); CHKERRQ(ierr); - ierr = MatSetOperations_DenseGA(*B); - - return ierr; -} - - -// ------------------------------------------------------------- -// MatCreateDenseGA -// ------------------------------------------------------------- -PetscErrorCode -MatCreateDenseGA(MPI_Comm comm, - PetscInt m,PetscInt n,PetscInt M,PetscInt N, - Mat *A) -{ - PetscErrorCode ierr = 0; - struct MatGACtx *ctx; - int lrows, grows, lcols, gcols; - - ierr = PetscMalloc(sizeof(struct MatGACtx), &ctx); CHKERRQ(ierr); - ierr = MPIComm2GApgroup(comm, &(ctx->gaGroup)); CHKERRQ(ierr); - - lrows = m; lcols = n; - grows = M; gcols = N; - - if (lrows == PETSC_DECIDE || lrows == PETSC_DETERMINE || - grows == PETSC_DECIDE || grows == PETSC_DETERMINE) { - ierr = PetscSplitOwnership(comm, &lrows, &grows); CHKERRQ(ierr); - } - - if (lcols == PETSC_DECIDE || lcols == PETSC_DETERMINE || - gcols == PETSC_DECIDE || gcols == PETSC_DETERMINE) { - ierr = PetscSplitOwnership(comm, &lcols, &gcols); CHKERRQ(ierr); - } - - ierr = CreateMatGA(ctx->gaGroup, lrows, lcols, grows, gcols, &(ctx->ga)); CHKERRQ(ierr); - ierr = MatCreateShell(comm, lrows, lcols, grows, gcols, ctx, A); CHKERRQ(ierr); - ierr = MatSetOperations_DenseGA(*A); - - return ierr; -} - -// ------------------------------------------------------------- -// MatDuplicate_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatDuplicate_DenseGA(Mat mat, MatDuplicateOption op, Mat *M) -{ - PetscErrorCode ierr = 0; - struct MatGACtx *ctx, *newctx; - ierr = MatShellGetContext(mat, &ctx); CHKERRQ(ierr); - - MPI_Comm comm; - ierr = PetscObjectGetComm((PetscObject)mat, &comm); CHKERRQ(ierr); - - PetscInt lrows, grows, lcols, gcols; - ierr = MatGetSize(mat, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(mat, &lrows, &lcols); CHKERRQ(ierr); - - ierr = PetscMalloc(sizeof(struct MatGACtx), &newctx); CHKERRQ(ierr); - ierr = MPIComm2GApgroup(comm, &(newctx->gaGroup)); - - ierr = CreateMatGA(newctx->gaGroup, - lrows, lcols, grows, gcols, &(newctx->ga)); CHKERRQ(ierr); - ierr = MatCreateShell(comm, lrows, lcols, grows, gcols, newctx, M); CHKERRQ(ierr); - ierr = MatSetOperations_DenseGA(*M); - - PetscScalar z(0.0); - switch (op) { - case (MAT_COPY_VALUES): - GA_Copy(ctx->ga, newctx->ga); - break; - default: - GA_Fill(newctx->ga, &z); - break; - } - - GA_Pgroup_sync(newctx->gaGroup); - - return ierr; -} - -// ------------------------------------------------------------- -// MatConvertToDenseGA -// ------------------------------------------------------------- -PetscErrorCode -MatConvertToDenseGA(Mat A, Mat *B) -{ - PetscErrorCode ierr = 0; - MPI_Comm comm; - int lrows, grows, lcols, gcols; - - ierr = PetscObjectGetComm((PetscObject)A, &comm); CHKERRQ(ierr); - - ierr = MatGetSize(A, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(A, &lrows, &lcols); CHKERRQ(ierr); - - ierr = MatCreateDenseGA(comm, lrows, lcols, grows, gcols, B); CHKERRQ(ierr); - ierr = MatCopy(A, *B, SAME_NONZERO_PATTERN); CHKERRQ(ierr); - - - return ierr; -} - -// ------------------------------------------------------------- -// MatConvertGAtoDense -// ------------------------------------------------------------- -PetscErrorCode -MatConvertGAToDense(Mat A, Mat *B) -{ - PetscErrorCode ierr = 0; - MPI_Comm comm; - int nproc; - struct MatGACtx *ctx; - PetscInt lrows, grows, lcols, gcols, lo, hi; - - ierr = PetscObjectGetComm((PetscObject)A, &comm); CHKERRQ(ierr); - ierr = MPI_Comm_size(comm, &nproc); - - ierr = MatShellGetContext(A, &ctx); CHKERRQ(ierr); - - ierr = MatGetSize(A, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(A, &lrows, &lcols); CHKERRQ(ierr); - - ierr = MatCreate(comm, B); CHKERRQ(ierr); - ierr = MatSetSizes(*B, lrows, lcols, grows, gcols); CHKERRQ(ierr); - if (nproc == 1) { - ierr = MatSetType(*B, MATSEQDENSE); CHKERRQ(ierr); - ierr = MatSeqDenseSetPreallocation(*B, PETSC_NULL); CHKERRQ(ierr); - } else { - ierr = MatSetType(*B, MATDENSE); CHKERRQ(ierr); - ierr = MatMPIDenseSetPreallocation(*B, PETSC_NULL); CHKERRQ(ierr); - } - ierr = MatGetOwnershipRange(*B, &lo, &hi); CHKERRQ(ierr); - - std::vector cidx(gcols); - for (PetscInt c = 0; c < gcols; ++c) { - cidx[c] = c; - } - std::vector rowvals(gcols); - for (PetscInt r = lo; r < hi; ++r) { - int glo[2] = {r, 0}; - int ghi[2] = {r, gcols - 1}; - int ld[2] = {1,1}; - NGA_Get(ctx->ga, glo, ghi, &rowvals[0], ld); - ierr = MatSetValues(*B, 1, &r, gcols, &cidx[0], &rowvals[0], INSERT_VALUES); CHKERRQ(ierr); - } - - ierr = MatAssemblyBegin(*B, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - ierr = MatAssemblyEnd(*B, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - return ierr; -} - -// ------------------------------------------------------------- -// MatSetOperations_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatSetOperations_DenseGA(Mat A) -{ - PetscErrorCode ierr = 0; - ierr = MatShellSetOperation(A, MATOP_SET_VALUES, (void(*)(void))MatSetValues_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_GET_VALUES, (void(*)(void))MatGetValues_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_ZERO_ENTRIES, (void(*)(void))MatZeroEntries_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_MULT, (void(*)(void))MatMult_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_MAT_MULT, (void(*)(void))MatMatMult_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_TRANSPOSE, (void(*)(void))MatTranspose_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_DUPLICATE, (void(*)(void))MatDuplicate_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_ASSEMBLY_BEGIN, (void(*)(void))MatAssemmblyBegin_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_ASSEMBLY_END, (void(*)(void))MatAssemmblyEnd_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_DESTROY, (void(*)(void))MatDestroy_DenseGA); CHKERRQ(ierr); - return ierr; -} - diff --git a/src/math/petsc/ga_matrix.hpp b/src/math/petsc/ga_matrix.hpp deleted file mode 100644 index 90292987c..000000000 --- a/src/math/petsc/ga_matrix.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/* ------------------------------------------------------------- - * Copyright (c) 2013 Battelle Memorial Institute - * Licensed under modified BSD License. A copy of this license can be found - * in the LICENSE file in the top level directory of this distribution. - * ------------------------------------------------------------- */ -/** - * @file ga_matrix.h - * @author William A. Perkins - * @date 2017-10-06 13:26:43 d3g096 - * - * @brief - * - * - */ - -#ifndef _ga_matrix_h_ -#define _ga_matrix_h_ - -#include "petscmat.h" - - -/** - * Create a dense PETSc matrix that uses Global Arrays for data - * storage. - * - * @param comm - * @param m local number of rows (or PETSC_DECIDE) - * @param n local number of columns (or PETSC_DECIDE) - * @param M total number of rows (or PETSC_DETERMINE) - * @param N total number of columns (or PETSC_DETERMINE) - * @param A new PETSc matrix instance - * - * @return - */ -extern -PetscErrorCode -MatCreateDenseGA(MPI_Comm comm, - PetscInt m,PetscInt n,PetscInt M,PetscInt N, - Mat *A); - - -/// Convert a PETSc Matrix to a GA-based one -/** - * - * @param A - * @param B - * - * @return - */ -extern -PetscErrorCode -MatConvertToDenseGA(Mat A, Mat *B); - -/// Convert a GA-based Matrix to a PETSc dense matrix -extern -PetscErrorCode -MatConvertGAToDense(Mat A, Mat *B); - - -extern -PetscErrorCode -MatMultbyGA(const Mat& A, const Mat& B, Mat& C); - -extern -PetscErrorCode -MatMultbyGA_new(const Mat& A, const Mat& B, Mat& C); - - -#endif diff --git a/src/math/petsc/petsc_matrix_operations.cpp b/src/math/petsc/petsc_matrix_operations.cpp index b89bb6909..4dc2d9ac9 100644 --- a/src/math/petsc/petsc_matrix_operations.cpp +++ b/src/math/petsc/petsc_matrix_operations.cpp @@ -8,7 +8,7 @@ /** * @file petsc_matrix_operations.cpp * @author William A. Perkins - * @date 2019-12-03 08:14:38 d3g096 + * @date 2023-08-23 13:14:17 d3g096 * * @brief * @@ -26,7 +26,6 @@ #include "petsc/petsc_matrix_extractor.hpp" #include "petsc/petsc_vector_implementation.hpp" #include "petsc/petsc_vector_extractor.hpp" -#include "petsc/ga_matrix.hpp" namespace gridpack { namespace math { @@ -35,6 +34,8 @@ namespace math { // ------------------------------------------------------------- // transpose +// This should be avoided in favor of MatrixT<> = transpose(MatrixT<>) +// because this can cause additional mallocs // ------------------------------------------------------------- template void @@ -47,10 +48,14 @@ transpose(const MatrixT& A, MatrixT& result) Mat *pAtrans(PETScMatrix(result)); PetscErrorCode ierr(0); try { + // + ierr = MatSetOption(*pAtrans, MAT_NEW_NONZERO_LOCATION_ERR, PETSC_FALSE); CHKERRXX(ierr); ierr = MatTranspose(*pA, MAT_REUSE_MATRIX, pAtrans); CHKERRXX(ierr); } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); } + // This appears to be necessary with PETSc >= 3.12 + result.ready(); if (!PETScMatrixImplementation::useLibrary) { result.conjugate(); } @@ -129,6 +134,7 @@ transpose(const MatrixT& A) } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); } + result->ready(); if (!PETScMatrixImplementation::useLibrary) { result->conjugate(); } @@ -357,27 +363,6 @@ multiply(const MatrixT& A, VectorT& result); -// ------------------------------------------------------------- -// multiply_dense -// ------------------------------------------------------------- -static -PetscErrorCode -multiply_dense(const Mat& A, const Mat& B, Mat& C) -{ - PetscErrorCode ierr(0); - Mat Aga, Bga, Cga; - - ierr = MatConvertToDenseGA(A, &Aga); CHKERRQ(ierr); - ierr = MatConvertToDenseGA(B, &Bga); CHKERRQ(ierr); - ierr = MatMatMult(Aga, Bga, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &Cga); CHKERRQ(ierr); - ierr = MatConvertGAToDense(Cga, &C); CHKERRXX(ierr); - - ierr = MatDestroy(&Aga); CHKERRQ(ierr); - ierr = MatDestroy(&Bga); CHKERRQ(ierr); - ierr = MatDestroy(&Cga); CHKERRQ(ierr); - return ierr; -} - // ------------------------------------------------------------- // (Matrix) multiply // ------------------------------------------------------------- @@ -387,16 +372,6 @@ multiply(const MatrixT& A, const MatrixT& B, MatrixT& result) { PetscErrorCode ierr(0); - // special method required for parallel dense*dense - if (A.communicator().size() > 1 && - A.storageType() == Dense && - B.storageType() == Dense) { - const Mat *Amat(PETScMatrix(A)); - const Mat *Bmat(PETScMatrix(B)); - Mat *Cmat(PETScMatrix(result)); - ierr = MatMultbyGA(*Amat, *Bmat, *Cmat); CHKERRXX(ierr); - // ierr = multiply_dense(*Amat, *Bmat, *Cmat); CHKERRXX(ierr); - } else { const Mat *Amat(PETScMatrix(A)); const Mat *Bmat(PETScMatrix(B)); Mat *Cmat(PETScMatrix(result)); @@ -407,7 +382,7 @@ multiply(const MatrixT& A, const MatrixT& B, MatrixT& result) } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); } - } + } template @@ -428,19 +403,6 @@ multiply(const MatrixT& A, const MatrixT& B) { PetscErrorCode ierr(0); MatrixT *result; - // special method required for parallel dense*dense - if (A.communicator().size() > 1 && - A.storageType() == Dense && - B.storageType() == Dense) { - const Mat *Amat(PETScMatrix(A)); - const Mat *Bmat(PETScMatrix(B)); - Mat Cmat; - ierr = MatMultbyGA_new(*Amat, *Bmat, Cmat); - - PETScMatrixImplementation *result_impl = - new PETScMatrixImplementation(Cmat, false, true); - result = new MatrixT(result_impl); - } else { const Mat *Amat(PETScMatrix(A)); const Mat *Bmat(PETScMatrix(B)); Mat Cmat; @@ -454,7 +416,7 @@ multiply(const MatrixT& A, const MatrixT& B) PETScMatrixImplementation *result_impl = new PETScMatrixImplementation(Cmat, false, true); result = new MatrixT(result_impl); - } + return result; } diff --git a/src/math/test/petsc_ga_matrix.cpp b/src/math/test/petsc_ga_matrix.cpp deleted file mode 100644 index f815b1727..000000000 --- a/src/math/test/petsc_ga_matrix.cpp +++ /dev/null @@ -1,358 +0,0 @@ -// ------------------------------------------------------------- -/* - * Copyright (c) 2013 Battelle Memorial Institute - * Licensed under modified BSD License. A copy of this license can be found - * in the LICENSE file in the top level directory of this distribution. - */ - -// ------------------------------------------------------------- -// ------------------------------------------------------------- -/** - * @file petsc_ga_matrix.cpp - * @author William A. Perkins - * @date 2019-08-01 08:51:08 d3g096 - * - * @brief - * - * - */ -// ------------------------------------------------------------- - -#include - -#include "math.hpp" -#include "gridpack/parallel/parallel.hpp" -#include "petsc/ga_matrix.hpp" -#include "petsc/petsc_exception.hpp" - -#include "test_main.cpp" - -static const PetscInt local_size(5); - -static -PetscErrorCode -fill_pattern(Mat A, InsertMode addv) -{ - PetscErrorCode ierr(0); - PetscScalar x(0.0); - PetscInt lo, hi; - - ierr = MatGetOwnershipRange(A, &lo, &hi); CHKERRQ(ierr); - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - ierr = MatSetValues(A, 1, &i, 1, &j, &x, addv); CHKERRQ(ierr); - x += 1.0; - } - } - - ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - return ierr; -} - -static -void -convert_and_check(Mat A) -{ - PetscErrorCode ierr(0); - PetscInt lo, hi; - Mat B; - - ierr = MatConvertToDenseGA(A, &B); CHKERRXX(ierr); - ierr = MatGetOwnershipRange(B, &lo, &hi); CHKERRXX(ierr); - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - PetscScalar x; - PetscScalar y; - ierr = MatGetValues(A, 1, &i, 1, &j, &x); CHKERRXX(ierr); - ierr = MatGetValues(B, 1, &i, 1, &j, &y); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(x, y); - } - } - ierr = MatDestroy(&B); CHKERRXX(ierr); -} - -BOOST_AUTO_TEST_SUITE(GAMatrixTest) - -BOOST_AUTO_TEST_CASE( ConstructConvertDuplicate ) -{ - PetscErrorCode ierr(0); - gridpack::parallel::Communicator world; - - Mat A, B, C; - PetscScalar x(0.0); - PetscInt lrows, lcols; - PetscInt lo, hi; - - - BOOST_TEST_MESSAGE("Create Matrix"); - - ierr = MatCreateDenseGA(world, local_size, local_size, PETSC_DETERMINE, PETSC_DETERMINE, &A); CHKERRXX(ierr); - ierr = MatGetLocalSize(A, &lrows, &lcols); CHKERRXX(ierr); - - BOOST_CHECK_EQUAL(lrows, 5); - BOOST_CHECK_EQUAL(lcols, 5); - - BOOST_TEST_MESSAGE("Fill Matrix"); - - ierr = fill_pattern(A, INSERT_VALUES); CHKERRXX(ierr); - - BOOST_TEST_MESSAGE("Get Matrix Values"); - - ierr = MatGetOwnershipRange(A, &lo, &hi); CHKERRXX(ierr); - PetscScalar y(0.0); - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - ierr = MatGetValues(A, 1, &i, 1, &j, &x); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(x, y); - y += 1.0; - } - } - - BOOST_TEST_MESSAGE("Add Matrix Values"); - - ierr = fill_pattern(A, ADD_VALUES); CHKERRXX(ierr); - - BOOST_TEST_MESSAGE("Get Matrix Values"); - - y = 0.0; - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - ierr = MatGetValues(A, 1, &i, 1, &j, &x); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(x, 2.0*y); - y += 1.0; - } - } - - BOOST_TEST_MESSAGE("Convert Matrix"); - - ierr = MatConvert(A, MATDENSE, MAT_INITIAL_MATRIX, &B); CHKERRXX(ierr); - - BOOST_TEST_MESSAGE("View Converted Matrix"); - - ierr = MatView(B, PETSC_VIEWER_STDOUT_WORLD); CHKERRXX(ierr); - - BOOST_TEST_MESSAGE("Get Matrix Values"); - - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - ierr = MatGetValues(A, 1, &i, 1, &j, &x); CHKERRXX(ierr); - ierr = MatGetValues(B, 1, &i, 1, &j, &y); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(x, y); - } - } - ierr = MatDestroy(&B); CHKERRXX(ierr); - - BOOST_TEST_MESSAGE("Duplicate Matrix"); - ierr = MatDuplicate(A, MAT_COPY_VALUES, &C); CHKERRXX(ierr); - - BOOST_TEST_MESSAGE("Get Matrix Values"); - - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - ierr = MatGetValues(A, 1, &i, 1, &j, &x); CHKERRXX(ierr); - ierr = MatGetValues(C, 1, &i, 1, &j, &y); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(x, y); - } - } - ierr = MatDestroy(&C); CHKERRXX(ierr); - - ierr = MatDestroy(&A); CHKERRXX(ierr); -} - -BOOST_AUTO_TEST_CASE( Dense2GA ) -{ - PetscErrorCode ierr(0); - gridpack::parallel::Communicator comm; - - Mat A; - PetscInt lrows = 5, lcols = 5; - - ierr = MatCreate(comm, &A); CHKERRXX(ierr); - ierr = MatSetSizes(A, lrows, lcols, PETSC_DETERMINE, PETSC_DETERMINE); CHKERRXX(ierr); - ierr = MatSetType(A, MATDENSE); CHKERRXX(ierr); - ierr = MatSetFromOptions(A); CHKERRXX(ierr); - ierr = MatSetUp(A); CHKERRXX(ierr); - - ierr = fill_pattern(A, INSERT_VALUES); CHKERRXX(ierr); - - convert_and_check(A); - - ierr = MatDestroy(&A); CHKERRXX(ierr); -} - -BOOST_AUTO_TEST_CASE( Sparse2GA ) -{ - PetscErrorCode ierr(0); - gridpack::parallel::Communicator comm; - - Mat A; - PetscInt lrows = 5, lcols = 5; - - ierr = MatCreate(comm, &A); CHKERRXX(ierr); - ierr = MatSetSizes(A, lrows, lcols, PETSC_DETERMINE, PETSC_DETERMINE); CHKERRXX(ierr); - ierr = MatSetType(A, MATAIJ); CHKERRXX(ierr); - ierr = MatSetFromOptions(A); CHKERRXX(ierr); - ierr = MatSetUp(A); CHKERRXX(ierr); - - ierr = fill_pattern(A, INSERT_VALUES); CHKERRXX(ierr); - - convert_and_check(A); - - ierr = MatDestroy(&A); CHKERRXX(ierr); -} - - -BOOST_AUTO_TEST_CASE( VectorMultiply ) -{ - PetscErrorCode ierr(0); - gridpack::parallel::Communicator world; - - Mat A; - PetscInt lrows, lcols; - PetscInt grows, gcols; - PetscInt lo, hi; - - ierr = MatCreateDenseGA(world, local_size, local_size, PETSC_DETERMINE, PETSC_DETERMINE, &A); CHKERRXX(ierr); - ierr = MatGetLocalSize(A, &lrows, &lcols); CHKERRXX(ierr); - ierr = MatGetSize(A, &grows, &gcols); CHKERRXX(ierr); - - BOOST_CHECK_EQUAL(lrows, local_size); - BOOST_CHECK_EQUAL(lcols, local_size); - - PetscScalar v(2.0); - ierr = MatGetOwnershipRange(A, &lo, &hi); CHKERRXX(ierr); - for (int i = lo; i < hi; ++i) { - ierr = MatSetValues(A, 1, &i, 1, &i, &v, INSERT_VALUES); CHKERRXX(ierr); - } - ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY); CHKERRXX(ierr); - ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY); CHKERRXX(ierr); - - Vec x, y; - ierr = VecCreate(world, &x); CHKERRXX(ierr); - ierr = VecSetSizes(x, lcols, gcols); CHKERRXX(ierr); - ierr = VecSetFromOptions(x); CHKERRXX(ierr); - v = 1.0; - ierr = VecSet(x, v); CHKERRXX(ierr); - ierr = VecAssemblyBegin(x); - ierr = VecAssemblyEnd(x); - - ierr = VecCreate(world, &y); CHKERRXX(ierr); - ierr = VecSetSizes(y, lrows, grows); CHKERRXX(ierr); - ierr = VecSetFromOptions(y); CHKERRXX(ierr); - v = 0.0; - ierr = VecSet(y, v); CHKERRXX(ierr); - ierr = VecAssemblyBegin(y); - ierr = VecAssemblyEnd(y); - - ierr = MatMult(A, x, y); CHKERRXX(ierr); - - ierr = VecGetOwnershipRange(y, &lo, &hi); CHKERRXX(ierr); - v = 2.0; - for (int i = lo; i < hi; ++i) { - PetscScalar xtmp; - ierr = VecGetValues(y, 1, &i, &xtmp); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(v, xtmp); - } - - ierr = VecDestroy(&x); CHKERRXX(ierr); - ierr = VecDestroy(&y); CHKERRXX(ierr); - ierr = MatDestroy(&A); CHKERRXX(ierr); -} - -BOOST_AUTO_TEST_CASE( SquareMatrixMultiply ) -{ - Mat A, B, C, Cbase; - PetscErrorCode ierr(0); - gridpack::parallel::Communicator comm; - - // do a multiply with a pair of normal sparse matrices for a check - // (dense multiply may not be available in parallel) - - ierr = MatCreate(comm, &A); CHKERRXX(ierr); - ierr = MatSetSizes(A, local_size, local_size, PETSC_DETERMINE, PETSC_DETERMINE); CHKERRXX(ierr); - ierr = MatSetType(A, MATAIJ); CHKERRXX(ierr); - ierr = MatSetFromOptions(A); CHKERRXX(ierr); - ierr = MatSetUp(A); CHKERRXX(ierr); - ierr = fill_pattern(A, INSERT_VALUES); CHKERRXX(ierr); - ierr = MatDuplicate(A, MAT_COPY_VALUES, &B); CHKERRXX(ierr); - ierr = MatMatMult(A, B, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &Cbase); CHKERRXX(ierr); - - ierr = MatDestroy(&A); CHKERRXX(ierr); - ierr = MatDestroy(&B); CHKERRXX(ierr); - - // Do the same multiply with GA-based matrices - - ierr = MatCreateDenseGA(comm, local_size, local_size, PETSC_DETERMINE, PETSC_DETERMINE, &A); - CHKERRXX(ierr); - ierr = fill_pattern(A, INSERT_VALUES); CHKERRXX(ierr); - ierr = MatDuplicate(A, MAT_COPY_VALUES, &B); CHKERRXX(ierr); - - ierr = MatMatMult(A, B, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &C); - PetscInt lo, hi; - ierr = MatGetOwnershipRange(Cbase, &lo, &hi); CHKERRXX(ierr); - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - PetscScalar x; - PetscScalar y; - ierr = MatGetValues(Cbase, 1, &i, 1, &j, &x); CHKERRXX(ierr); - ierr = MatGetValues(C, 1, &i, 1, &j, &y); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(x, y); - } - } - - ierr = MatDestroy(&A); CHKERRXX(ierr); - ierr = MatDestroy(&B); CHKERRXX(ierr); - ierr = MatDestroy(&C); CHKERRXX(ierr); - ierr = MatDestroy(&Cbase); CHKERRXX(ierr); - -} - -BOOST_AUTO_TEST_CASE(Transpose) -{ - PetscErrorCode ierr(0); - gridpack::parallel::Communicator world; - - Mat A, Atrans; - PetscInt lrows, lcols; - PetscInt grows, gcols; - PetscInt lo, hi; - - ierr = MatCreateDenseGA(world, local_size, local_size, PETSC_DETERMINE, PETSC_DETERMINE, &A); CHKERRXX(ierr); - ierr = MatGetLocalSize(A, &lrows, &lcols); CHKERRXX(ierr); - ierr = MatGetSize(A, &grows, &gcols); CHKERRXX(ierr); - - BOOST_CHECK_EQUAL(lrows, local_size); - BOOST_CHECK_EQUAL(lcols, local_size); - - PetscScalar v; - ierr = MatGetOwnershipRange(A, &lo, &hi); CHKERRXX(ierr); - for (int i = lo; i < hi; ++i) { - v = (PetscScalar)i; - for (int j = lo; j < hi; ++j) { - ierr = MatSetValues(A, 1, &i, 1, &j, &v, INSERT_VALUES); CHKERRXX(ierr); - } - } - ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY); CHKERRXX(ierr); - ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY); CHKERRXX(ierr); - - ierr = MatTranspose(A, MAT_INITIAL_MATRIX, &Atrans); CHKERRXX(ierr); - - PetscScalar x; - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - v = (PetscScalar)j; - ierr = MatGetValues(Atrans, 1, &i, 1, &j, &x); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(v, x); - } - } - - ierr = MatDestroy(&Atrans); CHKERRXX(ierr); - ierr = MatDestroy(&A); CHKERRXX(ierr); -} - -BOOST_AUTO_TEST_SUITE_END() - - - - From e85428a1db99fe043d001408a769f76086270fdd Mon Sep 17 00:00:00 2001 From: William Perkins Date: Thu, 24 Aug 2023 10:41:33 -0700 Subject: [PATCH 013/195] Remove all PETSC_VERSION usage. --- src/math/petsc/petsc_configurable.cpp | 6 +- .../petsc/petsc_dae_solver_implementation.hpp | 59 +------------------ src/math/petsc/petsc_exception.hpp | 19 +----- .../petsc/petsc_linear_matrix_solver_impl.hpp | 12 +--- .../petsc_linear_solver_implementation.hpp | 6 +- .../petsc/petsc_matrix_implementation.hpp | 27 +-------- src/math/petsc/petsc_matrix_operations.cpp | 8 +-- src/math/petsc/petsc_matrix_wrapper.cpp | 26 +------- .../petsc_nonlinear_solver_implementation.hpp | 36 +---------- src/math/petsc/petsc_vector_wrapper.cpp | 9 +-- src/math/vector_implementation.hpp | 3 +- 11 files changed, 14 insertions(+), 197 deletions(-) diff --git a/src/math/petsc/petsc_configurable.cpp b/src/math/petsc/petsc_configurable.cpp index 23d2879e6..d9389df71 100644 --- a/src/math/petsc/petsc_configurable.cpp +++ b/src/math/petsc/petsc_configurable.cpp @@ -9,7 +9,7 @@ /** * @file petsc_configurable.cpp * @author William A. Perkins - * @date 2016-06-16 12:24:02 d3g096 + * @date 2023-08-24 09:41:08 d3g096 * * @brief * @@ -173,9 +173,7 @@ PETScConfigurable::p_processOptions(utility::Configuration::CursorPtr props) PetscErrorCode ierr(0); try { ierr = PetscOptionsInsertString( -#if PETSC_VERSION_GE(3,7,0) NULL, -#endif p_loadedOptions.c_str()); CHKERRXX(ierr); } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); @@ -202,9 +200,7 @@ PETScConfigurable::p_unprocessOptions(void) << *o << "\"" << std::endl; } ierr = PetscOptionsClearValue( -#if PETSC_VERSION_GE(3,7,0) NULL, -#endif o->c_str()); } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); diff --git a/src/math/petsc/petsc_dae_solver_implementation.hpp b/src/math/petsc/petsc_dae_solver_implementation.hpp index f3fa65482..f10886fda 100644 --- a/src/math/petsc/petsc_dae_solver_implementation.hpp +++ b/src/math/petsc/petsc_dae_solver_implementation.hpp @@ -10,7 +10,7 @@ /** * @file petsc_dae_solver_implementation.hpp * @author William A. Perkins - * @date 2019-12-05 10:01:29 d3g096 + * @date 2023-08-24 09:42:12 d3g096 * * @brief * @@ -156,15 +156,9 @@ class PETScDAESolverImplementation pterm[i] = (gterm[i] ? PETSC_TRUE : PETSC_FALSE); } -#if PETSC_VERSION_LT(3,7,0) - ierr = TSSetEventMonitor(p_ts, ne, &(pdir[0]), &(pterm[0]), - &EventHandler, &PostEventHandler, - NULL); -#else ierr = TSSetEventHandler(p_ts, ne, &(pdir[0]), &(pterm[0]), &EventHandler, &PostEventHandler, NULL); -#endif } ierr = TSSetProblemType(p_ts, TS_NONLINEAR); CHKERRXX(ierr); @@ -201,12 +195,8 @@ class PETScDAESolverImplementation { PetscErrorCode ierr(0); try { -#if PETSC_VERSION_LT(3,8,0) - ierr = TSSetInitialTimeStep(p_ts, t0, deltat0); CHKERRXX(ierr); -#else ierr = TSSetTime(p_ts, t0); CHKERRXX(ierr); ierr = TSSetTimeStep(p_ts, deltat0); CHKERRXX(ierr); -#endif Vec *xvec(PETScVector(x0)); ierr = TSSetSolution(p_ts, *xvec); } catch (const PETSC_EXCEPTION_TYPE& e) { @@ -221,13 +211,8 @@ class PETScDAESolverImplementation { PetscErrorCode ierr(0); try { -#if PETSC_VERSION_LT(3,8,0) - ierr = TSSetDuration(p_ts, maxsteps, maxtime); CHKERRXX(ierr); -#else ierr = TSSetMaxSteps(p_ts, maxsteps); CHKERRXX(ierr); ierr = TSSetMaxTime(p_ts, maxtime); CHKERRXX(ierr); -#endif - ierr = TSSetExactFinalTime(p_ts, TS_EXACTFINALTIME_MATCHSTEP); CHKERRXX(ierr); ierr = TSSolve(p_ts, PETSC_NULL); // std::cout << this->processor_rank() << ": " @@ -239,11 +224,7 @@ class PETScDAESolverImplementation ierr = TSGetConvergedReason(p_ts, &reason); CHKERRXX(ierr); PetscInt nstep; -#if PETSC_VERSION_LT(3,8,0) - ierr = TSGetTimeStepNumber(p_ts,&nstep);CHKERRXX(ierr); -#else ierr = TSGetStepNumber(p_ts,&nstep);CHKERRXX(ierr); -#endif maxsteps = nstep; if (reason >= 0) { @@ -285,42 +266,6 @@ class PETScDAESolverImplementation } } - -#if PETSC_VERSION_LT(3,5,0) - - /// Routine to assemble Jacobian that is sent to PETSc - static PetscErrorCode FormIJacobian(TS ts, PetscReal t, Vec x, Vec xdot, - PetscReal a, Mat *jac, Mat *B, - MatStructure *flag, void *dummy) - { - PetscErrorCode ierr(0); - - // Necessary C cast - PETScDAESolverImplementation *solver = - (PETScDAESolverImplementation *)dummy; - - // Copy PETSc's current estimate into - - // Should be the case, but just make sure - BOOST_ASSERT(*jac == *solver->p_petsc_J); - BOOST_ASSERT(*B == *solver->p_petsc_J); - - boost::scoped_ptr - xtmp(new VectorType(new PETScVectorImplementation(x, false))), - xdottmp(new VectorType(new PETScVectorImplementation(xdot, false))); - - // Call the user-specified function (object) to form the Jacobian - (solver->p_Jbuilder)(t, *xtmp, *xdottmp, a, *(solver->p_J)); - - *flag = SAME_NONZERO_PATTERN; - - return ierr; - - } - - -#else - /// Routine to assemble Jacobian that is sent to PETSc static PetscErrorCode FormIJacobian(TS ts, PetscReal t, Vec x, Vec xdot, PetscReal a, Mat jac, Mat B, @@ -350,8 +295,6 @@ class PETScDAESolverImplementation } -#endif - /// Routine to assemble RHS that is sent to PETSc static PetscErrorCode FormIFunction(TS ts, PetscReal t, Vec x, Vec xdot, Vec F, void *dummy) diff --git a/src/math/petsc/petsc_exception.hpp b/src/math/petsc/petsc_exception.hpp index a11a492da..edb6313d6 100644 --- a/src/math/petsc/petsc_exception.hpp +++ b/src/math/petsc/petsc_exception.hpp @@ -8,7 +8,7 @@ /** * @file petsc_exception.hpp * @author William A. Perkins - * @date 2019-08-13 09:12:24 d3g096 + * @date 2023-08-24 09:43:26 d3g096 * * @brief * @@ -26,29 +26,12 @@ // You gotta love PETSc consistency -#if PETSC_VERSION_(3,4,0) -#undef PETSC_VERSION_RELEASE -#define PETSC_VERSION_RELEASE 0 -#endif - // Default type for PETSc exceptions #define PETSC_EXCEPTION_TYPE std::runtime_error // If PETSc is compiled with the C++, use its C++ exception // facility. Otherwise, we need to make our own. -#ifdef PETSC_CLANGUAGE_CXX -// With PETSc version 3.5, PETSc::Exception was no longer defined. It -// was replaced with std::runtime_error. - -#if PETSC_VERSION_LT(3,5,0) -#include -#undef PETSC_EXCEPTION_TYPE -#define PETSC_EXCEPTION_TYPE PETSc::Exception -#define GRIDPACK_USES_PETSC_EXCEPTION 1 -#endif -#endif - #include "gridpack/utilities/exception.hpp" namespace gridpack { diff --git a/src/math/petsc/petsc_linear_matrix_solver_impl.hpp b/src/math/petsc/petsc_linear_matrix_solver_impl.hpp index 4c419af3a..ae577178c 100644 --- a/src/math/petsc/petsc_linear_matrix_solver_impl.hpp +++ b/src/math/petsc/petsc_linear_matrix_solver_impl.hpp @@ -10,7 +10,7 @@ /** * @file petsc_linear_matrx_solver_impl.hpp * @author William A. Perkins - * @date 2019-12-03 08:18:32 d3g096 + * @date 2023-08-24 09:43:53 d3g096 * * @brief * @@ -84,11 +84,7 @@ class PetscLinearMatrixSolverImplementation /// Choose a matrix solver type based on PETSc version typedef -#if PETSC_VERSION_LT(3,9,0) - MatSolverPackage -#else MatSolverType -#endif ThePetscMatSolverType; /// The underlying PETSc factored coefficient matrix @@ -292,12 +288,6 @@ PetscLinearMatrixSolverImplementation::p_supportedOrderingType[] = { MATORDERINGRCM, MATORDERINGQMD, MATORDERINGROWLENGTH -#if PETSC_VERSION_GT(3,5,0) - , - MATORDERINGWBM, - MATORDERINGSPECTRAL, - MATORDERINGAMD -#endif }; template diff --git a/src/math/petsc/petsc_linear_solver_implementation.hpp b/src/math/petsc/petsc_linear_solver_implementation.hpp index 0309dc9e1..33569c404 100644 --- a/src/math/petsc/petsc_linear_solver_implementation.hpp +++ b/src/math/petsc/petsc_linear_solver_implementation.hpp @@ -9,7 +9,7 @@ /** * @file petsc_linear_solver_implementation.hpp * @author William A. Perkins - * @date 2015-08-18 13:40:01 d3g096 + * @date 2023-08-24 09:44:02 d3g096 * * @brief * @@ -121,11 +121,7 @@ class PETScLinearSolverImplementation if (p_matrixSet && this->p_constSerialMatrix) { // KSPSetOperators can be skipped } else { -#if PETSC_VERSION_LT(3,5,0) - ierr = KSPSetOperators(p_KSP, *Amat, *Amat, SAME_NONZERO_PATTERN); CHKERRXX(ierr); -#else ierr = KSPSetOperators(p_KSP, *Amat, *Amat); CHKERRXX(ierr); -#endif p_matrixSet = true; } diff --git a/src/math/petsc/petsc_matrix_implementation.hpp b/src/math/petsc/petsc_matrix_implementation.hpp index e83515af2..91314e809 100644 --- a/src/math/petsc/petsc_matrix_implementation.hpp +++ b/src/math/petsc/petsc_matrix_implementation.hpp @@ -9,7 +9,7 @@ /** * @file petsc_matrix_implementation.h * @author William A. Perkins - * @date 2019-12-03 08:14:08 d3g096 + * @date 2023-08-24 09:44:50 d3g096 * * @brief * @@ -501,16 +501,10 @@ class PETScMatrixImplementation ridx[i] = rows[i]*elementSize; } -#if PETSC_VERSION_LT(3,6,0) - std::vector permidx(sortPermutation(ridx)); - ierr = ISCreateGeneral(PETSC_COMM_SELF, nrow, &permidx[0], - PETSC_COPY_VALUES, &irowperm); CHKERRXX(ierr); -#else ierr = ISCreateGeneral(PETSC_COMM_SELF, nrow, &ridx[0], PETSC_COPY_VALUES, &irow); CHKERRXX(ierr); ierr = ISSortPermutation(irow, PETSC_TRUE, &irowperm); CHKERRXX(ierr); ierr = ISDestroy(&irow); CHKERRXX(ierr); -#endif ierr = ISSetPermutation(irowperm); CHKERRXX(ierr); ierr = ISInvertPermutation(irowperm, PETSC_DECIDE, &irowinvert); CHKERRXX(ierr); ierr = ISDestroy(&irowperm); CHKERRXX(ierr); @@ -534,12 +528,8 @@ class PETScMatrixImplementation // ierr = PetscViewerDestroy(&v); CHKERRXX(ierr); Mat *sub; -#if PETSC_VERSION_LT(3,8,0) - ierr = MatGetSubMatrices(*mat, 1, &irow, &icol, MAT_INITIAL_MATRIX, &sub); CHKERRXX(ierr); -#else ierr = MatCreateSubMatrices(*mat, 1, &irow, &icol, MAT_INITIAL_MATRIX, &sub); CHKERRXX(ierr); - -#endif + std::vector cval(ncol); for (PetscInt j = 0; j < ncol; ++j) { cidx[j] = j; @@ -570,11 +560,7 @@ class PETScMatrixImplementation ierr = ISDestroy(&irow); CHKERRXX(ierr); ierr = ISDestroy(&icol); CHKERRXX(ierr); -#if PETSC_VERSION_LT(3,8,0) - ierr = MatDestroyMatrices(1, &sub); CHKERRXX(ierr); -#else ierr = MatDestroySubMatrices(1, &sub); CHKERRXX(ierr); -#endif } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); @@ -700,11 +686,7 @@ class PETScMatrixImplementation ierr = ISCreateStride(this->communicator(), p_mwrap->rows(), 0, 1, &irow); CHKERRXX(ierr); ierr = ISCreateStride(this->communicator(), p_mwrap->cols(), 0, 1, &icol); CHKERRXX(ierr); -#if PETSC_VERSION_LT(3,8,0) - ierr = MatGetSubMatrices(*Aorig, 1, &irow, &icol, MAT_INITIAL_MATRIX, &Aloc); CHKERRXX(ierr); -#else ierr = MatCreateSubMatrices(*Aorig, 1, &irow, &icol, MAT_INITIAL_MATRIX, &Aloc); CHKERRXX(ierr); -#endif // MatType mt; // ierr = MatGetType(*Aloc, &mt); CHKERRXX(ierr); @@ -716,12 +698,7 @@ class PETScMatrixImplementation ierr = ISDestroy(&irow); CHKERRXX(ierr); ierr = ISDestroy(&icol); CHKERRXX(ierr); -#if PETSC_VERSION_LT(3,8,0) - ierr = MatDestroyMatrices(1, &Aloc); CHKERRXX(ierr); -#else ierr = MatDestroySubMatrices(1, &Aloc); CHKERRXX(ierr); -#endif - return result; } diff --git a/src/math/petsc/petsc_matrix_operations.cpp b/src/math/petsc/petsc_matrix_operations.cpp index 4dc2d9ac9..52367142a 100644 --- a/src/math/petsc/petsc_matrix_operations.cpp +++ b/src/math/petsc/petsc_matrix_operations.cpp @@ -8,7 +8,7 @@ /** * @file petsc_matrix_operations.cpp * @author William A. Perkins - * @date 2023-08-23 13:14:17 d3g096 + * @date 2023-08-24 09:45:38 d3g096 * * @brief * @@ -559,16 +559,10 @@ matrixLoadBinary(const parallel::Communicator& comm, const char* filename) filename, FILE_MODE_READ, &viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE); CHKERRXX(ierr); -#else - ierr = PetscViewerSetFormat(viewer, PETSC_VIEWER_NATIVE); CHKERRXX(ierr); -#endif ierr = MatLoad(mat, viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPopFormat(viewer); CHKERRXX(ierr); -#endif ierr = PetscViewerDestroy(&viewer); CHKERRXX(ierr); PETScMatrixImplementation *result_impl = diff --git a/src/math/petsc/petsc_matrix_wrapper.cpp b/src/math/petsc/petsc_matrix_wrapper.cpp index 82e1fffe3..d87fbee45 100644 --- a/src/math/petsc/petsc_matrix_wrapper.cpp +++ b/src/math/petsc/petsc_matrix_wrapper.cpp @@ -9,7 +9,7 @@ /** * @file petsc_matrix_wrapper.cpp * @author William A. Perkins - * @date 2019-09-13 12:53:47 d3g096 + * @date 2023-08-24 09:46:49 d3g096 * * @brief * @@ -472,22 +472,14 @@ petsc_print_matrix(const Mat mat, const char* filename, PetscViewerFormat format } else { ierr = PetscViewerASCIIGetStdout(comm, &viewer); CHKERRXX(ierr); } -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPushFormat(viewer, format); -#else - ierr = PetscViewerSetFormat(viewer, format); -#endif CHKERRXX(ierr); PetscInt grow, gcol, lrow, lcol; switch (format) { case PETSC_VIEWER_DEFAULT: ierr = MatGetSize(mat, &grow, &gcol); CHKERRXX(ierr); ierr = MatGetLocalSize(mat, &lrow, &lcol); CHKERRXX(ierr); -#if PETSC_VERSION_LT(3,7,0) - ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE); CHKERRXX(ierr); -#else ierr = PetscViewerASCIIPushSynchronized(viewer); CHKERRXX(ierr); -#endif ierr = PetscViewerASCIIPrintf(viewer, "# Matrix distribution\n"); CHKERRXX(ierr); ierr = PetscViewerASCIIPrintf(viewer, @@ -502,16 +494,12 @@ petsc_print_matrix(const Mat mat, const char* filename, PetscViewerFormat format "# ---- -------- --------\n"); CHKERRXX(ierr); ierr = PetscViewerASCIIPrintf(viewer, "# %4d %8d %8d\n", nproc, grow, gcol); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerASCIIPopSynchronized(viewer); CHKERRXX(ierr); -#endif default: break; } ierr = MatView(mat, viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPopFormat(viewer); CHKERRXX(ierr); -#endif if (filename != NULL) { ierr = PetscViewerDestroy(&viewer); CHKERRXX(ierr); @@ -557,16 +545,10 @@ PetscMatrixWrapper::loadBinary(const char* filename) filename, FILE_MODE_READ, &viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE); -#else - ierr = PetscViewerSetFormat(viewer, PETSC_VIEWER_NATIVE); -#endif CHKERRXX(ierr); ierr = MatLoad(p_matrix, viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPopFormat(viewer); CHKERRXX(ierr); -#endif ierr = PetscViewerDestroy(&viewer); CHKERRXX(ierr); } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); @@ -587,16 +569,10 @@ PetscMatrixWrapper::saveBinary(const char* filename) const filename, FILE_MODE_WRITE, &viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE); -#else - ierr = PetscViewerSetFormat(viewer, PETSC_VIEWER_NATIVE); -#endif CHKERRXX(ierr); ierr = MatView(p_matrix, viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPopFormat(viewer); CHKERRXX(ierr); -#endif ierr = PetscViewerDestroy(&viewer); CHKERRXX(ierr); } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); diff --git a/src/math/petsc/petsc_nonlinear_solver_implementation.hpp b/src/math/petsc/petsc_nonlinear_solver_implementation.hpp index 375ba0c51..f7eb3b240 100644 --- a/src/math/petsc/petsc_nonlinear_solver_implementation.hpp +++ b/src/math/petsc/petsc_nonlinear_solver_implementation.hpp @@ -8,7 +8,7 @@ /** * @file petsc_nonlinear_solver_implementation.hpp * @author William A. Perkins - * @date 2015-03-26 14:25:10 d3g096 + * @date 2023-08-24 09:47:08 d3g096 * * @brief * @@ -188,38 +188,6 @@ class PetscNonlinearSolverImplementation } -#if PETSC_VERSION_LT(3,5,0) - /// Routine to assemble Jacobian that is sent to PETSc - static PetscErrorCode FormJacobian(SNES snes, Vec x, Mat *jac, Mat *B, - MatStructure *flag, void *dummy) - { - PetscErrorCode ierr(0); - - // Necessary C cast - PetscNonlinearSolverImplementation *solver = - (PetscNonlinearSolverImplementation *)dummy; - - // Copy PETSc's current estimate into - - // Should be the case, but just make sure - BOOST_ASSERT(*jac == *solver->p_petsc_J); - BOOST_ASSERT(*B == *solver->p_petsc_J); - - // Not sure about this - BOOST_ASSERT(x == *(solver->p_petsc_X)); - - // May need to do this, which seems slow. - // ierr = VecCopy(x, *(solver->p_petsc_X)); CHKERRQ(ierr); - - // Call the user-specified function (object) to form the Jacobian - (solver->p_jacobian)(*(solver->p_X), *(solver->p_J)); - - *flag = SAME_NONZERO_PATTERN; - - return ierr; - } - -#else /// Routine to assemble Jacobian that is sent to PETSc static PetscErrorCode FormJacobian(SNES snes, Vec x, Mat jac, Mat B, void *dummy) @@ -248,8 +216,6 @@ class PetscNonlinearSolverImplementation return ierr; } -#endif - /// Routine to assemble RHS that is sent to PETSc static PetscErrorCode FormFunction(SNES snes, Vec x, Vec f, void *dummy) { diff --git a/src/math/petsc/petsc_vector_wrapper.cpp b/src/math/petsc/petsc_vector_wrapper.cpp index 8c77fa39a..15e5bd893 100644 --- a/src/math/petsc/petsc_vector_wrapper.cpp +++ b/src/math/petsc/petsc_vector_wrapper.cpp @@ -8,7 +8,7 @@ /** * @file petsc_vector_implementation.cpp * @author William A. Perkins - * @date 2019-09-13 13:56:23 d3g096 + * @date 2023-08-24 09:47:21 d3g096 * * @brief * @@ -362,16 +362,11 @@ petsc_print_vector(const Vec vec, const char* filename, PetscViewerFormat format } else { ierr = PetscViewerASCIIGetStdout(comm, &viewer); CHKERRXX(ierr); } -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPushFormat(viewer, format); -#else - ierr = PetscViewerSetFormat(viewer, format); -#endif CHKERRXX(ierr); ierr = VecView(vec, viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPopFormat(viewer); CHKERRXX(ierr); -#endif + if (filename != NULL) { ierr = PetscViewerDestroy(&viewer); CHKERRXX(ierr); } diff --git a/src/math/vector_implementation.hpp b/src/math/vector_implementation.hpp index 06aae47b1..bf2553cc8 100644 --- a/src/math/vector_implementation.hpp +++ b/src/math/vector_implementation.hpp @@ -9,7 +9,7 @@ /** * @file vector_implementation.h * @author William A. Perkins - * @date 2019-11-20 10:01:00 d3g096 + * @date 2023-08-23 14:53:57 d3g096 * * @brief * @@ -162,6 +162,7 @@ class VectorImplementation IdxType lo, hi; this->localIndexRange(lo, hi); this->setElementRange(lo, hi, p_localElements.get()); + this->ready(); // may want to keep this around if operation is done alot p_localElements.reset(); } From b9f6171b77167d67d9ec2b3526dd87adcb099245 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Mon, 28 Aug 2023 11:50:28 -0700 Subject: [PATCH 014/195] Update PETSc settings for tlaloc --- src/example_configuration.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/example_configuration.sh b/src/example_configuration.sh index ee870d678..afe9e6ffb 100755 --- a/src/example_configuration.sh +++ b/src/example_configuration.sh @@ -256,13 +256,13 @@ elif [ $host == "tlaloc" ]; then export CC CXX CFLAGS CXXFLAGS # Custom built 3.16, complex: - # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc.gitlab" \ - # -D PETSC_ARCH:STRING="ubuntu-complex-shared-3.16.6" \ + # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.16.6" \ + # -D PETSC_ARCH:STRING="ubuntu-complex-shared" \ # -D USE_OLD_PETSC:BOOL=OFF \ # Custom built 3.16, real: - # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc.gitlab" \ - # -D PETSC_ARCH:STRING="ubuntu-real-shared-3.16.6" \ + # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.16.6" \ + # -D PETSC_ARCH:STRING="ubuntu-real-shared" \ # -D USE_OLD_PETSC:BOOL=OFF \ # Custom built 3.14, complex: @@ -270,14 +270,19 @@ elif [ $host == "tlaloc" ]; then # -D PETSC_ARCH:STRING="ubuntu-complex-shared" \ # -D USE_OLD_PETSC:BOOL=OFF \ + # Custom built 3.14, real: + # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.14.6" \ + # -D PETSC_ARCH:STRING="ubuntu-real-shared" \ + # -D USE_OLD_PETSC:BOOL=OFF \ + prefix="$HOME/Projects/GridPakLDRD/gridpack-install" cmake -Wdev --debug-trycompile \ prefix="$HOME/Projects/GridPakLDRD/gridpack-install" cmake -Wdev --debug-trycompile \ --graphviz=GridPACK.dot \ - -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc.gitlab" \ - -D PETSC_ARCH:STRING="ubuntu-complex-shared-3.16.6" \ + -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.14.6" \ + -D PETSC_ARCH:STRING="ubuntu-real-shared" \ -D USE_OLD_PETSC:BOOL=OFF \ -D BOOST_ROOT:PATH="/usr" \ -D Boost_NO_BOOST_CMAKE:BOOL=TRUE \ From 4efdd6bdb16f9713647bdb637f8ebe7e9e40a42f Mon Sep 17 00:00:00 2001 From: William Perkins Date: Mon, 28 Aug 2023 13:12:58 -0700 Subject: [PATCH 015/195] Add smoke test for 240-bus dynamic simulation --- .../dynamic_simulation_full_y/CMakeLists.txt | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/applications/dynamic_simulation_full_y/CMakeLists.txt b/src/applications/dynamic_simulation_full_y/CMakeLists.txt index e308b5318..051a749fc 100644 --- a/src/applications/dynamic_simulation_full_y/CMakeLists.txt +++ b/src/applications/dynamic_simulation_full_y/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2017-12-08 09:39:48 d3g096 +# Last Change: 2023-08-28 13:11:06 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -114,6 +114,16 @@ add_custom_command( DEPENDS "${GRIDPACK_DATA_DIR}/input/ds/input_3000.xml" ) +add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml" + COMMAND ${CMAKE_COMMAND} + -D INPUT:PATH="${GRIDPACK_DATA_DIR}/input/ds/input_240bus.xml" + -D OUTPUT:PATH="${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml" + -D PKG:STRING="${GRIDPACK_MATSOLVER_PKG}" + -P "${PROJECT_SOURCE_DIR}/cmake-modules/set_lu_solver_pkg.cmake" + DEPENDS "${GRIDPACK_DATA_DIR}/input/ds/input_240bus.xml" + ) + add_custom_target(dsf.x.input COMMAND ${CMAKE_COMMAND} -E copy @@ -148,6 +158,14 @@ add_custom_target(dsf.x.input ${GRIDPACK_DATA_DIR}/dyr/classical_model_3000bus.dyr ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy + ${GRIDPACK_DATA_DIR}/dyr/240busWECC_2018_PSS_mod.dyr + ${CMAKE_CURRENT_BINARY_DIR} + + COMMAND ${CMAKE_COMMAND} -E copy + ${GRIDPACK_DATA_DIR}/raw/240busWECC_2018_PSS_fixedshunt.raw + ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/input_145.xml ${GRIDPACK_DATA_DIR}/raw/IEEE_145bus_v23_PSLF.raw @@ -161,6 +179,9 @@ add_custom_target(dsf.x.input ${CMAKE_CURRENT_BINARY_DIR}/input_3000.xml ${GRIDPACK_DATA_DIR}/raw/bus3000_gen_no0imp_v23_pslf.raw ${GRIDPACK_DATA_DIR}/dyr/classical_model_3000bus.dyr + ${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml + ${GRIDPACK_DATA_DIR}/dyr/240busWECC_2018_PSS_mod.dyr + ${GRIDPACK_DATA_DIR}/raw/240busWECC_2018_PSS_fixedshunt.raw ) add_dependencies(dsf.x dsf.x.input) @@ -197,5 +218,6 @@ install(TARGETS dsf2.x DESTINATION bin) # ------------------------------------------------------------- # run application as test # ------------------------------------------------------------- -gridpack_add_run_test("dynamic_simulation_full_y" dsf.x input_145.xml) +gridpack_add_run_test("dynamic_simulation_full_y_145_bus" dsf.x input_145.xml) +gridpack_add_run_test("dynamic_simulation_full_y_240_bus" dsf.x input_240bus.xml) From a0a0869faba7875c8f7f61031706d71fcf44ace9 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Mon, 28 Aug 2023 13:39:31 -0700 Subject: [PATCH 016/195] Add a CMake macro to do the LU solver check and use it --- .../dynamic_simulation_full_y/CMakeLists.txt | 75 +++++++------------ src/cmake-modules/GridPACK.cmake | 23 +++++- 2 files changed, 47 insertions(+), 51 deletions(-) diff --git a/src/applications/dynamic_simulation_full_y/CMakeLists.txt b/src/applications/dynamic_simulation_full_y/CMakeLists.txt index 051a749fc..c94db802a 100644 --- a/src/applications/dynamic_simulation_full_y/CMakeLists.txt +++ b/src/applications/dynamic_simulation_full_y/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2023-08-28 13:11:06 d3g096 +# Last Change: 2023-08-28 13:21:26 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -74,55 +74,30 @@ add_executable(dsf2.x target_link_libraries(dsf.x ${target_libraries}) target_link_libraries(dsf2.x ${target_libraries}) -add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/input_145.xml" - COMMAND ${CMAKE_COMMAND} - -D INPUT:PATH="${GRIDPACK_DATA_DIR}/input/ds/input_145.xml" - -D OUTPUT:PATH="${CMAKE_CURRENT_BINARY_DIR}/input_145.xml" - -D PKG:STRING="${GRIDPACK_MATSOLVER_PKG}" - -P "${PROJECT_SOURCE_DIR}/cmake-modules/set_lu_solver_pkg.cmake" - DEPENDS "${GRIDPACK_DATA_DIR}/input/ds/input_145.xml" - ) - -add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/input_9b3g.xml" - COMMAND ${CMAKE_COMMAND} - -D INPUT:PATH="${GRIDPACK_DATA_DIR}/input/ds/input_9b3g.xml" - -D OUTPUT:PATH="${CMAKE_CURRENT_BINARY_DIR}/input_9b3g.xml" - -D PKG:STRING="${GRIDPACK_MATSOLVER_PKG}" - -P "${PROJECT_SOURCE_DIR}/cmake-modules/set_lu_solver_pkg.cmake" - DEPENDS "${GRIDPACK_DATA_DIR}/input/ds/input_9b3g.xml" - ) - -add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/input_300_cmpld.xml" - COMMAND ${CMAKE_COMMAND} - -D INPUT:PATH="${GRIDPACK_DATA_DIR}/input/ds/input_300_cmpld.xml" - -D OUTPUT:PATH="${CMAKE_CURRENT_BINARY_DIR}/input_300_cmpld.xml" - -D PKG:STRING="${GRIDPACK_MATSOLVER_PKG}" - -P "${PROJECT_SOURCE_DIR}/cmake-modules/set_lu_solver_pkg.cmake" - DEPENDS "${GRIDPACK_DATA_DIR}/input/ds/input_300_cmpld.xml" - ) - -add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/input_3000.xml" - COMMAND ${CMAKE_COMMAND} - -D INPUT:PATH="${GRIDPACK_DATA_DIR}/input/ds/input_3000.xml" - -D OUTPUT:PATH="${CMAKE_CURRENT_BINARY_DIR}/input_3000.xml" - -D PKG:STRING="${GRIDPACK_MATSOLVER_PKG}" - -P "${PROJECT_SOURCE_DIR}/cmake-modules/set_lu_solver_pkg.cmake" - DEPENDS "${GRIDPACK_DATA_DIR}/input/ds/input_3000.xml" - ) - -add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml" - COMMAND ${CMAKE_COMMAND} - -D INPUT:PATH="${GRIDPACK_DATA_DIR}/input/ds/input_240bus.xml" - -D OUTPUT:PATH="${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml" - -D PKG:STRING="${GRIDPACK_MATSOLVER_PKG}" - -P "${PROJECT_SOURCE_DIR}/cmake-modules/set_lu_solver_pkg.cmake" - DEPENDS "${GRIDPACK_DATA_DIR}/input/ds/input_240bus.xml" - ) +gridpack_set_lu_solver( + "${GRIDPACK_DATA_DIR}/input/ds/input_145.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_145.xml" +) + +gridpack_set_lu_solver( + "${GRIDPACK_DATA_DIR}/input/ds/input_9b3g.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_9b3g.xml" +) + +gridpack_set_lu_solver( + "${GRIDPACK_DATA_DIR}/input/ds/input_300_cmpld.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_300_cmpld.xml" +) + +gridpack_set_lu_solver( + "${GRIDPACK_DATA_DIR}/input/ds/input_3000.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_3000.xml" +) + +gridpack_set_lu_solver( + "${GRIDPACK_DATA_DIR}/input/ds/input_240bus.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml" +) add_custom_target(dsf.x.input diff --git a/src/cmake-modules/GridPACK.cmake b/src/cmake-modules/GridPACK.cmake index bd07c7ebe..8c0dc6c80 100644 --- a/src/cmake-modules/GridPACK.cmake +++ b/src/cmake-modules/GridPACK.cmake @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created June 10, 2013 by William A. Perkins -# Last Change: 2019-08-16 13:41:26 d3g096 +# Last Change: 2023-08-28 13:18:39 d3g096 # ------------------------------------------------------------- @@ -163,3 +163,24 @@ function(gridpack_add_run_test test_name test_target test_input) gridpack_add_parallel_run_test("${test_name}" ${test_target} "${test_input}") endif () endfunction(gridpack_add_run_test) + + +# ------------------------------------------------------------- +# gridpack_set_lu_solver +# +# This takes a GridPACK input XML file, looks for any occurance of +# "superlu_dist" and changes it to "mumps" if SuperLU_DIST is not +# available in the build. The result is copied to the specified +# output. +# ------------------------------------------------------------- +macro(gridpack_set_lu_solver inputxml outputxml) +add_custom_command( + OUTPUT "${outputxml}" + COMMAND ${CMAKE_COMMAND} + -D INPUT:PATH="${inputxml}" + -D OUTPUT:PATH="${outputxml}" + -D PKG:STRING="${GRIDPACK_MATSOLVER_PKG}" + -P "${PROJECT_SOURCE_DIR}/cmake-modules/set_lu_solver_pkg.cmake" + DEPENDS "${inputxml}" +) +endmacro() From b7135058c25abab8ed5b48abbbe9d98c36facc3a Mon Sep 17 00:00:00 2001 From: William Perkins Date: Mon, 28 Aug 2023 14:30:18 -0700 Subject: [PATCH 017/195] Add smoke test for 2-area dynamic simulation --- src/CMakeLists.txt | 65 +++++++++---------- .../dynamic_simulation_full_y/CMakeLists.txt | 19 +++++- 2 files changed, 50 insertions(+), 34 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 729327468..17c20e67d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 3, 2013 by William A. Perkins -# Last Change: 2022-10-05 09:26:46 d3g096 +# Last Change: 2023-08-28 13:46:15 d3g096 # ------------------------------------------------------------- # @@ -313,44 +313,43 @@ endif() message(STATUS "Using PETSc version ${PETSC_VERSION}") if (NOT USE_OLD_PETSC) -if(EXISTS "${PETSC_DIR}/${PETSC_ARCH}/include/petscconf.h") - set(petscconf "${PETSC_DIR}/${PETSC_ARCH}/include/petscconf.h") -elseif(EXISTS "${PETSC_DIR}/include/petscconf.h") - set(petscconf "${PETSC_DIR}/include/petscconf.h") - else() - message(FATAL_ERROR "Could not find PETSc configuration header file + if(EXISTS "${PETSC_DIR}/${PETSC_ARCH}/include/petscconf.h") + set(petscconf "${PETSC_DIR}/${PETSC_ARCH}/include/petscconf.h") + elseif(EXISTS "${PETSC_DIR}/include/petscconf.h") + set(petscconf "${PETSC_DIR}/include/petscconf.h") + else() + message(FATAL_ERROR "Could not find PETSc configuration header file petscconf.h") -endif() + endif() -# checks -# define PETSc variables -include(CheckSymbolExists) -check_symbol_exists(PETSC_HAVE_PARMETIS ${petscconf} PETSC_HAVE_PARMETIS) -check_symbol_exists(PETSC_USE_REAL_DOUBLE ${petscconf} PETSC_USE_REAL_DOUBLE) -check_symbol_exists(PETSC_USE_COMPLEX ${petscconf} PETSC_USE_COMPLEX) -check_symbol_exists(PETSC_LANGUAGE_CXX ${petscconf} PETSC_CLANGUAGE_Cxx) -check_symbol_exists(PETSC_HAVE_SUPERLU_DIST ${petscconf} -PETSC_HAVE_SUPERLU_DIST) -check_symbol_exists(PETSC_HAVE_MUMPS ${petscconf} PETSC_HAVE_MUMPS) + # checks + # define PETSc variables + include(CheckSymbolExists) + check_symbol_exists(PETSC_HAVE_PARMETIS ${petscconf} PETSC_HAVE_PARMETIS) + check_symbol_exists(PETSC_USE_REAL_DOUBLE ${petscconf} PETSC_USE_REAL_DOUBLE) + check_symbol_exists(PETSC_USE_COMPLEX ${petscconf} PETSC_USE_COMPLEX) + check_symbol_exists(PETSC_LANGUAGE_CXX ${petscconf} PETSC_CLANGUAGE_Cxx) + check_symbol_exists(PETSC_HAVE_SUPERLU_DIST ${petscconf} + PETSC_HAVE_SUPERLU_DIST) + check_symbol_exists(PETSC_HAVE_MUMPS ${petscconf} PETSC_HAVE_MUMPS) else() -if (EXISTS "${PETSC_DIR}/${PETSC_ARCH}/conf/PETScConfig.cmake") - include("${PETSC_DIR}/${PETSC_ARCH}/conf/PETScConfig.cmake") -elseif (EXISTS "${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScConfig.cmake") - include("${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScConfig.cmake") -elseif (EXISTS "${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScBuildInternal.cmake") - include("${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScBuildInternal.cmake") -else() - message(FATAL_ERROR "PETSc found, but CMake configuration for PETSc installation not found?") -endif() - -message(STATUS "PETSC_LIBRARY_SINGLE: ${PETSC_LIBRARY_SINGLE}") + if (EXISTS "${PETSC_DIR}/${PETSC_ARCH}/conf/PETScConfig.cmake") + include("${PETSC_DIR}/${PETSC_ARCH}/conf/PETScConfig.cmake") + elseif (EXISTS "${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScConfig.cmake") + include("${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScConfig.cmake") + elseif (EXISTS "${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScBuildInternal.cmake") + include("${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScBuildInternal.cmake") + else() + message(FATAL_ERROR "PETSc found, but CMake configuration for PETSc installation not found?") + endif() + message(STATUS "PETSC_LIBRARY_SINGLE: ${PETSC_LIBRARY_SINGLE}") -# checks + # checks -if (NOT PETSC_HAVE_MPI) - message(FATAL_ERROR "PETSc installation is not parallel (--with-mpi=1)") -endif() + if (NOT PETSC_HAVE_MPI) + message(FATAL_ERROR "PETSc installation is not parallel (--with-mpi=1)") + endif() endif() # PETSc can be compiled for double precsion (--with-precision=double) diff --git a/src/applications/dynamic_simulation_full_y/CMakeLists.txt b/src/applications/dynamic_simulation_full_y/CMakeLists.txt index c94db802a..83a362b89 100644 --- a/src/applications/dynamic_simulation_full_y/CMakeLists.txt +++ b/src/applications/dynamic_simulation_full_y/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2023-08-28 13:21:26 d3g096 +# Last Change: 2023-08-28 14:29:22 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -99,6 +99,11 @@ gridpack_set_lu_solver( "${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml" ) +gridpack_set_lu_solver( + "${GRIDPACK_DATA_DIR}/input/ds/input_twoarea.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_twoarea.xml" +) + add_custom_target(dsf.x.input COMMAND ${CMAKE_COMMAND} -E copy @@ -141,6 +146,14 @@ add_custom_target(dsf.x.input ${GRIDPACK_DATA_DIR}/raw/240busWECC_2018_PSS_fixedshunt.raw ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy + ${GRIDPACK_DATA_DIR}/dyr/kundur-twoarea.dyr + ${CMAKE_CURRENT_BINARY_DIR} + + COMMAND ${CMAKE_COMMAND} -E copy + ${GRIDPACK_DATA_DIR}/raw/kundur-twoarea_v33.raw + ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/input_145.xml ${GRIDPACK_DATA_DIR}/raw/IEEE_145bus_v23_PSLF.raw @@ -157,6 +170,9 @@ add_custom_target(dsf.x.input ${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml ${GRIDPACK_DATA_DIR}/dyr/240busWECC_2018_PSS_mod.dyr ${GRIDPACK_DATA_DIR}/raw/240busWECC_2018_PSS_fixedshunt.raw + ${CMAKE_CURRENT_BINARY_DIR}/input_twoarea.xml + ${GRIDPACK_DATA_DIR}/dyr/kundur-twoarea.dyr + ${GRIDPACK_DATA_DIR}/raw/kundur-twoarea_v33.raw ) add_dependencies(dsf.x dsf.x.input) @@ -195,4 +211,5 @@ install(TARGETS dsf2.x DESTINATION bin) # ------------------------------------------------------------- gridpack_add_run_test("dynamic_simulation_full_y_145_bus" dsf.x input_145.xml) gridpack_add_run_test("dynamic_simulation_full_y_240_bus" dsf.x input_240bus.xml) +gridpack_add_run_test("dynamic_simulation_two_area" dsf.x input_twoarea.xml) From f99aeaca54c26fe4257171681c1e4c9d957cd36e Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 30 Aug 2023 06:01:46 -0700 Subject: [PATCH 018/195] Add smoke test for two area case w/ renewables --- .../input/ds/input_twoarea_renewable_mech.xml | 4 ++-- .../dynamic_simulation_full_y/CMakeLists.txt | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/applications/data_sets/input/ds/input_twoarea_renewable_mech.xml b/src/applications/data_sets/input/ds/input_twoarea_renewable_mech.xml index f35697b5c..52b1ab687 100644 --- a/src/applications/data_sets/input/ds/input_twoarea_renewable_mech.xml +++ b/src/applications/data_sets/input/ds/input_twoarea_renewable_mech.xml @@ -1,7 +1,7 @@ - Benchmark_twoarea_v33.raw + kundur-twoarea_v33.raw 50 1.0e-6 @@ -53,7 +53,7 @@ - Benchmark_twoarea_4renewable_mech.dyr + kundur-twoarea_4renewable_mech.dyr 20 0.005 diff --git a/src/applications/dynamic_simulation_full_y/CMakeLists.txt b/src/applications/dynamic_simulation_full_y/CMakeLists.txt index 83a362b89..7e8b86a1d 100644 --- a/src/applications/dynamic_simulation_full_y/CMakeLists.txt +++ b/src/applications/dynamic_simulation_full_y/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2023-08-28 14:29:22 d3g096 +# Last Change: 2023-08-30 06:00:46 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -104,6 +104,11 @@ gridpack_set_lu_solver( "${CMAKE_CURRENT_BINARY_DIR}/input_twoarea.xml" ) +gridpack_set_lu_solver( + "${GRIDPACK_DATA_DIR}/input/ds/input_twoarea_renewable_mech.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_twoarea_renewable_mech.xml" +) + add_custom_target(dsf.x.input COMMAND ${CMAKE_COMMAND} -E copy @@ -154,6 +159,10 @@ add_custom_target(dsf.x.input ${GRIDPACK_DATA_DIR}/raw/kundur-twoarea_v33.raw ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy + ${GRIDPACK_DATA_DIR}/dyr/kundur-twoarea_4renewable_mech.dyr + ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/input_145.xml ${GRIDPACK_DATA_DIR}/raw/IEEE_145bus_v23_PSLF.raw @@ -173,6 +182,8 @@ add_custom_target(dsf.x.input ${CMAKE_CURRENT_BINARY_DIR}/input_twoarea.xml ${GRIDPACK_DATA_DIR}/dyr/kundur-twoarea.dyr ${GRIDPACK_DATA_DIR}/raw/kundur-twoarea_v33.raw + ${CMAKE_CURRENT_BINARY_DIR}/input_twoarea_renewable_mech.xml + ${GRIDPACK_DATA_DIR}/dyr/kundur-twoarea_4renewable_mech.dyr ) add_dependencies(dsf.x dsf.x.input) @@ -212,4 +223,5 @@ install(TARGETS dsf2.x DESTINATION bin) gridpack_add_run_test("dynamic_simulation_full_y_145_bus" dsf.x input_145.xml) gridpack_add_run_test("dynamic_simulation_full_y_240_bus" dsf.x input_240bus.xml) gridpack_add_run_test("dynamic_simulation_two_area" dsf.x input_twoarea.xml) +gridpack_add_run_test("dynamic_simulation_two_area_renewable" dsf.x input_twoarea_renewable_mech.xml) From e4c21716ea4afbdf3f72dea2bdb493abba000370 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 30 Aug 2023 09:59:25 -0700 Subject: [PATCH 019/195] Remove math::transpose(Matrix<>, Matrix<>) because of PETSc transpose semantics --- src/math/matrix.hpp | 22 +++++++++++----------- src/math/test/matrix_test.cpp | 24 ++++++++++++------------ src/math/test/matrix_transpose_test.cpp | 5 ++--- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/src/math/matrix.hpp b/src/math/matrix.hpp index 9ea04d009..3b4c58be3 100644 --- a/src/math/matrix.hpp +++ b/src/math/matrix.hpp @@ -9,7 +9,7 @@ /** * @file matrix.hpp * @author William A. Perkins - * @date 2015-08-18 14:09:32 d3g096 + * @date 2023-08-30 07:58:18 d3g096 * * @brief Declaration of the Matrix class. */ @@ -482,16 +482,16 @@ void add(const MatrixT& A, const MatrixT& B, MatrixT& result) result.add(B); } -/// Make the transpose of a Matrix and put it in another -/** - * - * - * @param A - * @param result - */ -template -void -transpose(const MatrixT& A, MatrixT& result); +// /// Make the transpose of a Matrix and put it in another +// /** +// * +// * +// * @param A +// * @param result +// */ +// template +// void +// transpose(const MatrixT& A, MatrixT& result); /// Get a column from the Matrix and put in specified Vector /** diff --git a/src/math/test/matrix_test.cpp b/src/math/test/matrix_test.cpp index 94f6d20d1..d398f35dd 100644 --- a/src/math/test/matrix_test.cpp +++ b/src/math/test/matrix_test.cpp @@ -8,7 +8,7 @@ /** * @file matrix_test.cpp * @author William A. Perkins - * @date 2016-12-16 09:35:46 d3g096 + * @date 2023-08-30 07:59:49 d3g096 * * @brief Unit tests for Matrix * @@ -990,15 +990,15 @@ BOOST_AUTO_TEST_CASE( NonSquareTranspose ) // FIXME: check B contents - boost::scoped_ptr - C(new TestMatrixType(world, 3, 2, the_storage_type)); - transpose(*A, *C); - C->print(); + // boost::scoped_ptr + // C(new TestMatrixType(world, 3, 2, the_storage_type)); + // transpose(*A, *C); + // C->print(); // FIXME: check C contents - C.reset(A->clone()); - BOOST_CHECK_THROW(transpose(*A, *C), gridpack::Exception); + // C.reset(A->clone()); + // BOOST_CHECK_THROW(transpose(*A, *C), gridpack::Exception); } BOOST_AUTO_TEST_CASE( AnotherNonSquareTranspose ) @@ -1098,13 +1098,13 @@ BOOST_AUTO_TEST_CASE( AnotherNonSquareTranspose ) throw gridpack::Exception("Unknown Matrix storage type"); } - transpose(*A, *C); - C->print(); + // transpose(*A, *C); + // C->print(); - // FIXME: check C contents + // // FIXME: check C contents - C.reset(A->clone()); - BOOST_CHECK_THROW(transpose(*A, *C), gridpack::Exception); + // C.reset(A->clone()); + // BOOST_CHECK_THROW(transpose(*A, *C), gridpack::Exception); } diff --git a/src/math/test/matrix_transpose_test.cpp b/src/math/test/matrix_transpose_test.cpp index 4b8800b46..ffba0d49c 100644 --- a/src/math/test/matrix_transpose_test.cpp +++ b/src/math/test/matrix_transpose_test.cpp @@ -8,7 +8,7 @@ /** * @file matrix_transpose_test.cpp * @author William A. Perkins - * @date 2018-12-18 09:31:40 d3g096 + * @date 2023-08-30 08:02:21 d3g096 * * @brief Unit tests for Matrix * @@ -173,8 +173,7 @@ BOOST_AUTO_TEST_CASE( TransposeRandom ) // see if the transpose is reversible (transpose of transpose needs // to be distributed the same as A - boost::scoped_ptr C(A->clone()); - gridpack::math::transpose(*B, *C); + boost::scoped_ptr C(gridpack::math::transpose(*B)); C->scale(-1.0); boost::scoped_ptr E(gridpack::math::add(*A, *C)); From 6580fda51baae6983ddb412c83c6e03d2c281531 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 30 Aug 2023 13:06:02 -0700 Subject: [PATCH 020/195] Add smoke tests for dsf2.x (same as for dsf.x). --- src/applications/dynamic_simulation_full_y/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/applications/dynamic_simulation_full_y/CMakeLists.txt b/src/applications/dynamic_simulation_full_y/CMakeLists.txt index 7e8b86a1d..b21738753 100644 --- a/src/applications/dynamic_simulation_full_y/CMakeLists.txt +++ b/src/applications/dynamic_simulation_full_y/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2023-08-30 06:00:46 d3g096 +# Last Change: 2023-08-30 10:04:14 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -221,7 +221,11 @@ install(TARGETS dsf2.x DESTINATION bin) # run application as test # ------------------------------------------------------------- gridpack_add_run_test("dynamic_simulation_full_y_145_bus" dsf.x input_145.xml) +gridpack_add_run_test("dynamic_simulation_full_y_2_145_bus" dsf2.x input_145.xml) gridpack_add_run_test("dynamic_simulation_full_y_240_bus" dsf.x input_240bus.xml) +gridpack_add_run_test("dynamic_simulation_full_y2_240_bus" dsf2.x input_240bus.xml) gridpack_add_run_test("dynamic_simulation_two_area" dsf.x input_twoarea.xml) +gridpack_add_run_test("dynamic_simulation_2_two_area" dsf2.x input_twoarea.xml) gridpack_add_run_test("dynamic_simulation_two_area_renewable" dsf.x input_twoarea_renewable_mech.xml) +gridpack_add_run_test("dynamic_simulation_2_two_area_renewable" dsf2.x input_twoarea_renewable_mech.xml) From e7ab92bebf5b94854509b51a84aefce028310ed1 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Thu, 31 Aug 2023 08:46:38 -0700 Subject: [PATCH 021/195] Add smoke test for Wind DSA 9-bus case --- .../development/wind_dsa/CMakeLists.txt | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/applications/development/wind_dsa/CMakeLists.txt b/src/applications/development/wind_dsa/CMakeLists.txt index 253770f61..c9483529f 100644 --- a/src/applications/development/wind_dsa/CMakeLists.txt +++ b/src/applications/development/wind_dsa/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2019-08-16 13:51:41 d3g096 +# Last Change: 2023-08-31 08:45:11 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -81,16 +81,13 @@ target_link_libraries(wind.x ${target_libraries}) target_link_libraries(wind2.x ${target_libraries}) -# Put files necessary to run powerflow2 in binary directory. -# gridpack.petscrc is temporary -- it will be incorporated into -# input.xml +gridpack_set_lu_solver( + "${CMAKE_CURRENT_SOURCE_DIR}/input.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input.xml" +) add_custom_target(wind.x.input - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_SOURCE_DIR}/input.xml - ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/IEEE3G9B_V23.raw ${CMAKE_CURRENT_BINARY_DIR} @@ -112,13 +109,12 @@ add_custom_target(wind.x.input ${CMAKE_CURRENT_BINARY_DIR} DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/input.xml + ${CMAKE_CURRENT_BINARY_DIR}/input.xml ${CMAKE_CURRENT_SOURCE_DIR}/IEEE3G9B_V23.raw ${CMAKE_CURRENT_SOURCE_DIR}/3g9b_faults.xml ${CMAKE_CURRENT_SOURCE_DIR}/3g9b_classical.dyr ${CMAKE_CURRENT_SOURCE_DIR}/load.txt ${CMAKE_CURRENT_SOURCE_DIR}/wind.txt - ) add_dependencies(wind.x wind.x.input) add_dependencies(wind2.x wind.x.input) @@ -126,8 +122,10 @@ add_dependencies(wind2.x wind.x.input) # ------------------------------------------------------------- # run applications as tests # ------------------------------------------------------------- -# FIXME: temporarily disabled -#gridpack_add_run_test("wind" wind.x input.xml) +gridpack_add_run_test("wind_dsa" wind.x input.xml) +gridpack_add_run_test("wind_dsa_2" wind2.x input.xml) +gridpack_add_run_test("wind_dsa" wind.x input.xml) +gridpack_add_run_test("wind_dsa_2" wind2.x input.xml) # ------------------------------------------------------------- # install as a sample application From 5b38344f80535a35c4d5761d923d677405c473d9 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Thu, 31 Aug 2023 08:57:09 -0700 Subject: [PATCH 022/195] Fix typos in Wind DSA TAMU 2000 case --- src/applications/development/wind_dsa/input_tamu2000_dsf.xml | 2 +- src/applications/development/wind_dsa/tamu_2000_faults.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/applications/development/wind_dsa/input_tamu2000_dsf.xml b/src/applications/development/wind_dsa/input_tamu2000_dsf.xml index afa923b46..3246eba51 100644 --- a/src/applications/development/wind_dsa/input_tamu2000_dsf.xml +++ b/src/applications/development/wind_dsa/input_tamu2000_dsf.xml @@ -51,7 +51,7 @@ 0.005 texas_2k_scenfile_5_full.txt - tamu_2000_faults.xml + tamu_2000_faults.xml 1048 diff --git a/src/applications/development/wind_dsa/tamu_2000_faults.xml b/src/applications/development/wind_dsa/tamu_2000_faults.xml index 55df4d1a8..5366ba2c0 100644 --- a/src/applications/development/wind_dsa/tamu_2000_faults.xml +++ b/src/applications/development/wind_dsa/tamu_2000_faults.xml @@ -1,5 +1,5 @@ - + @@ -16,4 +16,4 @@ - + From 7514dccbc6dcdfb03cde96129d1a1cf38323608a Mon Sep 17 00:00:00 2001 From: William Perkins Date: Thu, 31 Aug 2023 11:00:53 -0700 Subject: [PATCH 023/195] Add a smoke test for Wind DSA TAMU 2000 case --- .../development/wind_dsa/CMakeLists.txt | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/applications/development/wind_dsa/CMakeLists.txt b/src/applications/development/wind_dsa/CMakeLists.txt index c9483529f..058257bd8 100644 --- a/src/applications/development/wind_dsa/CMakeLists.txt +++ b/src/applications/development/wind_dsa/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2023-08-31 08:45:11 d3g096 +# Last Change: 2023-08-31 09:05:17 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -86,6 +86,11 @@ gridpack_set_lu_solver( "${CMAKE_CURRENT_BINARY_DIR}/input.xml" ) +gridpack_set_lu_solver( + "${CMAKE_CURRENT_SOURCE_DIR}/input_tamu2000_dsf.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_tamu2000_dsf.xml" +) + add_custom_target(wind.x.input COMMAND ${CMAKE_COMMAND} -E copy @@ -108,6 +113,22 @@ add_custom_target(wind.x.input ${CMAKE_CURRENT_SOURCE_DIR}/wind.txt ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/texas2000_hr_4882_mod_33.raw + ${CMAKE_CURRENT_BINARY_DIR} + + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/texas2000_v23_fullsys_noac_wind_mod.dyr + ${CMAKE_CURRENT_BINARY_DIR} + + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/tamu_2000_faults.xml + ${CMAKE_CURRENT_BINARY_DIR} + + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/texas_2k_scenfile_5_full.txt + ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/input.xml ${CMAKE_CURRENT_SOURCE_DIR}/IEEE3G9B_V23.raw @@ -115,6 +136,11 @@ add_custom_target(wind.x.input ${CMAKE_CURRENT_SOURCE_DIR}/3g9b_classical.dyr ${CMAKE_CURRENT_SOURCE_DIR}/load.txt ${CMAKE_CURRENT_SOURCE_DIR}/wind.txt + ${CMAKE_CURRENT_BINARY_DIR}/input_tamu2000_dsf.xml + ${CMAKE_CURRENT_SOURCE_DIR}/texas2000_hr_4882_mod_33.raw + ${CMAKE_CURRENT_SOURCE_DIR}/texas2000_v23_fullsys_noac_wind_mod.dyr + ${CMAKE_CURRENT_SOURCE_DIR}/tamu_2000_faults.xml + ${CMAKE_CURRENT_SOURCE_DIR}/texas_2k_scenfile_5_full.txt ) add_dependencies(wind.x wind.x.input) add_dependencies(wind2.x wind.x.input) @@ -124,8 +150,8 @@ add_dependencies(wind2.x wind.x.input) # ------------------------------------------------------------- gridpack_add_run_test("wind_dsa" wind.x input.xml) gridpack_add_run_test("wind_dsa_2" wind2.x input.xml) -gridpack_add_run_test("wind_dsa" wind.x input.xml) -gridpack_add_run_test("wind_dsa_2" wind2.x input.xml) +gridpack_add_run_test("wind_dsa_tamu2000" wind.x input_tamu2000_dsf.xml) +gridpack_add_run_test("wind_dsa_2_tamu2000" wind2.x input_tamu2000_dsf.xml) # ------------------------------------------------------------- # install as a sample application From f84ae13fe42e8fb30a3eaa313dc0708c81877280 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 6 Sep 2023 06:57:30 -0700 Subject: [PATCH 024/195] Remove ancient test from `applications/development/dynamic_simulation_reduced_y` --- .../development/dynamic_simulation_reduced_y/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/applications/development/dynamic_simulation_reduced_y/CMakeLists.txt b/src/applications/development/dynamic_simulation_reduced_y/CMakeLists.txt index fd0b11f4c..ddd838b4a 100644 --- a/src/applications/development/dynamic_simulation_reduced_y/CMakeLists.txt +++ b/src/applications/development/dynamic_simulation_reduced_y/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2019-01-04 08:39:22 d3g096 +# Last Change: 2023-09-06 06:50:50 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -100,5 +100,5 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.install.in # ------------------------------------------------------------- # run application as test # ------------------------------------------------------------- -gridpack_add_run_test("dynamic_simulation" ds.x input.xml) +# gridpack_add_run_test("dynamic_simulation" ds.x input.xml) From 7ed20ef898370766d5c139de03b8a06a26243863 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 6 Sep 2023 10:15:20 -0500 Subject: [PATCH 025/195] Fix env var --- install_gridpack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index fdadabf24..38b96e05f 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -106,7 +106,7 @@ date build_dir=${PWD}/src/build install_dir=${PWD}/src/install -install_gridpack "${GRIDPACK_EXT_DEPS:?}" "$build_dir" "$install_dir" +install_gridpack "${GP_EXT_DEPS:?}" "$build_dir" "$install_dir" install_gridpack_python "$build_dir" "$install_dir" echo "Completed GridPACK installation" From d3aecd995bf6fbf78492df9d9455ec6d1d88558c Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 6 Sep 2023 10:15:24 -0500 Subject: [PATCH 026/195] Set number of make threads dynamically --- install_gridpack.sh | 2 +- install_gridpack_deps.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 38b96e05f..3d0f53d27 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -50,7 +50,7 @@ function install_gridpack { # install echo "Installing GridPACK" - make -j 10 install + make -j "$(nproc)" install popd || exit diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 74a5dd15a..98a6bd29e 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -90,7 +90,7 @@ function install_ga { # install echo "Installing Global Arrays" - make -j 10 install >../log/ga_install.log 2>&1 + make -j "$(nproc)" install >../log/ga_install.log 2>&1 popd || exit From 795505d80f4993ee6fbb1004d36611ff8ec72df1 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 6 Sep 2023 10:16:43 -0500 Subject: [PATCH 027/195] Add ci script and container image definition --- .gitlab-ci.yml | 44 ++++++++++++++++++++++++++++++++++++++++++++ dockerfile | 17 +++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..1b87d0b95 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,44 @@ +default: + + tags: [ basic, gridpack, ikp, k8s ] + + +variables: + + GP_EXT_DEPS: /gridpack-dependencies + LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} + + +# https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy +build-container: + + image: + + name: gcr.io/kaniko-project/executor:v1.9.0-debug + entrypoint: [""] + + script: + + - /kaniko/executor + --context "${CI_PROJECT_DIR}" + --build-arg http_proxy=$http_proxy + --build-arg https_proxy=$https_proxy + --dockerfile "${CI_PROJECT_DIR}/Dockerfile" + --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF}" + + rules: + + - if: $CI_PIPELINE_SOURCE == "branch" + changes: [ "dockerfile", "install/*" ] + - if: $CI_PIPELINE_SOURCE == "tag" + + +test: + + image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF} + + needs: [ build-container ] + + script: + + - ./install_gridpack.sh diff --git a/dockerfile b/dockerfile new file mode 100644 index 000000000..1a0ee60f3 --- /dev/null +++ b/dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:22.04 + +RUN apt update \ + && apt upgrade -y \ + && apt install -y wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config + +ENV GP_EXT_DEPS=/gridpack-dependencies +ENV BOOST_VERSION "1.78.0" +ENV GA_VERSION "5.8" +ENV PETSC_VERSION "3.16.4" +ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} + +WORKDIR ${GP_EXT_DEPS} + +COPY *.sh . + +RUN ./install_gridpack_deps.sh && rm *.sh From 66fbc23a13bfd5edd55e4cb3809e0339dcedf02a Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 13 Sep 2023 08:53:25 -0700 Subject: [PATCH 028/195] Fix incorrect Matrix type Jacobian supplied to DAESolver constructor --- src/math/dae_solver.hpp | 4 ++-- src/math/dae_solver_functions.hpp | 4 ++-- src/math/dae_solver_implementation.hpp | 13 +++++++++---- src/math/petsc/petsc_dae_solver.cpp | 8 ++++---- src/math/petsc/petsc_dae_solver_implementation.hpp | 4 ++-- src/math/test/dae_solver_test.cpp | 5 ++++- 6 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/math/dae_solver.hpp b/src/math/dae_solver.hpp index 8a7f05067..d13c6c896 100644 --- a/src/math/dae_solver.hpp +++ b/src/math/dae_solver.hpp @@ -9,7 +9,7 @@ /** * @file dae_solver.hpp * @author William A. Perkins - * @date 2019-12-05 07:52:33 d3g096 + * @date 2023-09-13 07:37:27 d3g096 * * @brief * @@ -72,7 +72,7 @@ class DAESolverT DAESolverT(const parallel::Communicator& comm, const int local_size, - Matrix* J, + MatrixType* J, JacobianBuilder& jbuilder, FunctionBuilder& fbuilder, EventManagerPtr eman); diff --git a/src/math/dae_solver_functions.hpp b/src/math/dae_solver_functions.hpp index 70cdf64cd..553da0bb2 100644 --- a/src/math/dae_solver_functions.hpp +++ b/src/math/dae_solver_functions.hpp @@ -9,7 +9,7 @@ /** * @file dae_solver_functions.hpp * @author William A. Perkins - * @date 2015-05-07 13:15:18 d3g096 + * @date 2023-09-13 07:56:50 d3g096 * * @brief * @@ -31,7 +31,7 @@ template struct DAEBuilder { typedef VectorT VectorType; - typedef Matrix MatrixType; + typedef MatrixT MatrixType; /// Functions that compute a Jacobian typedef diff --git a/src/math/dae_solver_implementation.hpp b/src/math/dae_solver_implementation.hpp index 33d6373cc..b1484e742 100644 --- a/src/math/dae_solver_implementation.hpp +++ b/src/math/dae_solver_implementation.hpp @@ -9,7 +9,7 @@ /** * @file dae_solver_implementation.hpp * @author William A. Perkins - * @date 2019-12-05 09:15:09 d3g096 + * @date 2023-09-13 08:34:39 d3g096 * * @brief * @@ -57,12 +57,12 @@ class DAESolverImplementation : parallel::Distributed(comm), utility::Configurable("DAESolver"), utility::Uncopyable(), + p_J(new MatrixType(comm,local_size,local_size)), + p_J_allocated(true), p_Fbuilder(fbuilder), p_Jbuilder(jbuilder), p_eventManager(eman), p_doAdaptive(true) - { - p_J = new gridpack::math::Matrix(comm,local_size,local_size); - } + { } DAESolverImplementation(const parallel::Communicator& comm, const int local_size, @@ -74,6 +74,7 @@ class DAESolverImplementation utility::Configurable("DAESolver"), utility::Uncopyable(), p_J(J), + p_J_allocated(false), p_Fbuilder(fbuilder), p_Jbuilder(jbuilder), p_eventManager(eman), p_doAdaptive(true) @@ -84,6 +85,7 @@ class DAESolverImplementation /// Destructor ~DAESolverImplementation(void) { + if (p_J_allocated) delete p_J; } protected: @@ -91,6 +93,9 @@ class DAESolverImplementation /// A matrix to hold the Jacobian MatrixType *p_J; + /// Is p_J locally allocated? + bool p_J_allocated; + /// A function to build the RHS vector FunctionBuilder p_Fbuilder; diff --git a/src/math/petsc/petsc_dae_solver.cpp b/src/math/petsc/petsc_dae_solver.cpp index 51b8982b9..47f2720e7 100644 --- a/src/math/petsc/petsc_dae_solver.cpp +++ b/src/math/petsc/petsc_dae_solver.cpp @@ -9,7 +9,7 @@ /** * @file dae_solver.cpp * @author William A. Perkins - * @date 2019-11-21 07:41:15 d3g096 + * @date 2023-09-13 07:48:08 d3g096 * * @brief * @@ -47,7 +47,7 @@ DAESolverT::DAESolverT(const parallel::Communicator& comm, template DAESolverT::DAESolverT(const parallel::Communicator& comm, const int local_size, - Matrix* J, + DAESolverT::MatrixType* J, DAESolverT::JacobianBuilder& jbuilder, DAESolverT::FunctionBuilder& fbuilder, DAESolverT::EventManagerPtr eman) @@ -71,7 +71,7 @@ DAESolverT::DAESolverT(const parallel::Communicator& comm, template DAESolverT::DAESolverT(const parallel::Communicator& comm, const int local_size, - gridpack::math::Matrix* J, + DAESolverT::MatrixType* J, DAESolverT::JacobianBuilder& jbuilder, DAESolverT::FunctionBuilder& fbuilder, DAESolverT::EventManagerPtr eman); @@ -86,7 +86,7 @@ DAESolverT::DAESolverT(const parallel::Communicator& comm, template DAESolverT::DAESolverT(const parallel::Communicator& comm, const int local_size, - Matrix *J, + DAESolverT::MatrixType *J, DAESolverT::JacobianBuilder& jbuilder, DAESolverT::FunctionBuilder& fbuilder, DAESolverT::EventManagerPtr eman); diff --git a/src/math/petsc/petsc_dae_solver_implementation.hpp b/src/math/petsc/petsc_dae_solver_implementation.hpp index f10886fda..aa1334ac7 100644 --- a/src/math/petsc/petsc_dae_solver_implementation.hpp +++ b/src/math/petsc/petsc_dae_solver_implementation.hpp @@ -10,7 +10,7 @@ /** * @file petsc_dae_solver_implementation.hpp * @author William A. Perkins - * @date 2023-08-24 09:42:12 d3g096 + * @date 2023-09-13 07:43:15 d3g096 * * @brief * @@ -70,7 +70,7 @@ class PETScDAESolverImplementation PETScDAESolverImplementation(const parallel::Communicator& comm, const int local_size, - Matrix* J, + MatrixType* J, JacobianBuilder& jbuilder, FunctionBuilder& fbuilder, EventManagerPtr eman) diff --git a/src/math/test/dae_solver_test.cpp b/src/math/test/dae_solver_test.cpp index 27a8e075e..7a8208729 100644 --- a/src/math/test/dae_solver_test.cpp +++ b/src/math/test/dae_solver_test.cpp @@ -9,7 +9,7 @@ /** * @file dae_solver_test.cpp * @author William A. Perkins - * @date 2019-12-04 12:40:57 d3g096 + * @date 2023-09-13 07:44:15 d3g096 * * @brief * @@ -169,8 +169,11 @@ class RoberProblem const double& shift, MatrixType& J) { int lo, hi; + int rlo, rhi; X.localIndexRange(lo, hi); + J.localRowRange(rlo, rhi); BOOST_ASSERT((hi-lo) == this->size()); + BOOST_ASSERT((rhi-rlo) == this->size()); std::vector x(this->size()); X.getElementRange(lo, hi, &x[0]); J.setElement(lo+0, lo+0, shift + 0.04); From 2c0744eb8f1a08969d5a76a1036885d99c0b2483 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 13 Sep 2023 09:16:50 -0700 Subject: [PATCH 029/195] Update my build script --- src/example_configuration.sh | 53 ++---------------------------------- 1 file changed, 2 insertions(+), 51 deletions(-) diff --git a/src/example_configuration.sh b/src/example_configuration.sh index afe9e6ffb..f3508ec33 100755 --- a/src/example_configuration.sh +++ b/src/example_configuration.sh @@ -145,55 +145,6 @@ elif [ $host == "we32673" ]; then -D CMAKE_INSTALL_PREFIX:PATH="$prefix/gridpack-install" \ $common_flags .. -elif [ $host == "WE30729" ]; then - - # Macbook using CLang 6.0 compilers and MPICH via MacPorts - # Pretty much the same as WE32673 - - CC=/opt/local/bin/clang - export CC - CXX=/opt/local/bin/clang++ - export CXX - - prefix="$HOME/Projects/GridPACK" - - if [ "$shared"x = "ON"x ]; then - pdir="$prefix/petsc-3.10.5" - parch="macosx-complex-c-shared" - else - pdir="$prefix/petsc-3.8.4" - parch="arch-macosx-clang-real-opt" - fi - cmake $options \ - -D BOOST_ROOT:STRING="/opt/local" \ - -D PETSC_DIR:PATH="$pdir" \ - -D PETSC_ARCH:STRING="$parch" \ - -D MPI_CXX_COMPILER:STRING='/opt/local/bin/mpicxx' \ - -D MPI_C_COMPILER:STRING='/opt/local/bin/mpicc' \ - -D MPIEXEC:STRING='/opt/local/bin/mpiexec' \ - -D MPIEXEC_MAX_NUMPROCS:STRING="2" \ - -D GRIDPACK_TEST_TIMEOUT:STRING=60 \ - -D USE_CPLEX:BOOL=OFF \ - -D USE_GLPK:BOOL=ON \ - -D GLPK_ROOT_DIR:PATH="/opt/local" \ - -D CMAKE_INSTALL_PREFIX:PATH="$prefix/gridpack-hadrec" \ - $common_flags .. - -elif [ $host == "olympus.local" ]; then - - prefix="/pic/projects/gridpack/software" - cmake $options \ - -D GA_DIR:STRING="/pic/projects/gridpack/ga-5-2" \ - -D GA_EXTRA_LIBS:STRING="-libverbs" \ - -D BOOST_ROOT:STRING="$prefix" \ - -D PETSC_DIR:STRING="$prefix/petsc-3.4.0" \ - -D PETSC_ARCH:STRING='olympus-openmpi-gnu-cxx-complex-opt' \ - -D MPI_CXX_COMPILER:STRING='mpicxx' \ - -D MPI_C_COMPILER:STRING='mpicc' \ - -D MPIEXEC:STRING='mpiexec' \ - $common_flags .. - - elif [ $host == "constance" ]; then CC=`which gcc` @@ -281,8 +232,8 @@ elif [ $host == "tlaloc" ]; then prefix="$HOME/Projects/GridPakLDRD/gridpack-install" cmake -Wdev --debug-trycompile \ --graphviz=GridPACK.dot \ - -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.14.6" \ - -D PETSC_ARCH:STRING="ubuntu-real-shared" \ + -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.16.6" \ + -D PETSC_ARCH:STRING="ubuntu-real-shared-debug" \ -D USE_OLD_PETSC:BOOL=OFF \ -D BOOST_ROOT:PATH="/usr" \ -D Boost_NO_BOOST_CMAKE:BOOL=TRUE \ From 7ac8094e06f4d8ed2d850dc1773c0e0882ee790c Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Thu, 14 Sep 2023 10:26:58 -0500 Subject: [PATCH 030/195] Test pipeline trigger --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b87d0b95..c5207349c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,4 +41,5 @@ test: script: - - ./install_gridpack.sh + - echo "hello world" + # - ./install_gridpack.sh From 7744e719120f3f5a65d53dafeafb9cdcbcdc3cea Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Thu, 14 Sep 2023 10:32:51 -0500 Subject: [PATCH 031/195] Build container, make dep from test optional --- .gitlab-ci.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c5207349c..e9093731c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,18 +26,20 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/Dockerfile" --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF}" - rules: + # rules: - - if: $CI_PIPELINE_SOURCE == "branch" - changes: [ "dockerfile", "install/*" ] - - if: $CI_PIPELINE_SOURCE == "tag" + # - if: $CI_PIPELINE_SOURCE == "branch" + # changes: [ "dockerfile", "install/*" ] + # - if: $CI_PIPELINE_SOURCE == "tag" test: image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF} - needs: [ build-container ] + needs: + - job: build-container + optional: true script: From 9cc9d04b11c62f0761fc5a4f2668fd5afa7a4a24 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Thu, 14 Sep 2023 10:46:52 -0500 Subject: [PATCH 032/195] Update dockerfile path --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e9093731c..55c2e2bf2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,7 +23,7 @@ build-container: --context "${CI_PROJECT_DIR}" --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy - --dockerfile "${CI_PROJECT_DIR}/Dockerfile" + --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF}" # rules: From 09dd060ec902048380231ab5e1f7feb2cc81f2a7 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 27 Sep 2023 11:26:41 -0500 Subject: [PATCH 033/195] Increase k8s memory limit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 55c2e2bf2..43d09ce3e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ variables: GP_EXT_DEPS: /gridpack-dependencies LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - + KUBERNETES_MEMORY_LIMIT: 4Gi # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy build-container: From 1a5b94a892b2676a985f9d5918e62bd2ecd2e748 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 27 Sep 2023 12:39:43 -0500 Subject: [PATCH 034/195] Increase mem request/limit and job timeout --- .gitlab-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 43d09ce3e..fdac37d45 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,11 +7,14 @@ variables: GP_EXT_DEPS: /gridpack-dependencies LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - KUBERNETES_MEMORY_LIMIT: 4Gi + KUBERNETES_MEMORY_REQUEST: 4Gi + KUBERNETES_MEMORY_LIMIT: 6Gi # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy build-container: + timeout: 3 hours + image: name: gcr.io/kaniko-project/executor:v1.9.0-debug From 60572f0f775fce7e5fd04b5944ead667284ae84a Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 27 Sep 2023 15:51:45 -0500 Subject: [PATCH 035/195] Add cpu and increase timeout further --- .gitlab-ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fdac37d45..19308aff4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,13 +7,17 @@ variables: GP_EXT_DEPS: /gridpack-dependencies LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - KUBERNETES_MEMORY_REQUEST: 4Gi - KUBERNETES_MEMORY_LIMIT: 6Gi + + KUBERNETES_CPU_REQUEST: "500m" + KUBERNETES_CPU_LIMIT: "1" + KUBERNETES_MEMORY_REQUEST: "4Gi" + KUBERNETES_MEMORY_LIMIT: "6Gi" + # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy build-container: - timeout: 3 hours + timeout: 5 hours image: From 50adb0d824992b147fe79842a281a4fdc82cce82 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 2 Oct 2023 11:49:47 -0500 Subject: [PATCH 036/195] Troubleshooting env --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 19308aff4..ae5e17b0b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,6 +26,7 @@ build-container: script: + - env - /kaniko/executor --context "${CI_PROJECT_DIR}" --build-arg http_proxy=$http_proxy From 8d052b2064bfe643e6c8342d4e3d71d0ee1cd91d Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 2 Oct 2023 11:59:13 -0500 Subject: [PATCH 037/195] Use correct predefined CI variable --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ae5e17b0b..0f36dde68 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,13 +26,12 @@ build-container: script: - - env - /kaniko/executor --context "${CI_PROJECT_DIR}" --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --dockerfile "${CI_PROJECT_DIR}/dockerfile" - --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF}" + --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}" # rules: @@ -43,7 +42,7 @@ build-container: test: - image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF} + image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} needs: - job: build-container From e0cecaafb966c3c00ca370badccbcada6c3aef25 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 2 Oct 2023 15:06:36 -0500 Subject: [PATCH 038/195] Add conditions for building container And enable real test job --- .gitlab-ci.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f36dde68..da56af30d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,11 +33,11 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}" - # rules: + rules: - # - if: $CI_PIPELINE_SOURCE == "branch" - # changes: [ "dockerfile", "install/*" ] - # - if: $CI_PIPELINE_SOURCE == "tag" + - if: $CI_PIPELINE_SOURCE == "branch" + changes: [ "dockerfile", "install/*" ] + - if: $CI_PIPELINE_SOURCE == "tag" test: @@ -50,5 +50,4 @@ test: script: - - echo "hello world" - # - ./install_gridpack.sh + - ./install_gridpack.sh From 4d60837a5da16bfa6302d6a52096280be9193eed Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 3 Oct 2023 09:02:16 -0500 Subject: [PATCH 039/195] Adjust k8s parameters and gitlab job timeout --- .gitlab-ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index da56af30d..878580f0e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,10 +8,10 @@ variables: GP_EXT_DEPS: /gridpack-dependencies LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - KUBERNETES_CPU_REQUEST: "500m" - KUBERNETES_CPU_LIMIT: "1" + KUBERNETES_CPU_REQUEST: "2" + KUBERNETES_CPU_LIMIT: "4" KUBERNETES_MEMORY_REQUEST: "4Gi" - KUBERNETES_MEMORY_LIMIT: "6Gi" + KUBERNETES_MEMORY_LIMIT: "8Gi" # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy @@ -40,11 +40,14 @@ build-container: - if: $CI_PIPELINE_SOURCE == "tag" -test: +build-gridpack: + + timeout: 5 hours image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} needs: + - job: build-container optional: true From 503dabdc508bc26d8bde2009c7a7265bf58d6ef2 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 3 Oct 2023 09:09:48 -0500 Subject: [PATCH 040/195] Update CPU limit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 878580f0e..c7954635a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ variables: LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} KUBERNETES_CPU_REQUEST: "2" - KUBERNETES_CPU_LIMIT: "4" + KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" From a07da9a0fc3ca5b2c5902ed44bf534406015c8b8 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 3 Oct 2023 10:08:16 -0500 Subject: [PATCH 041/195] Adjust CPU req/limits --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c7954635a..754b59834 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,8 +8,8 @@ variables: GP_EXT_DEPS: /gridpack-dependencies LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - KUBERNETES_CPU_REQUEST: "2" - KUBERNETES_CPU_LIMIT: "2" + KUBERNETES_CPU_REQUEST: "1500m" + KUBERNETES_CPU_LIMIT: "6" KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" From 3f0fca80874bae699ebd53d7d690ca239d4730d6 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 3 Oct 2023 10:14:38 -0500 Subject: [PATCH 042/195] Adjust CPU limit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 754b59834..128ebf2f3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ variables: LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} KUBERNETES_CPU_REQUEST: "1500m" - KUBERNETES_CPU_LIMIT: "6" + KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" From 263135fc745c9e3cf12ff3540b61163e5b398b7f Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 3 Oct 2023 11:57:28 -0500 Subject: [PATCH 043/195] Adjust cpu request --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 128ebf2f3..3af2de4ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: GP_EXT_DEPS: /gridpack-dependencies LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - KUBERNETES_CPU_REQUEST: "1500m" + KUBERNETES_CPU_REQUEST: "1" KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" From 7d97be1814deb2f88978ac75c3462152ac3d5f36 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 3 Oct 2023 15:47:06 -0500 Subject: [PATCH 044/195] Set GRIDPACK_DIR --- install_gridpack.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 3d0f53d27..1cb799cbc 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -1,6 +1,6 @@ #! /bin/bash -# installs GridPACK and its python wrappter +# installs GridPACK and its python wrapper # gridPACK is built in src/build and installed to src/install # run this from the top level GridPACK directory @@ -71,7 +71,10 @@ function install_gridpack_python { pushd python || exit - export RHEL_OPENMPI_HACK=yes + os_id=$(source /etc/os-release; echo "$ID") + if [[ $os_id == "rhel" ]] || [[ $os_id == "centos" ]]; then + export RHEL_OPENMPI_HACK=yes + fi # set python executable path python_exe=$(which python || which python3) @@ -79,6 +82,9 @@ function install_gridpack_python { # remove existing build dir rm -rf build + # export GRIDPACK_DIR + export GRIDPACK_DIR="${gridpack_install_dir}" + # build echo "Building GridPACK python wrapper" ${python_exe} setup.py build @@ -93,7 +99,7 @@ function install_gridpack_python { # install echo "Installing GridPACK python wrapper" - ${python_exe} setup.py install --home="$GRIDPACK_DIR" + ${python_exe} setup.py install --home="$gridpack_install_dir" popd || exit From 8ba354292847c80e6cf12984a6668128cf7600ae Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 3 Oct 2023 16:41:20 -0500 Subject: [PATCH 045/195] Uncomment gp_include and gp_libs --- src/lib/GridPACK.cmake.in | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib/GridPACK.cmake.in b/src/lib/GridPACK.cmake.in index eaa59dc56..c08a55776 100644 --- a/src/lib/GridPACK.cmake.in +++ b/src/lib/GridPACK.cmake.in @@ -233,15 +233,15 @@ function(gridpack_setup) list(APPEND gp_libs @GLPK_LIBRARY@) endif() -# list(APPEND gp_include -# @Boost_INCLUDE_DIR@ -# @MPI_INCLUDE_PATH@ -# ) - -# list(APPEND gp_libs -# @Boost_LIBRARIES@ -# @MPI_CXX_LIBRARIES@ -# ) + list(APPEND gp_include + @Boost_INCLUDE_DIR@ + @MPI_INCLUDE_PATH@ + ) + + list(APPEND gp_libs + @Boost_LIBRARIES@ + @MPI_CXX_LIBRARIES@ + ) set(GRIDPACK_INCLUDE_DIRS ${gp_include} From 9b533f022e8c56bd9b4aed1348f86374aa1e979b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 4 Oct 2023 12:38:31 -0500 Subject: [PATCH 046/195] Add Boost_NO_BOOST_CMAKE --- install_gridpack.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install_gridpack.sh b/install_gridpack.sh index 1cb799cbc..56396e7d9 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -46,6 +46,7 @@ function install_gridpack { -D CMAKE_BUILD_TYPE:STRING=Debug \ -D BUILD_SHARED_LIBS=YES \ -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ + -D Boost_NO_BOOST_CMAKE:BOOL=TRUE \ .. # install From 48d1adc4f48c1ce8edb6f9151a1e86fdb9bcad2e Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 4 Oct 2023 13:27:01 -0500 Subject: [PATCH 047/195] Set max number of make jobs --- .gitlab-ci.yml | 4 ++++ install_gridpack.sh | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3af2de4ca..b7082ee9b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,6 +46,10 @@ build-gridpack: image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} + variables: + + MAKE_JOBS: "2" + needs: - job: build-container diff --git a/install_gridpack.sh b/install_gridpack.sh index 56396e7d9..a0dfa5031 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -51,7 +51,7 @@ function install_gridpack { # install echo "Installing GridPACK" - make -j "$(nproc)" install + make -j "${MAKE_JOBS:-$(nproc)}" install popd || exit @@ -72,7 +72,10 @@ function install_gridpack_python { pushd python || exit - os_id=$(source /etc/os-release; echo "$ID") + os_id=$( + source /etc/os-release + echo "$ID" + ) if [[ $os_id == "rhel" ]] || [[ $os_id == "centos" ]]; then export RHEL_OPENMPI_HACK=yes fi From e88724d5a942c6fc491f3ecfeb9dfde3be15d524 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 4 Oct 2023 14:07:03 -0500 Subject: [PATCH 048/195] Only append python path if exists --- install_gridpack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index a0dfa5031..072d883c6 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -99,7 +99,7 @@ function install_gridpack_python { mkdir -p "${py_lib}" # add lib to python path - export PYTHONPATH="${py_lib}:${PYTHONPATH}" + export PYTHONPATH="${py_lib}${PYTHONPATH:+:$PYTHONPATH}" # install echo "Installing GridPACK python wrapper" From a442f45ee3d0cadd1f0266bc5d877192aec4c597 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 11 Oct 2023 11:33:24 -0500 Subject: [PATCH 049/195] Add comment --- install_gridpack.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install_gridpack.sh b/install_gridpack.sh index 072d883c6..a509fdfd9 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -72,6 +72,7 @@ function install_gridpack_python { pushd python || exit + # set an env var if we are running on RHEL os_id=$( source /etc/os-release echo "$ID" From 1107658c42ed387be1ece8d4b28c66b90cda2052 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 11 Oct 2023 11:33:37 -0500 Subject: [PATCH 050/195] Add test job --- .gitlab-ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b7082ee9b..f23bd8591 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,3 +58,19 @@ build-gridpack: script: - ./install_gridpack.sh + + + artifacts: + + untracked: true + + +test-gridpack: + + image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} + + needs: [ build-gridpack ] + + script: + + - make test From 725cf47ffcac4c8f4b0014d2141f100d6dd03957 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 11 Oct 2023 12:01:06 -0500 Subject: [PATCH 051/195] Use default timeout for build --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f23bd8591..db5bae442 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,8 +42,6 @@ build-container: build-gridpack: - timeout: 5 hours - image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} variables: From 920d93e41366ec517f746497fa91aa4d6876bb84 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 11 Oct 2023 10:40:29 -0700 Subject: [PATCH 052/195] Remove 2000-bus Wind-DSA smoke tests - they take too long --- src/applications/development/wind_dsa/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/applications/development/wind_dsa/CMakeLists.txt b/src/applications/development/wind_dsa/CMakeLists.txt index 058257bd8..315e8c2d9 100644 --- a/src/applications/development/wind_dsa/CMakeLists.txt +++ b/src/applications/development/wind_dsa/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2023-08-31 09:05:17 d3g096 +# Last Change: 2023-10-11 10:33:21 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -150,8 +150,8 @@ add_dependencies(wind2.x wind.x.input) # ------------------------------------------------------------- gridpack_add_run_test("wind_dsa" wind.x input.xml) gridpack_add_run_test("wind_dsa_2" wind2.x input.xml) -gridpack_add_run_test("wind_dsa_tamu2000" wind.x input_tamu2000_dsf.xml) -gridpack_add_run_test("wind_dsa_2_tamu2000" wind2.x input_tamu2000_dsf.xml) +# gridpack_add_run_test("wind_dsa_tamu2000" wind.x input_tamu2000_dsf.xml) +# gridpack_add_run_test("wind_dsa_2_tamu2000" wind2.x input_tamu2000_dsf.xml) # ------------------------------------------------------------- # install as a sample application From f5e91bc8278ca99351408db4424b78fee322ffcf Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 11 Oct 2023 10:41:32 -0700 Subject: [PATCH 053/195] Switch to a direct linear solver because this was diverging --- .../development/wind_dsa/input.xml | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/applications/development/wind_dsa/input.xml b/src/applications/development/wind_dsa/input.xml index 4a6b36bd1..d290f615c 100644 --- a/src/applications/development/wind_dsa/input.xml +++ b/src/applications/development/wind_dsa/input.xml @@ -51,11 +51,10 @@ 1.0E-08 50 - -ksp_type bicg - -pc_type bjacobi - -sub_pc_type ilu -sub_pc_factor_levels 5 -sub_ksp_type preonly - + -ksp_type richardson + -pc_type lu + -pc_factor_mat_solver_type superlu_dist + -ksp_max_it 1 @@ -64,13 +63,10 @@ 1.0E-05 50 - -ksp_type bicg - -pc_type bjacobi - -sub_pc_type ilu -sub_pc_factor_levels 5 -sub_ksp_type preonly - + -ksp_type richardson + -pc_type lu + -pc_factor_mat_solver_type superlu_dist + -ksp_max_it 1 @@ -110,7 +106,7 @@ -ksp_type richardson -pc_type lu - -pc_factor_mat_solver_type petsc + -pc_factor_mat_solver_type superlu_dist -pc_factor_shift_type NONZERO -ksp_max_it 1 From d7d8907fa6ae22275c69b9814cb15484e5b1629a Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 16 Oct 2023 13:18:21 -0500 Subject: [PATCH 054/195] Whitespace change --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index db5bae442..4099b5be0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,7 +57,6 @@ build-gridpack: - ./install_gridpack.sh - artifacts: untracked: true From 1231f64ca1c4ee7c68f3d01d6c4f732c03b2f895 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 16 Oct 2023 13:56:36 -0500 Subject: [PATCH 055/195] Troubleshooting --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4099b5be0..9c1663e18 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,4 +70,5 @@ test-gridpack: script: + - find . -type d -maxdepth 4 - make test From d4c0929badb81ebaa0dc85e6e53d1dc4264cabbc Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 17 Oct 2023 01:01:06 -0500 Subject: [PATCH 056/195] Revert block comment --- src/lib/GridPACK.cmake.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/GridPACK.cmake.in b/src/lib/GridPACK.cmake.in index 2d47cbe57..e32e09989 100644 --- a/src/lib/GridPACK.cmake.in +++ b/src/lib/GridPACK.cmake.in @@ -238,10 +238,10 @@ function(gridpack_setup) @MPI_INCLUDE_PATH@ ) -# list(APPEND gp_libs -# @Boost_LIBRARIES@ -# @MPI_CXX_LIBRARIES@ -# ) + list(APPEND gp_libs + @Boost_LIBRARIES@ + @MPI_CXX_LIBRARIES@ + ) set(GRIDPACK_INCLUDE_DIRS ${gp_include} From 9f7746186c2727045c8eafa28277a3875576af48 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 17 Oct 2023 01:01:22 -0500 Subject: [PATCH 057/195] Fix test command and save results --- .gitlab-ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c1663e18..c31219db9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,5 +70,10 @@ test-gridpack: script: - - find . -type d -maxdepth 4 - - make test + - make -C src/build test + + artifacts: + + paths: + + - src/build/Testing/Temporary From 1fd0ea7bfa57934e3ec23a07b87971803553dd02 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 17 Oct 2023 02:06:58 -0500 Subject: [PATCH 058/195] Always save artifacts for the test job --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c31219db9..538c9375d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,3 +77,5 @@ test-gridpack: paths: - src/build/Testing/Temporary + + when: always From ea037d83defdb82a85c7c98cef3f9885ceed92d8 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 17 Oct 2023 03:02:52 -0500 Subject: [PATCH 059/195] Remove version numbers from LD_LIBRARY_PATH --- dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfile b/dockerfile index 1a0ee60f3..72a728933 100644 --- a/dockerfile +++ b/dockerfile @@ -8,7 +8,7 @@ ENV GP_EXT_DEPS=/gridpack-dependencies ENV BOOST_VERSION "1.78.0" ENV GA_VERSION "5.8" ENV PETSC_VERSION "3.16.4" -ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} +ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost/install_for_gridpack/lib:${GP_EXT_DEPS}/ga/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} WORKDIR ${GP_EXT_DEPS} From 098a8465f95bfd77386fe4d8ca5c36179258858f Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 17 Oct 2023 03:11:01 -0500 Subject: [PATCH 060/195] Run tests as non-root user --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 538c9375d..ebd5bef07 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,7 +70,9 @@ test-gridpack: script: - - make -C src/build test + - useradd gridpack + - chown -R gridpack:gridpack . + - su gridpack -c 'make -C src/build test' artifacts: From 22b88f52e1b47b41ccb219881e30d0c0f0ae1bf5 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 17 Oct 2023 07:58:27 -0500 Subject: [PATCH 061/195] Fix rules for build-container job --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ebd5bef07..d1096dbe1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,9 +35,10 @@ build-container: rules: - - if: $CI_PIPELINE_SOURCE == "branch" - changes: [ "dockerfile", "install/*" ] + - if: $CI_PIPELINE_SOURCE == "push" + changes: [ "dockerfile" ] - if: $CI_PIPELINE_SOURCE == "tag" + - when: manual build-gridpack: From ddf9948cfdcd253242309a864618dd4631356bd8 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 23 Oct 2023 06:49:29 -0500 Subject: [PATCH 062/195] Don't override env vars --- .gitlab-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d1096dbe1..d482146ac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,9 +5,6 @@ default: variables: - GP_EXT_DEPS: /gridpack-dependencies - LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - KUBERNETES_CPU_REQUEST: "1" KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" From fc96b3661b6519c522f36d0460a5f5af9000ac13 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 23 Oct 2023 06:49:58 -0500 Subject: [PATCH 063/195] Remove manual container build rule --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d482146ac..edfa1d193 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,6 @@ build-container: - if: $CI_PIPELINE_SOURCE == "push" changes: [ "dockerfile" ] - if: $CI_PIPELINE_SOURCE == "tag" - - when: manual build-gridpack: From 45b04639b9a5f59a4bc0b3a4caaafe46f5508554 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 23 Oct 2023 06:57:59 -0500 Subject: [PATCH 064/195] Add build status badge --- README.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 08d243154..5b7422099 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,9 @@ # GridPACKTM--> # GridPACK: High-Performance Electric Grid Simulation +![PNNL Build Status](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fapi.github.com%2Frepos%2FGridOPTICS%2FGridPACK%2Fcommits%2Fdevelop%2Fstatus&query=%24.state&logo=gitlab&label=build%20status&color=white +) + GridPACK is an open-source high-performance (HPC) package for simulation of large-scale electrical grids. Powered by distributed (parallel) computing and high-performance numerical solvers, GridPACK offers several applications forfast simulation of electrical transmission systems. GridPACK includes a number of prebuilt applications that can be directly used. The most commonly used and well-developed are: - AC Power Flow - Dynamics Simulation @@ -18,7 +21,7 @@ In addition, GridPACK is also a framework to simplify the development of new app See the [instructions](docs/markdown/BASIC_INSTALL.md) for installing GridPACK, prerequisite software, and installation notes for different platforms. ## Usage -See [User manual](docs/user_manual/GridPACK.pdf) for a deep dive on GridPACK internals and/or refer to the [tutorials](docs/markdown/TUTORIALS.md) for more info. +See [User manual](docs/user_manual/GridPACK.pdf) for a deep dive on GridPACK internals and/or refer to the [tutorials](docs/markdown/TUTORIALS.md) for more info. - Quick Guide (To do) @@ -36,16 +39,16 @@ The best (and fastest) way to reach us for any technical questions is by posting ## Citing GridPACK ``` -@article{doi:10.1177/1094342015607609, -author = {Bruce Palmer and William Perkins and Yousu Chen and Shuangshuang Jin and David C allahan and Kevin Glass and Ruisheng Diao and Mark Rice and Stephen Elbert and Mallikarjun a Vallem and Zhenyu Huang}, -title ={GridPACKTM: A framework for developing power grid simulations on high-performance computing platforms}, -journal = {The International Journal of High Performance Computing Applications}, -volume = {30}, -number = {2}, -pages = {223-240}, -year = {2016}, -doi = {10.1177/1094342015607609}, -URL = {https://doi.org/10.1177/1094342015607609}, +@article{doi:10.1177/1094342015607609, +author = {Bruce Palmer and William Perkins and Yousu Chen and Shuangshuang Jin and David C allahan and Kevin Glass and Ruisheng Diao and Mark Rice and Stephen Elbert and Mallikarjun a Vallem and Zhenyu Huang}, +title ={GridPACKTM: A framework for developing power grid simulations on high-performance computing platforms}, +journal = {The International Journal of High Performance Computing Applications}, +volume = {30}, +number = {2}, +pages = {223-240}, +year = {2016}, +doi = {10.1177/1094342015607609}, +URL = {https://doi.org/10.1177/1094342015607609}, eprint = {https://doi.org/10.1177/1094342015607609} ``` @@ -69,7 +72,7 @@ GridPACK has been developed through funding from various sources over the years. ## Copyright Copyright © 2013, Battelle Memorial Institute. -GridPACKTM is a free software distributed under a BSD 2-clause license. You may reuse, modify, and redistribute the software. +GridPACKTM is a free software distributed under a BSD 2-clause license. You may reuse, modify, and redistribute the software. See the [license](src/LICENSE.md) file for details. From d71c50cf974980e433d3dfcdf92e255908362f93 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 24 Oct 2023 16:11:03 -0500 Subject: [PATCH 065/195] Only use two processors --- install_gridpack.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install_gridpack.sh b/install_gridpack.sh index a509fdfd9..688832d4e 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -47,6 +47,7 @@ function install_gridpack { -D BUILD_SHARED_LIBS=YES \ -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ -D Boost_NO_BOOST_CMAKE:BOOL=TRUE \ + -D MPIEXEC_MAX_NUMPROCS:STRING="2" \ .. # install From 0fe20f01db64c25f7363c5024d893d7b7cf81e38 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 8 Nov 2023 10:52:48 -0600 Subject: [PATCH 066/195] Show output for failed tests --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index edfa1d193..bbb6ad26d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -69,7 +69,7 @@ test-gridpack: - useradd gridpack - chown -R gridpack:gridpack . - - su gridpack -c 'make -C src/build test' + - su gridpack -c 'ctest --test-dir src/build --output-on-failure' artifacts: From b7003fc5df4cbe7a593e9c88e13a41cf8227f534 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Thu, 28 Dec 2023 15:04:02 -0600 Subject: [PATCH 067/195] Test with multiple distributions --- .gitlab-ci.yml | 26 +++++++++++++++++++++++--- dockerfile | 7 +++---- install_environment_packages.sh | 29 +++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 install_environment_packages.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bbb6ad26d..b3e38ecb7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,9 +11,24 @@ variables: KUBERNETES_MEMORY_LIMIT: "8Gi" +.multi-distro: + + parallel: + + matrix: + - BASE_IMAGE: rockylinux:9 + - BASE_IMAGE: ubuntu:22.04 + + variables: + + IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE%%:*}-${CI_COMMIT_REF_NAME}" + + # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy build-container: + extends: .multi-distro + timeout: 5 hours image: @@ -25,10 +40,11 @@ build-container: - /kaniko/executor --context "${CI_PROJECT_DIR}" + --build-arg BASE_IMAGE=${BASE_IMAGE} --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --dockerfile "${CI_PROJECT_DIR}/dockerfile" - --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}" + --destination "${IMAGE_PATH}" rules: @@ -39,7 +55,9 @@ build-container: build-gridpack: - image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} + extends: .multi-distro + + image: "${IMAGE_PATH}" variables: @@ -61,7 +79,9 @@ build-gridpack: test-gridpack: - image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} + extends: .multi-distro + + image: ${IMAGE_PATH} needs: [ build-gridpack ] diff --git a/dockerfile b/dockerfile index 72a728933..b40b5f5f6 100644 --- a/dockerfile +++ b/dockerfile @@ -1,8 +1,7 @@ -FROM ubuntu:22.04 +ARG BASE_IMAGE +FROM ${BASE_IMAGE} -RUN apt update \ - && apt upgrade -y \ - && apt install -y wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config +RUN install_environment_packages.sh ENV GP_EXT_DEPS=/gridpack-dependencies ENV BOOST_VERSION "1.78.0" diff --git a/install_environment_packages.sh b/install_environment_packages.sh new file mode 100644 index 000000000..9109cc8d7 --- /dev/null +++ b/install_environment_packages.sh @@ -0,0 +1,29 @@ +#! /bin/bash + +# installs necessary packages for a given distribution + +set -xeuo pipefail + +distribution=$( + # shellcheck source=/dev/null + source /etc/os-release + echo "$ID" +) + +case $distribution in +debian | ubuntu) + apt update && + apt upgrade -y && + apt install -y \ + wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config + ;; +fedora | rhel | centos | rocky) + dnf update -y && + dnf install -y \ + wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf + ;; +*) + echo "$distribution not supported" + exit 1 + ;; +esac From 0f5883dcd895629fff6986461397ce1b9bff3c80 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 29 Dec 2023 11:44:37 -0600 Subject: [PATCH 068/195] Back out changes to src --- src/CMakeLists.txt | 50 +++++++++---------- .../dynamic_simulation_full_y/CMakeLists.txt | 40 +++++++-------- src/cmake-modules/GridPACK.cmake | 16 +++--- src/example_configuration.sh | 36 +++++++------ src/lib/GridPACK.cmake.in | 14 +++--- 5 files changed, 81 insertions(+), 75 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 29db21417..9aa459965 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,7 +12,7 @@ # Created May 3, 2013 by William A. Perkins # ------------------------------------------------------------- -# +# cmake_minimum_required (VERSION 3.5.0) project (GridPACK) @@ -25,7 +25,7 @@ set (GridPACK_VERSION_PATCH 0) enable_language(CXX) # where to look for special .cmake files -list (APPEND CMAKE_MODULE_PATH +list (APPEND CMAKE_MODULE_PATH "${GridPACK_SOURCE_DIR}/cmake-modules" ) include(GridPACK) @@ -39,7 +39,7 @@ endif() # add GOSS directory option (GOSS_DIR "Point to directory with GOSS files" OFF) if (GOSS_DIR) - add_definitions (-DUSE_GOSS=1) + add_definitions (-DUSE_GOSS=1) include_directories(AFTER ${GOSS_DIR}/include/activemq-cpp-3.8.4) set(GOSS_INCLUDE "${GOSS_DIR}/include/activemq-cpp-3.8.4") set (GOSS_LIBRARY "${GOSS_DIR}/lib/libactivemq-cpp.a") @@ -159,14 +159,14 @@ include_directories(AFTER ${MPI_CXX_INCLUDE_PATH}) # set (CMAKE_FIND_LIBRARY_SUFFIXES ".a") #endif() -# This is here for cmake version 2.6 +# This is here for cmake version 2.6 if (NOT MPI_LIBRARY OR NOT MPI_EXTRA_LIBRARY) # Punt if MPI_LIBRARY or MPI_EXTRA_LIBRARY not found set(MPI_CXX_LIBRARIES "") else() if (NOT MPI_CXX_LIBRARIES) - set(MPI_CXX_LIBRARIES - ${MPI_LIBRARY} + set(MPI_CXX_LIBRARIES + ${MPI_LIBRARY} ${MPI_EXTRA_LIBRARY} ) endif() @@ -184,7 +184,7 @@ message(STATUS "MPI_EXTRA_LIBRARY: ${MPI_EXTRA_LIBRARY}") # Perkins found out that this was exactly the wrong thing to do: -# +# # set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER}) # set(CMAKE_CXX_COMPILE_FLAGS ${CMAKE_CXX_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS}) # set(CMAKE_CXX_LINK_FLAGS ${CMAKE_CXX_LINK_FLAGS} ${MPI_LINK_FLAGS}) @@ -197,12 +197,12 @@ message(STATUS "MPI_EXTRA_LIBRARY: ${MPI_EXTRA_LIBRARY}") # real problems, but there are so many of them they obscure errors # that are causing identifiable problems -if (CMAKE_COMPILER_IS_GNUCXX) +if (CMAKE_COMPILER_IS_GNUCXX) add_definitions( -pedantic -Wno-write-strings - -Wno-long-long - -Wno-sign-compare + -Wno-long-long + -Wno-sign-compare -Wno-unused-variable -Wno-unused-but-set-variable -Wno-maybe-uninitialized @@ -250,7 +250,7 @@ endif() option(BOOST_BUILD_PYTHON "Include Boost::Python in the Boost build" OFF) # These are the Boost libraries required here - + set(BOOST_BUILD_LIBS "") list(APPEND BOOST_BUILD_LIBS mpi serialization random filesystem system) @@ -294,8 +294,8 @@ endif() if (NOT PETSC_DIR) message(FATAL_ERROR "PETSC_DIR is not set, do so with -D PETSC_DIR=...") endif() - -# Include petsc package path in pkg_config_path + +# Include petsc package path in pkg_config_path set(ENV{PKG_CONFIG_PATH} ${PETSC_DIR}/lib/pkgconfig:${PETSC_DIR}/${PETSC_ARCH}/lib/pkgconfig ) @@ -314,7 +314,7 @@ petscconf.h") endif() # checks - # define PETSc variables + # define PETSc variables include(CheckSymbolExists) check_symbol_exists(PETSC_HAVE_PARMETIS ${petscconf} PETSC_HAVE_PARMETIS) check_symbol_exists(PETSC_USE_REAL_DOUBLE ${petscconf} PETSC_USE_REAL_DOUBLE) @@ -336,7 +336,7 @@ else() message(STATUS "PETSC_LIBRARY_SINGLE: ${PETSC_LIBRARY_SINGLE}") - # checks + # checks if (NOT PETSC_HAVE_MPI) message(FATAL_ERROR "PETSc installation is not parallel (--with-mpi=1)") @@ -348,13 +348,13 @@ endif() # (--with-scalar-type=real). This is to determine what that # underlying type is. -if (PETSC_USE_REAL_DOUBLE) +if (PETSC_USE_REAL_DOUBLE) message(STATUS "PETSc installation is double precision (--with-precision=double) -- good") else() message(FATAL_ERROR "PETSc installation is not double precision (--with-precision=double)") endif() -if (PETSC_USE_COMPLEX) +if (PETSC_USE_COMPLEX) message (STATUS "PETSc installation uses complex type (--with-scalar-type=complex)") else() message (STATUS "PETSc installation uses real type (--with-scalar-type=real)") @@ -365,7 +365,7 @@ if (PETSC_CLANGUAGE_Cxx) message (STATUS "PETSc installation uses C++ (--with-clanguage=c++) -- we can work with that.") endif() -if (PETSC_HAVE_SUPERLU_DIST) +if (PETSC_HAVE_SUPERLU_DIST) set(GRIDPACK_MATSOLVER_PKG "superlu_dist") message(STATUS "PETSc parallel LU linear solver will be from SuperLU_dist") elseif (PETSC_HAVE_MUMPS) @@ -378,7 +378,7 @@ endif() # ------------------------------------------------------------- # ParMETIS -# +# # Only versions > 4.0 have been used # ------------------------------------------------------------- message(STATUS "Checking ParMETIS ...") @@ -424,7 +424,7 @@ if(DOXYGEN_FOUND) COMMENT "Generating API documentation with Doxygen" VERBATIM ) - + # The custom target "devdoc" generates "developer" # documentation. This has all the nitty gritty details and code @@ -545,7 +545,7 @@ add_subdirectory(lib) INCLUDE(InstallRequiredSystemLibraries) -set(CPACK_PACKAGE_DESCRIPTION_SUMMARY +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "An open source toolkit for developing power grid simulation applications for high performance computing architectures") set(CPACK_PACKAGE_DESCRIPTION "GridPACK(tm) is a software framework consisting of a set of modules designed to simplify the development of programs that model the power grid and run on parallel, high performance computing platforms. The modules are available as a library and consist of components for setting up and distributing power grid networks, support for modeling the behavior of individual buses and branches in the network, converting the network models to the corresponding algebraic equations, and parallel routines for manipulating and solving large algebraic systems. Additional modules support input and output as well as basic profiling and error management. ") set(CPACK_PACKAGE_VENDOR "") @@ -553,11 +553,11 @@ set(CPACK_PACKAGE_CONTACT "william.perkins@pnnl.gov") set(CPACK_PACKAGE_VERSION_MAJOR "${GridPACK_VERSION_MAJOR}") set(CPACK_PACKAGE_VERSION_MINOR "${GridPACK_VERSION_MINOR}") set(CPACK_PACKAGE_VERSION_PATCH "${GridPACK_VERSION_PATCH}") -set(CPACK_PACKAGE_VERSION +set(CPACK_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") -set(CPACK_PACKAGE_FILE_NAME +set(CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}_${CPACK_PACKAGE_VERSION}") -set(CPACK_SOURCE_PACKAGE_FILE_NAME +set(CPACK_SOURCE_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}_${CPACK_PACKAGE_VERSION}") # set(CPACK_PACKAGING_INSTALL_PREFIX "") @@ -571,7 +571,7 @@ set(CPACK_SET_DESTDIR true) set(CPACK_DEBIAN_PACKAGE_PRIORITY "optional") set(CPACK_DEBIAN_PACKAGE_SECTION "science") set(CPACK_DEBIAN_ARCHITECTURE ${CMAKE_SYSTEM_PROCESSOR}) -set(CPACK_DEBIAN_PACKAGE_DEPENDS +set(CPACK_DEBIAN_PACKAGE_DEPENDS "libga-dev (>=5.3), libpetsc3.6.2-dev (>= 3.6.2), libparmetis-dev (>=4.0.3), libboost-mpi-dev (>=1.58.0), libboost-all-dev (>=1.58.0)" ) diff --git a/src/applications/dynamic_simulation_full_y/CMakeLists.txt b/src/applications/dynamic_simulation_full_y/CMakeLists.txt index b1117e357..4ea60b49f 100644 --- a/src/applications/dynamic_simulation_full_y/CMakeLists.txt +++ b/src/applications/dynamic_simulation_full_y/CMakeLists.txt @@ -33,7 +33,7 @@ set(target_libraries gridpack_parallel gridpack_block_parsers ${PETSC_LIBRARIES} - ${PARMETIS_LIBRARY} ${METIS_LIBRARY} + ${PARMETIS_LIBRARY} ${METIS_LIBRARY} ${Boost_LIBRARIES} ${GA_LIBRARIES} ${MPI_CXX_LIBRARIES}) @@ -124,60 +124,60 @@ gridpack_set_lu_solver( ) add_custom_target(dsf.x.input - - COMMAND ${CMAKE_COMMAND} -E copy + + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/raw/IEEE_145bus_v23_PSLF.raw ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/dyr/IEEE_145b_classical_model.dyr ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/raw/9b3g.raw ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/dyr/9b3g.dyr ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/raw/300bus_v23_no0imp_pslf.raw ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/dyr/300bus_detail_model_cmpld_combine.dyr ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/raw/bus3000_gen_no0imp_v23_pslf.raw ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/dyr/classical_model_3000bus.dyr ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/dyr/240busWECC_2018_PSS_mod.dyr ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/raw/240busWECC_2018_PSS_fixedshunt.raw - ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/dyr/kundur-twoarea.dyr ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/raw/kundur-twoarea_v33.raw - ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/dyr/kundur-twoarea_4renewable_mech.dyr ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/input_145.xml ${GRIDPACK_DATA_DIR}/raw/IEEE_145bus_v23_PSLF.raw ${GRIDPACK_DATA_DIR}/dyr/IEEE_145b_classical_model.dyr @@ -212,7 +212,7 @@ endif() configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.install.in ${CMAKE_CURRENT_BINARY_DIR}/CMakeLists.txt @ONLY) -install(FILES +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/CMakeLists.txt ${CMAKE_CURRENT_BINARY_DIR}/input_145.xml ${GRIDPACK_DATA_DIR}/raw/IEEE_145bus_v23_PSLF.raw @@ -233,7 +233,7 @@ install(FILES install(TARGETS dsf.x DESTINATION bin) install(TARGETS dsf2.x DESTINATION bin) - + # ------------------------------------------------------------- # run application as test # ------------------------------------------------------------- diff --git a/src/cmake-modules/GridPACK.cmake b/src/cmake-modules/GridPACK.cmake index 34ef15a05..cfedfe308 100644 --- a/src/cmake-modules/GridPACK.cmake +++ b/src/cmake-modules/GridPACK.cmake @@ -17,7 +17,7 @@ # This is used to specify a time out for GridPACK unit tests. It's 5 # seconds by default, but may need to be longer on some platforms. -if (NOT GRIDPACK_TEST_TIMEOUT) +if (NOT GRIDPACK_TEST_TIMEOUT) set (GRIDPACK_TEST_TIMEOUT 120 CACHE STRING "Time out for GridPACK unit tests.") endif () @@ -61,7 +61,7 @@ function(gridpack_add_serial_unit_test test_name test_target) set(the_test_name "${test_name}_serial") add_test("${the_test_name}" "${test_target}") set_tests_properties("${the_test_name}" - PROPERTIES + PROPERTIES PASS_REGULAR_EXPRESSION "No errors detected" FAIL_REGULAR_EXPRESSION "failure detected" TIMEOUT ${GRIDPACK_TEST_TIMEOUT} @@ -80,7 +80,7 @@ function(gridpack_add_serial_run_test test_name test_target test_input) set(the_test_name "${test_name}_serial") add_test("${the_test_name}" "${test_target}" ${test_input}) set_tests_properties("${the_test_name}" - PROPERTIES + PROPERTIES TIMEOUT ${GRIDPACK_TEST_TIMEOUT} ) set_tests_ldpath("${the_test_name}") @@ -95,13 +95,13 @@ function(gridpack_add_parallel_unit_test test_name test_target) add_test("${the_test_name}" ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} ${test_target} ${MPIEXEC_POSTFLAGS}) set_tests_properties("${the_test_name}" - PROPERTIES + PROPERTIES PASS_REGULAR_EXPRESSION "No errors detected" FAIL_REGULAR_EXPRESSION "failure detected" TIMEOUT ${GRIDPACK_TEST_TIMEOUT} ) set_tests_ldpath("${the_test_name}") - else() + else() message(FATAL_ERROR "gridpack_add_parallel_unit_test: target argument not target") endif() endfunction(gridpack_add_parallel_unit_test) @@ -119,7 +119,7 @@ function(gridpack_add_parallel_run_test test_name test_target test_input) add_test("${the_test_name}" ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} ${test_target} ${MPIEXEC_POSTFLAGS} ${test_input}) set_tests_properties("${the_test_name}" - PROPERTIES + PROPERTIES TIMEOUT ${GRIDPACK_TEST_TIMEOUT} ) set_tests_ldpath("${the_test_name}") @@ -140,7 +140,7 @@ function(gridpack_add_unit_test test_name test_target) if (NOT USE_PROGRESS_RANKS) gridpack_add_serial_unit_test("${test_name}" ${test_target}) endif() - if (MPIEXEC) + if (MPIEXEC) gridpack_add_parallel_unit_test("${test_name}" ${test_target}) endif () endfunction(gridpack_add_unit_test) @@ -157,7 +157,7 @@ function(gridpack_add_run_test test_name test_target test_input) if (NOT USE_PROGRESS_RANKS) gridpack_add_serial_run_test("${test_name}" ${test_target} "${test_input}") endif() - if (MPIEXEC) + if (MPIEXEC) gridpack_add_parallel_run_test("${test_name}" ${test_target} "${test_input}") endif () endfunction(gridpack_add_run_test) diff --git a/src/example_configuration.sh b/src/example_configuration.sh index 3ba337e70..52d459598 100755 --- a/src/example_configuration.sh +++ b/src/example_configuration.sh @@ -121,7 +121,7 @@ elif [ $host == "we32673" ]; then CC=/opt/local/bin/clang export CC CXX=/opt/local/bin/clang++ - export CC CXX + export CC CXX prefix="/Users/d3g096/Projects/GridPACK" pdir="$prefix/petsc.gitlab" @@ -132,7 +132,7 @@ elif [ $host == "we32673" ]; then else parch="darwin-mpich-clang-complex-opt-3.16-static" fi - + cmake $options \ --graphviz=GridPACK.dot \ -D GA_DIR:STRING="$prefix/gridpack-install" \ @@ -204,8 +204,8 @@ elif [ $host == "tlaloc" ]; then # Ubuntu 20 with as many system packages as possible: GNU # compilers 9.4.0, OpenMPI 4.0.3, Boost 1.71.0, PETSc 3.12, - # ParMETIS 4.0.3. - + # ParMETIS 4.0.3. + CC=gcc CXX=g++ CFLAGS=-pthread @@ -232,21 +232,27 @@ elif [ $host == "tlaloc" ]; then # -D PETSC_ARCH:STRING="ubuntu-real-shared" \ # -D USE_OLD_PETSC:BOOL=OFF \ - prefix="$HOME/Projects/GridPakLDRD/gridpack-install" - cmake -Wdev --debug-trycompile \ - - # Custom built 3.12.5, real: - # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.12.5" \ - # -D PETSC_ARCH:STRING="ubuntu-real-shared-3.12" \ + # Custom built 3.19, complex: + # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.19.4" \ + # -D PETSC_ARCH:STRING="ubuntu-complex-shared" \ # -D USE_OLD_PETSC:BOOL=OFF \ - # Custom built 3.10.5, real: - # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.10.5" \ - # -D PETSC_ARCH:STRING="ubuntu-real-shared-3.10" \ + # Custom built 3.19, real: + # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.19.4" \ + # -D PETSC_ARCH:STRING="ubuntu-real-shared" \ # -D USE_OLD_PETSC:BOOL=OFF \ + # Custom built 3.19, complex, w/o superlu_dist: + # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.19.4" \ + # -D PETSC_ARCH:STRING="ubuntu-complex-shared-mumps" \ + # -D USE_OLD_PETSC:BOOL=OFF \ - prefix="$HOME/Projects/GridPakLDRD/gridpack-install" + if [ -z "$GRIDPACK_DIR" ]; then + prefix="$HOME/Projects/ExaLearn/gridpack-install" + else + prefix="$GRIDPACK_DIR" + fi + cmake -Wdev --debug-trycompile \ --graphviz=GridPACK.dot \ -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.14.6" \ @@ -318,6 +324,6 @@ else echo "Unknown host: $host" exit 2 - + fi diff --git a/src/lib/GridPACK.cmake.in b/src/lib/GridPACK.cmake.in index 856faf52e..5e04e1617 100644 --- a/src/lib/GridPACK.cmake.in +++ b/src/lib/GridPACK.cmake.in @@ -20,7 +20,7 @@ function(gridpack_setup) endif() set(gp_include "${GRIDPACK_DIR}/include") - + set(GRIDPACK_LIB_DIR "${GRIDPACK_DIR}/lib") set(GRIDPACK_HAVE_PETSC @PETSC_FOUND@) @@ -80,7 +80,7 @@ function(gridpack_setup) ) message(STATUS "GRIDPACK_HAVE_GOSS: ${GRIDPACK_HAVE_GOSS}") - if (GRIDPACK_HAVE_GOSS) + if (GRIDPACK_HAVE_GOSS) message(STATUS "Adding GRIDPACK_GOSS library") find_library(GRIDPACK_GOSS_LIBRARY NAMES gridpack_goss @@ -162,12 +162,12 @@ function(gridpack_setup) ${GRIDPACK_PARALLEL_LIBRARY} ) - if (GRIDPACK_HAVE_PARMETIS) + if (GRIDPACK_HAVE_PARMETIS) list(APPEND gp_include @PARMETIS_INCLUDE_DIR@ ) list(APPEND gp_libs - @PARMETIS_LIBRARY@ + @PARMETIS_LIBRARY@ @METIS_LIBRARY@ ) endif() @@ -216,16 +216,16 @@ function(gridpack_setup) list(APPEND gp_libs @GLPK_LIBRARY@) endif() - list(APPEND gp_include + list(APPEND gp_include @Boost_INCLUDE_DIR@ @MPI_INCLUDE_PATH@ ) - + list(APPEND gp_libs @Boost_LIBRARIES@ @MPI_CXX_LIBRARIES@ ) - + set(GRIDPACK_INCLUDE_DIRS ${gp_include} CACHE STRING "Include directories for GridPACK" From 41bc8cc506e8360fdaf7e89a6d20c34e179803fb Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 29 Dec 2023 12:09:04 -0600 Subject: [PATCH 069/195] Remove attempted variable substitution --- .gitlab-ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3e38ecb7..c48156fb1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,12 +16,15 @@ variables: parallel: matrix: - - BASE_IMAGE: rockylinux:9 - - BASE_IMAGE: ubuntu:22.04 + + - BASE_IMAGE: rockylinux + BASE_IMAGE_TAG: "9" + - BASE_IMAGE: ubuntu + BASE_IMAGE_TAG: "22.04" variables: - IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE%%:*}-${CI_COMMIT_REF_NAME}" + IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-${CI_COMMIT_REF_NAME}" # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy @@ -40,7 +43,7 @@ build-container: - /kaniko/executor --context "${CI_PROJECT_DIR}" - --build-arg BASE_IMAGE=${BASE_IMAGE} + --build-arg BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG} --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --dockerfile "${CI_PROJECT_DIR}/dockerfile" From c985f451d1d90cf3c47f2d1f49c72a89fa35ad4d Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 2 Jan 2024 09:37:39 -0600 Subject: [PATCH 070/195] Set conditions for when container is built --- .gitlab-ci.yml | 60 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c48156fb1..a155f2640 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,30 @@ variables: variables: - IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-${CI_COMMIT_REF_NAME}" + IMAGE_TAG: ${BASE_IMAGE}-gridpack-env + IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} + + +check-container-definition-changed: + + extends: .multi-distro + + image: alpine + + script: + + - touch container-definition-changed + + artifacts: + + paths: + + - container-definition-changed + + rules: + + - if: $CI_PIPELINE_SOURCE == "push" + changes: [ "dockerfile" ] # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy @@ -39,22 +62,34 @@ build-container: name: gcr.io/kaniko-project/executor:v1.9.0-debug entrypoint: [""] + needs: + + - job: check-container-definition-changed + optional: true + script: + # build image if definition was changed or the tag does not exist in the registry + - | + tag_exists() { + curl \ + --silent \ + --show-error \ + --location \ + --fail \ + --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " \ + "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG\" \ + 1>$null 2>$null + } + - test -f container-definition-changed || ! tag_exists || exit 0 - /kaniko/executor --context "${CI_PROJECT_DIR}" - --build-arg BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG} - --build-arg http_proxy=$http_proxy - --build-arg https_proxy=$https_proxy + --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" + --build-arg "http_proxy=$http_proxy" + --build-arg "https_proxy=$https_proxy" --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" - rules: - - - if: $CI_PIPELINE_SOURCE == "push" - changes: [ "dockerfile" ] - - if: $CI_PIPELINE_SOURCE == "tag" - build-gridpack: @@ -66,10 +101,7 @@ build-gridpack: MAKE_JOBS: "2" - needs: - - - job: build-container - optional: true + needs: [ build-container ] script: From be884de76c4eaf6d1499e3e0e278283097f9031d Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 2 Jan 2024 09:42:02 -0600 Subject: [PATCH 071/195] Fix quoting --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a155f2640..bdd62ca71 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -78,7 +78,7 @@ build-container: --location \ --fail \ --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " \ - "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG\" \ + "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG" \ 1>$null 2>$null } - test -f container-definition-changed || ! tag_exists || exit 0 From 46969d6c2cde4c3ca0428efccd3972cc675f7263 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 2 Jan 2024 09:53:22 -0600 Subject: [PATCH 072/195] Make script available to setup env --- dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dockerfile b/dockerfile index b40b5f5f6..24d999ec8 100644 --- a/dockerfile +++ b/dockerfile @@ -1,8 +1,6 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -RUN install_environment_packages.sh - ENV GP_EXT_DEPS=/gridpack-dependencies ENV BOOST_VERSION "1.78.0" ENV GA_VERSION "5.8" @@ -13,4 +11,4 @@ WORKDIR ${GP_EXT_DEPS} COPY *.sh . -RUN ./install_gridpack_deps.sh && rm *.sh +RUN ./install_environment_packages.sh && ./install_gridpack_deps.sh && rm *.sh From b83499ed8842d141f035022538ac456037ad8795 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 2 Jan 2024 10:19:09 -0600 Subject: [PATCH 073/195] Make scripts executable for docker image build --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bdd62ca71..82944c5ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,8 +30,6 @@ variables: check-container-definition-changed: - extends: .multi-distro - image: alpine script: @@ -82,6 +80,7 @@ build-container: 1>$null 2>$null } - test -f container-definition-changed || ! tag_exists || exit 0 + - chmod +x *.sh - /kaniko/executor --context "${CI_PROJECT_DIR}" --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" From d7bf55758fc6d4a7a44a60850826fe0d0f8d760f Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 2 Jan 2024 10:57:23 -0600 Subject: [PATCH 074/195] Save logs from install_gridpack_deps --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 82944c5ae..7b001c15c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -89,6 +89,10 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" + artifacts: + + paths: [ log/ ] + build-gridpack: From 71099a31e185fe368c642bc7528817fcf04752fa Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 2 Jan 2024 14:53:13 -0600 Subject: [PATCH 075/195] Fix artifact path for logs --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7b001c15c..296bc6e2d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -91,7 +91,7 @@ build-container: artifacts: - paths: [ log/ ] + paths: [ /gridpack-dependencies/log/ ] build-gridpack: From 401494ae7c9f320e75fa38aabbd5b8822d2cd148 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 2 Jan 2024 16:21:45 -0600 Subject: [PATCH 076/195] Test - remove parallel config --- .gitlab-ci.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 296bc6e2d..6523c7562 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,17 +13,10 @@ variables: .multi-distro: - parallel: - - matrix: - - - BASE_IMAGE: rockylinux - BASE_IMAGE_TAG: "9" - - BASE_IMAGE: ubuntu - BASE_IMAGE_TAG: "22.04" - variables: + BASE_IMAGE: ubuntu + BASE_IMAGE_TAG: "22.04" IMAGE_TAG: ${BASE_IMAGE}-gridpack-env IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} From 6bd83368814515049773b59fff2e5aa508e8f346 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 3 Jan 2024 09:59:36 -0600 Subject: [PATCH 077/195] Link logs to project folder - artifacts can only be saved from there per gitlab docs --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6523c7562..2c20a8edd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,6 +60,7 @@ build-container: script: + - ln -s /gridpack-dependencies/log/ ${CI_PROJECT_DIR}/log # build image if definition was changed or the tag does not exist in the registry - | tag_exists() { @@ -84,7 +85,7 @@ build-container: artifacts: - paths: [ /gridpack-dependencies/log/ ] + paths: [ log/ ] build-gridpack: From f9a76ec86a30c2b9552dcb9148d29bcff9d9e750 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 3 Jan 2024 10:34:02 -0600 Subject: [PATCH 078/195] =?UTF-8?q?Send=20logs=20to=20back=20console=20?= =?UTF-8?q?=F0=9F=98=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 5 ----- install_gridpack_deps.sh | 21 +++++++++------------ 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c20a8edd..e58236720 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,7 +60,6 @@ build-container: script: - - ln -s /gridpack-dependencies/log/ ${CI_PROJECT_DIR}/log # build image if definition was changed or the tag does not exist in the registry - | tag_exists() { @@ -83,10 +82,6 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" - artifacts: - - paths: [ log/ ] - build-gridpack: diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 98a6bd29e..a9a1950c4 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -35,17 +35,16 @@ function install_boost { echo "Bootstrapping Boost" ./bootstrap.sh \ --prefix=install_for_gridpack \ - --with-libraries=mpi,serialization,random,filesystem,system \ - >../log/boost_bootstrap.log 2>&1 + --with-libraries=mpi,serialization,random,filesystem,system echo 'using mpi ;' >>project-config.jam # build echo "Building Boost" - ./b2 -a -d+2 link=shared stage >../log/boost_build.log 2>&1 + ./b2 -a -d+2 link=shared stage # install echo "Installing Boost" - ./b2 -a -d+2 link=shared install >../log/boost_install.log 2>&1 + ./b2 -a -d+2 link=shared install popd || exit @@ -85,12 +84,11 @@ function install_ga { --enable-cxx \ --enable-i4 \ --prefix="${PWD}/install_for_gridpack" \ - --enable-shared \ - >../log/ga_configure.log 2>&1 + --enable-shared # install echo "Installing Global Arrays" - make -j "$(nproc)" install >../log/ga_install.log 2>&1 + make -j "$(nproc)" install popd || exit @@ -128,20 +126,19 @@ function install_petsc { --prefix="${PWD}"/install_for_gridpack \ --scalar-type=complex \ --with-shared-libraries=1 \ - --download-f2cblaslapack=1 \ - >../log/petsc_configure.log 2>&1 + --download-f2cblaslapack=1 # build echo "Building PETSc" - make >../log/petsc_build.log 2>&1 + make # install echo "Installing PETSc" - make install >../log/petsc_install.log 2>&1 + make install # check echo "Checking PETSc" - make check >../log/petsc_check.log 2>&1 + make check popd || exit From 7d842a30bc5dcc48d3bb2cf276b9128585000f1d Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 3 Jan 2024 13:23:51 -0600 Subject: [PATCH 079/195] Redirect output to files --- .gitlab-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e58236720..2c82b27be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -80,7 +80,11 @@ build-container: --build-arg "http_proxy=$http_proxy" --build-arg "https_proxy=$https_proxy" --dockerfile "${CI_PROJECT_DIR}/dockerfile" - --destination "${IMAGE_PATH}" + --destination "${IMAGE_PATH}" 2> build_stderr.log | tee build_stdout.log + + artifacts: + + paths: [ build_stderr.log, build_stdout.log ] build-gridpack: From 2910f3bb80476a14ec3536e075f19a2d2eab0962 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 3 Jan 2024 13:25:40 -0600 Subject: [PATCH 080/195] Build PETSc with MUMPS instead of SuperLU_dist --- install_gridpack_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index a9a1950c4..46f4e9970 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -117,7 +117,7 @@ function install_petsc { # install echo "Configuring PETSc" ./configure \ - --download-superlu_dist \ + --download-mumps \ --download-metis \ --download-parmetis \ --download-suitesparse \ From 186b938b7fe73dddfbe7d945d1c4ca0f58b5d677 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 3 Jan 2024 14:23:25 -0600 Subject: [PATCH 081/195] Redirect both streams from kaniko exe to file Still not getting an artifact in gitlab... not sure if it's bc I'm overflowing the console --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c82b27be..70a7ba9b5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -80,11 +80,11 @@ build-container: --build-arg "http_proxy=$http_proxy" --build-arg "https_proxy=$https_proxy" --dockerfile "${CI_PROJECT_DIR}/dockerfile" - --destination "${IMAGE_PATH}" 2> build_stderr.log | tee build_stdout.log + --destination "${IMAGE_PATH}" > build.log 2>&1 artifacts: - paths: [ build_stderr.log, build_stdout.log ] + paths: [ build.log ] build-gridpack: From 65965a59d48dd9ea0e45170ba9678ba295ae9122 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 12:12:07 -0600 Subject: [PATCH 082/195] Move the container registry check to its own job with curl actually installed --- .gitlab-ci.yml | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 70a7ba9b5..512e209cc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,9 +31,7 @@ check-container-definition-changed: artifacts: - paths: - - - container-definition-changed + untracked: true rules: @@ -41,6 +39,28 @@ check-container-definition-changed: changes: [ "dockerfile" ] +check-container-tag-exists: + + image: alpine + + script: + + - apk add --no-cache curl + - > + curl \ + --silent \ + --show-error \ + --location \ + --fail \ + --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " \ + "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG" \ + 1>$null 2>$null || touch container-tag-not-found + + artifacts: + + untracked: true + + # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy build-container: @@ -57,22 +77,12 @@ build-container: - job: check-container-definition-changed optional: true + - job: check-container-tag-exists script: # build image if definition was changed or the tag does not exist in the registry - - | - tag_exists() { - curl \ - --silent \ - --show-error \ - --location \ - --fail \ - --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " \ - "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG" \ - 1>$null 2>$null - } - - test -f container-definition-changed || ! tag_exists || exit 0 + - test -f container-definition-changed || test -f container-tag-not-found || exit 0 - chmod +x *.sh - /kaniko/executor --context "${CI_PROJECT_DIR}" From 6ca911f9fdfc88938314bf9e8c22ae067ace085b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 13:26:02 -0600 Subject: [PATCH 083/195] Use block scalar to avoid issues with special chars --- .gitlab-ci.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 512e209cc..1e80a1f2a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -84,13 +84,14 @@ build-container: # build image if definition was changed or the tag does not exist in the registry - test -f container-definition-changed || test -f container-tag-not-found || exit 0 - chmod +x *.sh - - /kaniko/executor - --context "${CI_PROJECT_DIR}" - --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" - --build-arg "http_proxy=$http_proxy" - --build-arg "https_proxy=$https_proxy" - --dockerfile "${CI_PROJECT_DIR}/dockerfile" - --destination "${IMAGE_PATH}" > build.log 2>&1 + - > + /kaniko/executor + --context "${CI_PROJECT_DIR}" + --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" + --build-arg "http_proxy=$http_proxy" + --build-arg "https_proxy=$https_proxy" + --dockerfile "${CI_PROJECT_DIR}/dockerfile" + --destination "${IMAGE_PATH}" > build.log 2>&1 artifacts: From 25ddd7966c91cf040a9703669dc6ddc116b07f20 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 13:27:05 -0600 Subject: [PATCH 084/195] Always save untracked files --- .gitlab-ci.yml | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e80a1f2a..2c617e2b2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,11 @@ default: tags: [ basic, gridpack, ikp, k8s ] + artifacts: + + when: always + untracked: true + variables: @@ -29,10 +34,6 @@ check-container-definition-changed: - touch container-definition-changed - artifacts: - - untracked: true - rules: - if: $CI_PIPELINE_SOURCE == "push" @@ -56,10 +57,6 @@ check-container-tag-exists: "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG" \ 1>$null 2>$null || touch container-tag-not-found - artifacts: - - untracked: true - # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy build-container: @@ -93,10 +90,6 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 - artifacts: - - paths: [ build.log ] - build-gridpack: @@ -114,10 +107,6 @@ build-gridpack: - ./install_gridpack.sh - artifacts: - - untracked: true - test-gridpack: @@ -132,11 +121,3 @@ test-gridpack: - useradd gridpack - chown -R gridpack:gridpack . - su gridpack -c 'ctest --test-dir src/build --output-on-failure' - - artifacts: - - paths: - - - src/build/Testing/Temporary - - when: always From fe636033ae5692668310cf160b2955a806f2f7c4 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 13:39:54 -0600 Subject: [PATCH 085/195] Fix block scalar Lines with indentation are not folded! https://yaml-multiline.info/ --- .gitlab-ci.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c617e2b2..7d33e9186 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,14 +48,14 @@ check-container-tag-exists: - apk add --no-cache curl - > - curl \ - --silent \ - --show-error \ - --location \ - --fail \ - --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " \ - "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG" \ - 1>$null 2>$null || touch container-tag-not-found + curl + --silent + --show-error + --location + --fail + --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " + "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG" + 1>$null 2>$null || touch container-tag-not-found # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy @@ -83,12 +83,12 @@ build-container: - chmod +x *.sh - > /kaniko/executor - --context "${CI_PROJECT_DIR}" - --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" - --build-arg "http_proxy=$http_proxy" - --build-arg "https_proxy=$https_proxy" - --dockerfile "${CI_PROJECT_DIR}/dockerfile" - --destination "${IMAGE_PATH}" > build.log 2>&1 + --context "${CI_PROJECT_DIR}" + --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" + --build-arg "http_proxy=$http_proxy" + --build-arg "https_proxy=$https_proxy" + --dockerfile "${CI_PROJECT_DIR}/dockerfile" + --destination "${IMAGE_PATH}" > build.log 2>&1 build-gridpack: From 9e2637071f4b875136f309ea5c1eb1f40507c541 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 14:06:23 -0600 Subject: [PATCH 086/195] Swap out mpms for superlu for now PETSc config fails due to options and I need to figure out a way to get the more detailed log from outside the project folder (as a gitlab artifact). --- install_gridpack_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 46f4e9970..a9a1950c4 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -117,7 +117,7 @@ function install_petsc { # install echo "Configuring PETSc" ./configure \ - --download-mumps \ + --download-superlu_dist \ --download-metis \ --download-parmetis \ --download-suitesparse \ From 70deb07bd83a903b3ac30bd03b30ff748f7750cc Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 14:25:29 -0600 Subject: [PATCH 087/195] Pull back on untracked artifacts where it makes sense --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7d33e9186..778fe8728 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -90,6 +90,10 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 + artifacts: + + paths: [ build.log ] + build-gridpack: @@ -121,3 +125,7 @@ test-gridpack: - useradd gridpack - chown -R gridpack:gridpack . - su gridpack -c 'ctest --test-dir src/build --output-on-failure' + + artifacts: + + paths: [ src/build/Testing/Temporary ] From 8c916009316763f42708abedbbd556760a551ad6 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 14:34:03 -0600 Subject: [PATCH 088/195] Revert "Swap out mpms for superlu for now" This reverts commit 9e2637071f4b875136f309ea5c1eb1f40507c541. --- install_gridpack_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index a9a1950c4..46f4e9970 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -117,7 +117,7 @@ function install_petsc { # install echo "Configuring PETSc" ./configure \ - --download-superlu_dist \ + --download-mumps \ --download-metis \ --download-parmetis \ --download-suitesparse \ From 9253abf067157f1ae364e8d11a606917dfc96d9b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 14:35:00 -0600 Subject: [PATCH 089/195] Add scalapack per error message --- install_gridpack_deps.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 46f4e9970..640613561 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -118,6 +118,7 @@ function install_petsc { echo "Configuring PETSc" ./configure \ --download-mumps \ + --download-scalapack \ --download-metis \ --download-parmetis \ --download-suitesparse \ From 37ad180bf21a908914a61b84d23975c7ce12013c Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 15:18:45 -0600 Subject: [PATCH 090/195] Remove the artifact path spec for build container For some reason this wasn't saved again... --- .gitlab-ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 778fe8728..8d558ea7a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -90,10 +90,6 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 - artifacts: - - paths: [ build.log ] - build-gridpack: From 574be1cc49ec51df41882a4bec7bb4c411562b65 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 16:13:46 -0600 Subject: [PATCH 091/195] Pull container repo id into var --- .gitlab-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8d558ea7a..5296a0a25 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,6 +44,10 @@ check-container-tag-exists: image: alpine + variables: + + CONTAINER_REPO_ID: "294" + script: - apk add --no-cache curl @@ -54,7 +58,7 @@ check-container-tag-exists: --location --fail --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " - "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG" + "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$CONTAINER_REPO_ID/tags/$IMAGE_TAG" 1>$null 2>$null || touch container-tag-not-found From cc7ecaec0bf5bd5265be109d5749e1a792f5ccf0 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 16:14:13 -0600 Subject: [PATCH 092/195] Try to see log files I need when build fails --- dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfile b/dockerfile index 24d999ec8..38bc2ff35 100644 --- a/dockerfile +++ b/dockerfile @@ -11,4 +11,4 @@ WORKDIR ${GP_EXT_DEPS} COPY *.sh . -RUN ./install_environment_packages.sh && ./install_gridpack_deps.sh && rm *.sh +RUN (./install_environment_packages.sh && ./install_gridpack_deps.sh) || find ${GP_EXT_DEPS} -type f -regex ".*log.*" && rm *.sh From 1ae11b161f47d9b901c5d735f5b1f943e502df0e Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 9 Jan 2024 18:28:55 -0600 Subject: [PATCH 093/195] Set the proxy var for kaniko --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5296a0a25..89482c8c1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,6 +14,7 @@ variables: KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" + PROXY: "http://proxy01.pnl.gov:3128" .multi-distro: @@ -89,8 +90,8 @@ build-container: /kaniko/executor --context "${CI_PROJECT_DIR}" --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" - --build-arg "http_proxy=$http_proxy" - --build-arg "https_proxy=$https_proxy" + --build-arg "http_proxy=$PROXY" + --build-arg "https_proxy=$PROXY" --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 From 317ddc4c16832171242e71e1307c683414ed7f93 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 9 Jan 2024 19:23:35 -0600 Subject: [PATCH 094/195] Pass proxy to all jobs --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 89482c8c1..b8715374f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,8 @@ variables: KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" - PROXY: "http://proxy01.pnl.gov:3128" + HTTP_PROXY: "http://proxy01.pnl.gov:3128" + HTTPS_PROXY: HTTP_PROXY .multi-distro: @@ -90,8 +91,8 @@ build-container: /kaniko/executor --context "${CI_PROJECT_DIR}" --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" - --build-arg "http_proxy=$PROXY" - --build-arg "https_proxy=$PROXY" + --build-arg "http_proxy=$HTTP_PROXY" + --build-arg "https_proxy=$HTTPS_PROXY" --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 From bb44759a98c7bfbb2f944ed7e995f78f4c540eed Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 9 Jan 2024 21:07:03 -0600 Subject: [PATCH 095/195] Try lowercase --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b8715374f..31b770275 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,8 @@ variables: KUBERNETES_MEMORY_LIMIT: "8Gi" HTTP_PROXY: "http://proxy01.pnl.gov:3128" HTTPS_PROXY: HTTP_PROXY + http_proxy: HTTP_PROXY + https_proxy: HTTP_PROXY .multi-distro: From cae4119e83650255bcc8ca4396e7dbbae17de8e0 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 9 Jan 2024 21:11:02 -0600 Subject: [PATCH 096/195] Mark as variables --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 31b770275..ba4ab8533 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,9 +15,9 @@ variables: KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" HTTP_PROXY: "http://proxy01.pnl.gov:3128" - HTTPS_PROXY: HTTP_PROXY - http_proxy: HTTP_PROXY - https_proxy: HTTP_PROXY + HTTPS_PROXY: ${HTTP_PROXY} + http_proxy: ${HTTP_PROXY} + https_proxy: ${HTTP_PROXY} .multi-distro: From 6a576eee10ad926792d9c81aa79e701a17f96a26 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 19 Jan 2024 16:04:37 -0600 Subject: [PATCH 097/195] Remove petsc config option --- install_gridpack_deps.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 640613561..95e3a451d 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -126,8 +126,7 @@ function install_petsc { --download-cmake \ --prefix="${PWD}"/install_for_gridpack \ --scalar-type=complex \ - --with-shared-libraries=1 \ - --download-f2cblaslapack=1 + --with-shared-libraries=1 # build echo "Building PETSc" From d0113b420864ffd8bedb34d084d0a553d8dd96ba Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sat, 20 Jan 2024 15:51:25 -0600 Subject: [PATCH 098/195] Remove unused log folder --- install_gridpack_deps.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 95e3a451d..f0e0fa310 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -148,8 +148,6 @@ function install_petsc { echo "Installing GridPACK dependencies" date -mkdir -p log - install_boost "${BOOST_VERSION:?}" install_ga "${GA_VERSION:?}" install_petsc "${PETSC_VERSION:?}" From e21f58f193266bb8d5c4d4e7336ed5cb27a5288b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sat, 20 Jan 2024 15:52:25 -0600 Subject: [PATCH 099/195] Try to parallelize the pipeline again Do everything for ubuntu and rockylinux --- .gitlab-ci.yml | 62 ++++++++++++-------------------------------------- 1 file changed, 14 insertions(+), 48 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba4ab8533..853e022d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,48 +22,16 @@ variables: .multi-distro: - variables: - - BASE_IMAGE: ubuntu - BASE_IMAGE_TAG: "22.04" - IMAGE_TAG: ${BASE_IMAGE}-gridpack-env - IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} - - -check-container-definition-changed: - - image: alpine - - script: + parallel: - - touch container-definition-changed + matrix: - rules: - - - if: $CI_PIPELINE_SOURCE == "push" - changes: [ "dockerfile" ] - - -check-container-tag-exists: - - image: alpine - - variables: - - CONTAINER_REPO_ID: "294" - - script: - - - apk add --no-cache curl - - > - curl - --silent - --show-error - --location - --fail - --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " - "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$CONTAINER_REPO_ID/tags/$IMAGE_TAG" - 1>$null 2>$null || touch container-tag-not-found + - BASE_IMAGE: ubuntu + BASE_IMAGE_TAG: "22.04" + IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env + - BASE_IMAGE: rockylinux + BASE_IMAGE_TAG: "9" + IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy @@ -78,16 +46,8 @@ build-container: name: gcr.io/kaniko-project/executor:v1.9.0-debug entrypoint: [""] - needs: - - - job: check-container-definition-changed - optional: true - - job: check-container-tag-exists - script: - # build image if definition was changed or the tag does not exist in the registry - - test -f container-definition-changed || test -f container-tag-not-found || exit 0 - chmod +x *.sh - > /kaniko/executor @@ -98,6 +58,12 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 + rules: + + - if: $CI_PIPELINE_SOURCE == "push" + changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] + - when: manual + build-gridpack: From 2f9ae934c2223193d14c047641e8694c0acfb480 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sat, 20 Jan 2024 15:57:35 -0600 Subject: [PATCH 100/195] Revert troubleshooting command --- dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfile b/dockerfile index 38bc2ff35..ad7a5ee14 100644 --- a/dockerfile +++ b/dockerfile @@ -11,4 +11,4 @@ WORKDIR ${GP_EXT_DEPS} COPY *.sh . -RUN (./install_environment_packages.sh && ./install_gridpack_deps.sh) || find ${GP_EXT_DEPS} -type f -regex ".*log.*" && rm *.sh +RUN (./install_environment_packages.sh && ./install_gridpack_deps.sh) || rm *.sh From 8f1ccb4a433df6a03fb2e3bde68cb3467c2c32a1 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sat, 20 Jan 2024 16:24:48 -0600 Subject: [PATCH 101/195] Separate scripts to improve caching --- dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dockerfile b/dockerfile index ad7a5ee14..2ae8c0cd7 100644 --- a/dockerfile +++ b/dockerfile @@ -9,6 +9,8 @@ ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost/install_for_gridpack/lib:${GP_EXT_DEPS} WORKDIR ${GP_EXT_DEPS} -COPY *.sh . +COPY install_environment_packages.sh . +RUN ./install_environment_packages.sh && rm *.sh -RUN (./install_environment_packages.sh && ./install_gridpack_deps.sh) || rm *.sh +COPY install_gridpack_deps.sh . +RUN ./install_gridpack_deps.sh && rm *.sh From 9af3251b5148bebf4ee53586c85d0828e1fede7a Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sat, 20 Jan 2024 16:34:27 -0600 Subject: [PATCH 102/195] Install mpi4py dependency --- install_gridpack.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install_gridpack.sh b/install_gridpack.sh index 688832d4e..e2a3d44e6 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -91,6 +91,9 @@ function install_gridpack_python { # export GRIDPACK_DIR export GRIDPACK_DIR="${gridpack_install_dir}" + # install dependencies + pip install mpi4py --user + # build echo "Building GridPACK python wrapper" ${python_exe} setup.py build From da312308c9516d2fb450cff7288d583e9af390de Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 22 Jan 2024 12:08:41 -0600 Subject: [PATCH 103/195] Set proxy build args in container build env --- .gitlab-ci.yml | 10 ++++------ dockerfile | 5 +++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 853e022d6..756de6eb4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,10 +14,8 @@ variables: KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" - HTTP_PROXY: "http://proxy01.pnl.gov:3128" - HTTPS_PROXY: ${HTTP_PROXY} - http_proxy: ${HTTP_PROXY} - https_proxy: ${HTTP_PROXY} + http_proxy: "http://proxy01.pnl.gov:3128" + https_proxy: ${http_proxy} .multi-distro: @@ -53,8 +51,8 @@ build-container: /kaniko/executor --context "${CI_PROJECT_DIR}" --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" - --build-arg "http_proxy=$HTTP_PROXY" - --build-arg "https_proxy=$HTTPS_PROXY" + --build-arg "http_proxy=$http_proxy" + --build-arg "https_proxy=$https_proxy" --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 diff --git a/dockerfile b/dockerfile index 2ae8c0cd7..92bb67862 100644 --- a/dockerfile +++ b/dockerfile @@ -7,6 +7,11 @@ ENV GA_VERSION "5.8" ENV PETSC_VERSION "3.16.4" ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost/install_for_gridpack/lib:${GP_EXT_DEPS}/ga/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} +ARG http_proxy +ARG https_proxy +ENV http_proxy=${http_proxy} +ENV https_proxy=${https_proxy} + WORKDIR ${GP_EXT_DEPS} COPY install_environment_packages.sh . From 109ee8972fa11c24fae05ce4b41be8d3e6443a5f Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 22 Jan 2024 12:09:50 -0600 Subject: [PATCH 104/195] Get mpi4py from system repos instead of through pip --- install_environment_packages.sh | 4 ++-- install_gridpack.sh | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index 9109cc8d7..9b06d630e 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -15,12 +15,12 @@ debian | ubuntu) apt update && apt upgrade -y && apt install -y \ - wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config + wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py ;; fedora | rhel | centos | rocky) dnf update -y && dnf install -y \ - wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf + wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf python3-mpi4py-openmpi ;; *) echo "$distribution not supported" diff --git a/install_gridpack.sh b/install_gridpack.sh index e2a3d44e6..688832d4e 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -91,9 +91,6 @@ function install_gridpack_python { # export GRIDPACK_DIR export GRIDPACK_DIR="${gridpack_install_dir}" - # install dependencies - pip install mpi4py --user - # build echo "Building GridPACK python wrapper" ${python_exe} setup.py build From 65cd8639882ea9d16db5b75e62aee3ee684da412 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 22 Jan 2024 12:10:08 -0600 Subject: [PATCH 105/195] Use common check for distro between scripts --- install_gridpack.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 688832d4e..0b0e17140 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -74,13 +74,16 @@ function install_gridpack_python { pushd python || exit # set an env var if we are running on RHEL - os_id=$( + distribution=$( + # shellcheck source=/dev/null source /etc/os-release echo "$ID" ) - if [[ $os_id == "rhel" ]] || [[ $os_id == "centos" ]]; then - export RHEL_OPENMPI_HACK=yes - fi + case $distribution in + fedora | rhel | centos | rocky) + export RHEL_OPENMPI_HACK=yes + ;; + esac # set python executable path python_exe=$(which python || which python3) From 31d1fb274d43a75264f408cc3fe853f9a827cc6e Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 22 Jan 2024 16:47:56 -0600 Subject: [PATCH 106/195] Manually define job parallelization and adjust whitespace --- .gitlab-ci.yml | 79 +++++++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 756de6eb4..6074c8211 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,15 +1,11 @@ default: - tags: [ basic, gridpack, ikp, k8s ] - artifacts: - when: always untracked: true variables: - KUBERNETES_CPU_REQUEST: "1" KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" @@ -18,34 +14,15 @@ variables: https_proxy: ${http_proxy} -.multi-distro: - - parallel: - - matrix: - - - BASE_IMAGE: ubuntu - BASE_IMAGE_TAG: "22.04" - IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env - - BASE_IMAGE: rockylinux - BASE_IMAGE_TAG: "9" - IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env - - # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy -build-container: - - extends: .multi-distro - +.build-container: timeout: 5 hours - image: - name: gcr.io/kaniko-project/executor:v1.9.0-debug entrypoint: [""] - + variables: + IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env script: - - chmod +x *.sh - > /kaniko/executor @@ -55,45 +32,61 @@ build-container: --build-arg "https_proxy=$https_proxy" --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 - rules: - - if: $CI_PIPELINE_SOURCE == "push" changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] - when: manual -build-gridpack: - - extends: .multi-distro +build-container:ubuntu: + extends: .build-container + variables: + BASE_IMAGE: ubuntu + BASE_IMAGE_TAG: "22.04" - image: "${IMAGE_PATH}" +build-container:rockylinux: + extends: .build-container variables: + BASE_IMAGE: rockylinux + BASE_IMAGE_TAG: "9" - MAKE_JOBS: "2" - - needs: [ build-container ] +.build-gridpack: + variables: + MAKE_JOBS: "2" script: - - ./install_gridpack.sh -test-gridpack: +build-gridpack:ubuntu: + extends: .build-gridpack + image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env + needs: [ build-container:ubuntu ] - extends: .multi-distro - image: ${IMAGE_PATH} +build-gridpack:rockylinux: + extends: .build-gridpack + image: ${CI_REGISTRY_IMAGE}:rockylinux-gridpack-env + needs: [ build-container:rockylinux ] - needs: [ build-gridpack ] +.test-gridpack: script: - - useradd gridpack - chown -R gridpack:gridpack . - su gridpack -c 'ctest --test-dir src/build --output-on-failure' - artifacts: - paths: [ src/build/Testing/Temporary ] + + +test-gridpack:ubuntu: + extends: .test-gridpack + image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env + needs: [ build-gridpack:ubuntu ] + + +test-gridpack:rockylinux: + extends: .test-gridpack + image: ${CI_REGISTRY_IMAGE}:rockylinux-gridpack-env + needs: [ build-gridpack:rockylinux ] From 3fc6c43b6cb0c36e26248e7070763fc4cebd71cb Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 10:23:26 -0600 Subject: [PATCH 107/195] Try adding docker config and cert to kaniko container Also disable the ubuntu jobs for now --- .gitlab-ci.yml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6074c8211..d6226bc93 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,21 @@ variables: variables: IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env script: + - mkdir -p /kaniko/.docker + - | + { + "proxies": { + "default": { + "httpProxy": "${http_proxy}", + "httpsProxy": "${https_proxy}" + } + } + } >> /kaniko/.docker/config.json + + if [ -f /etc/gitlab-runner/certs/ca.crt ]; then + cat /etc/gitlab-runner/certs/ca.crt >> /kaniko/ssl/certs/ca-certificates.crt + fi + - chmod +x *.sh - > /kaniko/executor @@ -38,7 +53,8 @@ variables: - when: manual -build-container:ubuntu: +# disabled for troubleshooting +.build-container:ubuntu: extends: .build-container variables: BASE_IMAGE: ubuntu @@ -59,7 +75,8 @@ build-container:rockylinux: - ./install_gridpack.sh -build-gridpack:ubuntu: +# disabled for troubleshooting +.build-gridpack:ubuntu: extends: .build-gridpack image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env needs: [ build-container:ubuntu ] @@ -80,7 +97,8 @@ build-gridpack:rockylinux: paths: [ src/build/Testing/Temporary ] -test-gridpack:ubuntu: +# disabled for troubleshooting +. test-gridpack:ubuntu: extends: .test-gridpack image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env needs: [ build-gridpack:ubuntu ] From 6eede55a19fdd0191155dc127182945ab73f3220 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 10:24:27 -0600 Subject: [PATCH 108/195] Make the build container job an optional dep --- .gitlab-ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d6226bc93..f0e03fb84 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,13 +79,17 @@ build-container:rockylinux: .build-gridpack:ubuntu: extends: .build-gridpack image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env - needs: [ build-container:ubuntu ] + needs: + - job: build-container:ubuntu + optional: true build-gridpack:rockylinux: extends: .build-gridpack image: ${CI_REGISTRY_IMAGE}:rockylinux-gridpack-env - needs: [ build-container:rockylinux ] + needs: + - job: build-container:rockylinux + optional: true .test-gridpack: From 8464d3142ae0f50b00a04e330a49957581f21322 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 10:33:38 -0600 Subject: [PATCH 109/195] Add verbose flag --- install_environment_packages.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index 9b06d630e..162771989 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -18,8 +18,8 @@ debian | ubuntu) wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py ;; fedora | rhel | centos | rocky) - dnf update -y && - dnf install -y \ + dnf upgrade --assumeyes --verbose && + dnf install --assumeyes \ wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf python3-mpi4py-openmpi ;; *) From c54637c057ec37f9f8b21e93db9b484e6c326d45 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 10:37:07 -0600 Subject: [PATCH 110/195] Remove manual trigger from container build job This always adds the job to the pipeline and blocks - ignoring the intent of it being an optional dependency downstream. I suppose if we want to rebuild the container, you could always go back to the last pipeline where it ran and rerun. This should be the last place that any files relevant to the container build changed. --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f0e03fb84..4d49461ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,6 @@ variables: rules: - if: $CI_PIPELINE_SOURCE == "push" changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] - - when: manual # disabled for troubleshooting From d3222bb0eb706c99bad1c0ae62d975a26b711b9b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 10:40:25 -0600 Subject: [PATCH 111/195] Use a heredoc to populate docker config --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d49461ae..235de802e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,6 +25,7 @@ variables: script: - mkdir -p /kaniko/.docker - | + cat << EOF > /kaniko/.docker/config.json { "proxies": { "default": { @@ -32,7 +33,8 @@ variables: "httpsProxy": "${https_proxy}" } } - } >> /kaniko/.docker/config.json + } + EOF if [ -f /etc/gitlab-runner/certs/ca.crt ]; then cat /etc/gitlab-runner/certs/ca.crt >> /kaniko/ssl/certs/ca-certificates.crt From 98877aeb9520c92956b811b9e3e4b07aeecde951 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 11:06:36 -0600 Subject: [PATCH 112/195] Dump environment variables --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 235de802e..e666a6149 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,6 +41,7 @@ variables: fi - chmod +x *.sh + - env - > /kaniko/executor --context "${CI_PROJECT_DIR}" From 78f5ace1af96014051fb036ff11eaf67a00d0669 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 11:16:11 -0600 Subject: [PATCH 113/195] Disable rules for build container job for now --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e666a6149..9ea2ca4a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,9 +50,10 @@ variables: --build-arg "https_proxy=$https_proxy" --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 - rules: - - if: $CI_PIPELINE_SOURCE == "push" - changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] + # disabled for troubleshooting + # rules: + # - if: $CI_PIPELINE_SOURCE == "push" + # changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] # disabled for troubleshooting From 77809791213182bd36767fed94e4e1611e0f1586 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 12:43:44 -0600 Subject: [PATCH 114/195] Organize job into stages This should make the pipeline summary icons in the GitLab UI more useful. --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9ea2ca4a6..0717d1b2b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,8 @@ default: when: always untracked: true +stages: [ build-container, build-gridpack, test-gridpack ] + variables: KUBERNETES_CPU_REQUEST: "1" @@ -16,6 +18,7 @@ variables: # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy .build-container: + stage: build-container timeout: 5 hours image: name: gcr.io/kaniko-project/executor:v1.9.0-debug @@ -72,6 +75,7 @@ build-container:rockylinux: .build-gridpack: + stage: build-gridpack variables: MAKE_JOBS: "2" script: @@ -96,6 +100,7 @@ build-gridpack:rockylinux: .test-gridpack: + stage: test-gridpack script: - useradd gridpack - chown -R gridpack:gridpack . From 240306afeeb54cd62fe959063359e3c48332206b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 13:50:43 -0600 Subject: [PATCH 115/195] Add epel repo and enable crb repo For python3-mpi4py-openmpi and its deps --- install_environment_packages.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index 162771989..6cc945f05 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -19,8 +19,11 @@ debian | ubuntu) ;; fedora | rhel | centos | rocky) dnf upgrade --assumeyes --verbose && + dnf install epel-release --assumeyes && + crb enable && dnf install --assumeyes \ - wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf python3-mpi4py-openmpi + wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ + python3-mpi4py-openmpi ;; *) echo "$distribution not supported" From 11df6b1d5f7508fa36f968a39663107de1de6df7 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 13:54:29 -0600 Subject: [PATCH 116/195] Remove troubleshooting commands --- .gitlab-ci.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0717d1b2b..f3913b51c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,25 +26,7 @@ variables: variables: IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env script: - - mkdir -p /kaniko/.docker - - | - cat << EOF > /kaniko/.docker/config.json - { - "proxies": { - "default": { - "httpProxy": "${http_proxy}", - "httpsProxy": "${https_proxy}" - } - } - } - EOF - - if [ -f /etc/gitlab-runner/certs/ca.crt ]; then - cat /etc/gitlab-runner/certs/ca.crt >> /kaniko/ssl/certs/ca-certificates.crt - fi - - chmod +x *.sh - - env - > /kaniko/executor --context "${CI_PROJECT_DIR}" From 20051ddd4504aa2e976ea1793fc08cb4f9cec2ea Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 17:50:34 -0600 Subject: [PATCH 117/195] Normalize ENV declarations --- dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dockerfile b/dockerfile index 92bb67862..5ec6dc9f5 100644 --- a/dockerfile +++ b/dockerfile @@ -2,9 +2,9 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} ENV GP_EXT_DEPS=/gridpack-dependencies -ENV BOOST_VERSION "1.78.0" -ENV GA_VERSION "5.8" -ENV PETSC_VERSION "3.16.4" +ENV BOOST_VERSION="1.78.0" +ENV GA_VERSION="5.8" +ENV PETSC_VERSION="3.16.4" ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost/install_for_gridpack/lib:${GP_EXT_DEPS}/ga/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} ARG http_proxy From c580ba2971134aad5c78570ea818052d0e31b49d Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 19:15:02 -0600 Subject: [PATCH 118/195] Formatting changes --- install_environment_packages.sh | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index 6cc945f05..e763a31a6 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -12,18 +12,29 @@ distribution=$( case $distribution in debian | ubuntu) - apt update && - apt upgrade -y && - apt install -y \ - wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py + + apt-get update + apt-get upgrade --yes + + # install required packages + apt-get install --yes \ + wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py ;; + fedora | rhel | centos | rocky) - dnf upgrade --assumeyes --verbose && - dnf install epel-release --assumeyes && - crb enable && - dnf install --assumeyes \ - wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ - python3-mpi4py-openmpi + + dnf upgrade --assumeyes --verbose + + # use epel and crb repos + # https://wiki.rockylinux.org/rocky/repo/#notes-on-epel + dnf install epel-release --assumeyes + crb enable + + # install required packages + dnf install --assumeyes \ + wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ + python3-mpi4py-openmpi + ;; *) echo "$distribution not supported" From 08e019a87b5ab2baf59d935efdc5badaa63c9938 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 19:15:31 -0600 Subject: [PATCH 119/195] Add mpicc to path after openmpi-devel install --- install_environment_packages.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index e763a31a6..ea53fcd9e 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -35,6 +35,9 @@ fedora | rhel | centos | rocky) wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ python3-mpi4py-openmpi + # add mpicc to path (from openmpi-devel) + export PATH=$PATH:/usr/lib64/openmpi/bin + ;; *) echo "$distribution not supported" From a3773421545906998951db66c7b505f8612ea484 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 19:16:00 -0600 Subject: [PATCH 120/195] Reenable ubuntu build --- .gitlab-ci.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f3913b51c..d09c8a2e6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ default: when: always untracked: true + stages: [ build-container, build-gridpack, test-gridpack ] @@ -35,14 +36,12 @@ variables: --build-arg "https_proxy=$https_proxy" --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 - # disabled for troubleshooting - # rules: - # - if: $CI_PIPELINE_SOURCE == "push" - # changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] -# disabled for troubleshooting -.build-container:ubuntu: +build-container:ubuntu: extends: .build-container variables: BASE_IMAGE: ubuntu @@ -64,8 +63,7 @@ build-container:rockylinux: - ./install_gridpack.sh -# disabled for troubleshooting -.build-gridpack:ubuntu: +build-gridpack:ubuntu: extends: .build-gridpack image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env needs: @@ -91,8 +89,7 @@ build-gridpack:rockylinux: paths: [ src/build/Testing/Temporary ] -# disabled for troubleshooting -. test-gridpack:ubuntu: +test-gridpack:ubuntu: extends: .test-gridpack image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env needs: [ build-gridpack:ubuntu ] From b66c87ae56fa42500ae808bb7e1f9647b8334af2 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 21:50:30 -0600 Subject: [PATCH 121/195] Add mpicc to path if on RHEL --- install_gridpack_deps.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index f0e0fa310..37d63427a 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -75,6 +75,19 @@ function install_ga { pushd ga || exit + # check if we are running on RHEL + distribution=$( + # shellcheck source=/dev/null + source /etc/os-release + echo "$ID" + ) + case $distribution in + fedora | rhel | centos | rocky) + # add mpicc to path (from openmpi-devel) + export PATH=$PATH:/usr/lib64/openmpi/bin + ;; + esac + # build echo "Configuring Global Arrays" ./configure \ From c2089f13939317a68c3a185afebaf6f9c7ff67ae Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 21:52:35 -0600 Subject: [PATCH 122/195] Remove path mod for mpicc --- install_environment_packages.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index ea53fcd9e..eb5fa910c 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -34,10 +34,6 @@ fedora | rhel | centos | rocky) dnf install --assumeyes \ wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ python3-mpi4py-openmpi - - # add mpicc to path (from openmpi-devel) - export PATH=$PATH:/usr/lib64/openmpi/bin - ;; *) echo "$distribution not supported" From bdebd000f57e4c723239e328792903f00efeee23 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 24 Jan 2024 09:51:32 -0600 Subject: [PATCH 123/195] Load mpi module for RHEL --- install_gridpack.sh | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 0b0e17140..f95c8fe08 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -13,7 +13,8 @@ function install_gridpack { local gridpack_deps_dir=$1 local gridpack_build_dir=$2 local gridpack_install_dir=$3 - : "${gridpack_deps_dir:?}" "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" + local distribution=$4 + : "${gridpack_deps_dir:?}" "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" "${distribution:?}" # remove existing build and install dir rm -rf "$gridpack_build_dir" @@ -29,6 +30,15 @@ function install_gridpack { # remove existing cmake output rm -rf CMake* + + # load module related to mpi4py + case $distribution in + fedora | rhel | centos | rocky) + echo "Loading mpi4py module" + module load "mpi/openmpi-$(arch)" + ;; + esac + # generate make files echo "Generating GridPACK make files" cmake \ @@ -65,7 +75,8 @@ function install_gridpack_python { # args local gridpack_build_dir=$1 local gridpack_install_dir=$2 - : "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" + local distribution=$3 + : "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" "${distribution:?}" # update submodules echo "Updating GridPACK submodules" @@ -74,11 +85,6 @@ function install_gridpack_python { pushd python || exit # set an env var if we are running on RHEL - distribution=$( - # shellcheck source=/dev/null - source /etc/os-release - echo "$ID" - ) case $distribution in fedora | rhel | centos | rocky) export RHEL_OPENMPI_HACK=yes @@ -121,7 +127,13 @@ date build_dir=${PWD}/src/build install_dir=${PWD}/src/install -install_gridpack "${GP_EXT_DEPS:?}" "$build_dir" "$install_dir" -install_gridpack_python "$build_dir" "$install_dir" +distribution=$( + # shellcheck source=/dev/null + source /etc/os-release + echo "$ID" +) + +install_gridpack "${GP_EXT_DEPS:?}" "$build_dir" "$install_dir" "$distribution" +install_gridpack_python "$build_dir" "$install_dir" "$distribution" echo "Completed GridPACK installation" From 96214c0945007647474690a33d531ae2be847da5 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 24 Jan 2024 09:51:52 -0600 Subject: [PATCH 124/195] Load mpi module instead of altering PATH --- install_gridpack_deps.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 37d63427a..75740acd8 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -82,10 +82,10 @@ function install_ga { echo "$ID" ) case $distribution in - fedora | rhel | centos | rocky) - # add mpicc to path (from openmpi-devel) - export PATH=$PATH:/usr/lib64/openmpi/bin - ;; + fedora | rhel | centos | rocky) + echo "Loading mpi4py module" + module load "mpi/openmpi-$(arch)" + ;; esac # build From d85e3f156897eab58fa33b5a764cefdb44739535 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 24 Jan 2024 10:52:24 -0600 Subject: [PATCH 125/195] Add boost-openmpi package to rocky linux --- install_environment_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index eb5fa910c..211d962c4 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -33,7 +33,7 @@ fedora | rhel | centos | rocky) # install required packages dnf install --assumeyes \ wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ - python3-mpi4py-openmpi + python3-mpi4py-openmpi boost-openmpi ;; *) echo "$distribution not supported" From 073a9b23587e677734e5733ddb0fec722a42fadb Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 24 Jan 2024 14:59:56 -0600 Subject: [PATCH 126/195] Remove rocky linux jobs for now --- .gitlab-ci.yml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d09c8a2e6..96dc9b244 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,13 +48,6 @@ build-container:ubuntu: BASE_IMAGE_TAG: "22.04" -build-container:rockylinux: - extends: .build-container - variables: - BASE_IMAGE: rockylinux - BASE_IMAGE_TAG: "9" - - .build-gridpack: stage: build-gridpack variables: @@ -71,14 +64,6 @@ build-gridpack:ubuntu: optional: true -build-gridpack:rockylinux: - extends: .build-gridpack - image: ${CI_REGISTRY_IMAGE}:rockylinux-gridpack-env - needs: - - job: build-container:rockylinux - optional: true - - .test-gridpack: stage: test-gridpack script: @@ -93,9 +78,3 @@ test-gridpack:ubuntu: extends: .test-gridpack image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env needs: [ build-gridpack:ubuntu ] - - -test-gridpack:rockylinux: - extends: .test-gridpack - image: ${CI_REGISTRY_IMAGE}:rockylinux-gridpack-env - needs: [ build-gridpack:rockylinux ] From c566c0d5349c101b32864f923e0b631b3d4c664c Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 24 Jan 2024 15:04:00 -0600 Subject: [PATCH 127/195] Bump up timeout --- install_gridpack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index f95c8fe08..defd03494 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -51,7 +51,7 @@ function install_gridpack { -D MPI_CXX_COMPILER:STRING='mpicxx' \ -D MPI_C_COMPILER:STRING='mpicc' \ -D MPIEXEC:STRING='mpiexec' \ - -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ + -D GRIDPACK_TEST_TIMEOUT:STRING=60 \ -D CMAKE_INSTALL_PREFIX:PATH="${gridpack_install_dir}" \ -D CMAKE_BUILD_TYPE:STRING=Debug \ -D BUILD_SHARED_LIBS=YES \ From 2e13f34bc4f59e9acf29763a2edb15f21b52eb36 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 24 Jan 2024 17:08:30 -0600 Subject: [PATCH 128/195] Bump the test timeout again --- install_gridpack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index defd03494..a993cb71d 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -51,7 +51,7 @@ function install_gridpack { -D MPI_CXX_COMPILER:STRING='mpicxx' \ -D MPI_C_COMPILER:STRING='mpicc' \ -D MPIEXEC:STRING='mpiexec' \ - -D GRIDPACK_TEST_TIMEOUT:STRING=60 \ + -D GRIDPACK_TEST_TIMEOUT:STRING=120 \ -D CMAKE_INSTALL_PREFIX:PATH="${gridpack_install_dir}" \ -D CMAKE_BUILD_TYPE:STRING=Debug \ -D BUILD_SHARED_LIBS=YES \ From 90d222d7757adac59e32d83ca7f56fe02230db12 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Thu, 25 Jan 2024 17:24:25 -0600 Subject: [PATCH 129/195] Remove cmake download flag We're installing from source --- install_gridpack_deps.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 75740acd8..51767ce87 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -136,7 +136,6 @@ function install_petsc { --download-parmetis \ --download-suitesparse \ --download-f2cblaslapack \ - --download-cmake \ --prefix="${PWD}"/install_for_gridpack \ --scalar-type=complex \ --with-shared-libraries=1 From 73ec77adde19fc2b6db667845cfdf3408f9e90c8 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 30 Jan 2024 08:49:46 -0600 Subject: [PATCH 130/195] Remove boost-openmpi package --- install_environment_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index 211d962c4..eb5fa910c 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -33,7 +33,7 @@ fedora | rhel | centos | rocky) # install required packages dnf install --assumeyes \ wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ - python3-mpi4py-openmpi boost-openmpi + python3-mpi4py-openmpi ;; *) echo "$distribution not supported" From f8f9dd426d55bba75577d9d01b9ad59d7225ec6b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:08:04 -0600 Subject: [PATCH 131/195] Remove mpi module load from this script --- install_gridpack.sh | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index a993cb71d..1ad7ec4f9 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -13,8 +13,7 @@ function install_gridpack { local gridpack_deps_dir=$1 local gridpack_build_dir=$2 local gridpack_install_dir=$3 - local distribution=$4 - : "${gridpack_deps_dir:?}" "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" "${distribution:?}" + : "${gridpack_deps_dir:?}" "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" # remove existing build and install dir rm -rf "$gridpack_build_dir" @@ -30,15 +29,6 @@ function install_gridpack { # remove existing cmake output rm -rf CMake* - - # load module related to mpi4py - case $distribution in - fedora | rhel | centos | rocky) - echo "Loading mpi4py module" - module load "mpi/openmpi-$(arch)" - ;; - esac - # generate make files echo "Generating GridPACK make files" cmake \ @@ -75,8 +65,7 @@ function install_gridpack_python { # args local gridpack_build_dir=$1 local gridpack_install_dir=$2 - local distribution=$3 - : "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" "${distribution:?}" + : "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" # update submodules echo "Updating GridPACK submodules" @@ -84,6 +73,12 @@ function install_gridpack_python { pushd python || exit + # detect distribution + distribution=$( + source /etc/os-release + echo "$ID" + ) + # set an env var if we are running on RHEL case $distribution in fedora | rhel | centos | rocky) @@ -127,13 +122,7 @@ date build_dir=${PWD}/src/build install_dir=${PWD}/src/install -distribution=$( - # shellcheck source=/dev/null - source /etc/os-release - echo "$ID" -) - -install_gridpack "${GP_EXT_DEPS:?}" "$build_dir" "$install_dir" "$distribution" -install_gridpack_python "$build_dir" "$install_dir" "$distribution" +install_gridpack "${GP_EXT_DEPS:?}" "$build_dir" "$install_dir" +install_gridpack_python "$build_dir" "$install_dir" echo "Completed GridPACK installation" From 60929fad760b25cd7facef481c668a9d15f88a12 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:08:40 -0600 Subject: [PATCH 132/195] More specifically remove scripts --- dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerfile b/dockerfile index 5ec6dc9f5..b2a790ea2 100644 --- a/dockerfile +++ b/dockerfile @@ -15,7 +15,7 @@ ENV https_proxy=${https_proxy} WORKDIR ${GP_EXT_DEPS} COPY install_environment_packages.sh . -RUN ./install_environment_packages.sh && rm *.sh +RUN ./install_environment_packages.sh && rm ./install_environment_packages.sh COPY install_gridpack_deps.sh . -RUN ./install_gridpack_deps.sh && rm *.sh +RUN ./install_gridpack_deps.sh && rm ./install_gridpack_deps.sh From 37354aa83b7525fb770c8aa00174edc2acb183a6 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:09:20 -0600 Subject: [PATCH 133/195] Formatting --- install_environment_packages.sh | 64 ++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index eb5fa910c..839fa58e2 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -5,38 +5,44 @@ set -xeuo pipefail distribution=$( - # shellcheck source=/dev/null source /etc/os-release echo "$ID" ) case $distribution in -debian | ubuntu) - - apt-get update - apt-get upgrade --yes - - # install required packages - apt-get install --yes \ - wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py - ;; - -fedora | rhel | centos | rocky) - - dnf upgrade --assumeyes --verbose - - # use epel and crb repos - # https://wiki.rockylinux.org/rocky/repo/#notes-on-epel - dnf install epel-release --assumeyes - crb enable - - # install required packages - dnf install --assumeyes \ - wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ - python3-mpi4py-openmpi - ;; -*) - echo "$distribution not supported" - exit 1 - ;; + + debian | ubuntu) + + apt-get update + apt-get upgrade --yes + + # install required packages + apt-get install --yes \ + wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py + ;; + + fedora | rhel | centos | rocky) + + dnf upgrade --assumeyes --verbose + + # use epel and crb repos + # https://wiki.rockylinux.org/rocky/repo/#notes-on-epel + dnf install epel-release --assumeyes + crb enable + + # install required packages + dnf install --assumeyes \ + wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ + python3-mpi4py-openmpi + + # load the mpi module + source /etc/profile.d/modules.sh + module load "mpi/openmpi-$(arch)" + ;; + + *) + echo "$distribution not supported" + exit 1 + ;; + esac From 1eac86f8e878e0343129c831c923c7352f197479 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:12:37 -0600 Subject: [PATCH 134/195] Force remove tar file Don't confirm ever --- install_gridpack_deps.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 51767ce87..2a2e4f8b2 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -24,7 +24,7 @@ function install_boost { # unpack echo "Unpacking Boost" - tar -xf boost.tar.gz && rm boost.tar.gz + tar -xf boost.tar.gz && rm -f boost.tar.gz # remove version from dir name mv "boost_${boost_version//./_}" boost @@ -68,7 +68,7 @@ function install_ga { # unpack echo "Unpacking Global Arrays" - tar -xf ga.tar.gz && rm ga.tar.gz + tar -xf ga.tar.gz && rm -f ga.tar.gz # remove version from dir name mv "ga-${ga_version}" ga From 8dd6945aaf8a2d3d3efeb315cf2d63138345444d Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:13:38 -0600 Subject: [PATCH 135/195] Use MAKE_JOBS for global array install too --- install_gridpack_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 2a2e4f8b2..d58667539 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -101,7 +101,7 @@ function install_ga { # install echo "Installing Global Arrays" - make -j "$(nproc)" install + make -j "${MAKE_JOBS:-$(nproc)}" install popd || exit From 10535abeccf88e720ff30df8f39822831f695528 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:14:17 -0600 Subject: [PATCH 136/195] Remove mpi module load from install_gridpack_deps --- install_gridpack_deps.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index d58667539..c69d10770 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -75,19 +75,6 @@ function install_ga { pushd ga || exit - # check if we are running on RHEL - distribution=$( - # shellcheck source=/dev/null - source /etc/os-release - echo "$ID" - ) - case $distribution in - fedora | rhel | centos | rocky) - echo "Loading mpi4py module" - module load "mpi/openmpi-$(arch)" - ;; - esac - # build echo "Configuring Global Arrays" ./configure \ From 7e408d3b9f1fe5f8d0c3c42da344a93033715d3e Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:14:30 -0600 Subject: [PATCH 137/195] Add script to load mpi module on RHEL --- load_mpi_module.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 load_mpi_module.sh diff --git a/load_mpi_module.sh b/load_mpi_module.sh new file mode 100644 index 000000000..843901462 --- /dev/null +++ b/load_mpi_module.sh @@ -0,0 +1,16 @@ +#! /bin/bash + +# source this script to load the mpi module on RHEL distributions + +distribution=$( + source /etc/os-release + echo "$ID" +) + +case $distribution in + fedora | rhel | centos | rocky) + echo "Loading mpi module" + source /etc/profile.d/modules.sh + module load "mpi/openmpi-$(arch)" + ;; +esac From a4414e38aaad7445155f75260ee210d312083e76 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:36:48 -0600 Subject: [PATCH 138/195] Load the mpi module via script before bootstrapping boost --- install_gridpack_deps.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index c69d10770..a0d2203bb 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -31,6 +31,9 @@ function install_boost { pushd boost || exit + # load the mpi module + source load_mpi_module.sh + # bootstrap echo "Bootstrapping Boost" ./bootstrap.sh \ From 3afbf681f306bf86cccb33b1b49d2971adb5e183 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:40:57 -0600 Subject: [PATCH 139/195] Group scripts and dockerfile under build dir --- .gitlab-ci.yml | 4 ++-- dockerfile => build/dockerfile | 0 .../install_environment_packages.sh | 0 install_gridpack.sh => build/install_gridpack.sh | 0 install_gridpack_deps.sh => build/install_gridpack_deps.sh | 0 load_mpi_module.sh => build/load_mpi_module.sh | 0 6 files changed, 2 insertions(+), 2 deletions(-) rename dockerfile => build/dockerfile (100%) rename install_environment_packages.sh => build/install_environment_packages.sh (100%) rename install_gridpack.sh => build/install_gridpack.sh (100%) mode change 100755 => 100644 rename install_gridpack_deps.sh => build/install_gridpack_deps.sh (100%) mode change 100755 => 100644 rename load_mpi_module.sh => build/load_mpi_module.sh (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 96dc9b244..aca3ac6d3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,7 +38,7 @@ variables: --destination "${IMAGE_PATH}" > build.log 2>&1 rules: - if: $CI_PIPELINE_SOURCE == "push" - changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] + changes: [ "build/**/*" ] build-container:ubuntu: @@ -53,7 +53,7 @@ build-container:ubuntu: variables: MAKE_JOBS: "2" script: - - ./install_gridpack.sh + - ./build/install_gridpack.sh build-gridpack:ubuntu: diff --git a/dockerfile b/build/dockerfile similarity index 100% rename from dockerfile rename to build/dockerfile diff --git a/install_environment_packages.sh b/build/install_environment_packages.sh similarity index 100% rename from install_environment_packages.sh rename to build/install_environment_packages.sh diff --git a/install_gridpack.sh b/build/install_gridpack.sh old mode 100755 new mode 100644 similarity index 100% rename from install_gridpack.sh rename to build/install_gridpack.sh diff --git a/install_gridpack_deps.sh b/build/install_gridpack_deps.sh old mode 100755 new mode 100644 similarity index 100% rename from install_gridpack_deps.sh rename to build/install_gridpack_deps.sh diff --git a/load_mpi_module.sh b/build/load_mpi_module.sh similarity index 100% rename from load_mpi_module.sh rename to build/load_mpi_module.sh From 894235c759a960b721e890b511b1a53d9f86b75e Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:41:53 -0600 Subject: [PATCH 140/195] Move MAKE_JOBS var to global scope --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aca3ac6d3..91dcc6d4b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,7 @@ variables: KUBERNETES_MEMORY_LIMIT: "8Gi" http_proxy: "http://proxy01.pnl.gov:3128" https_proxy: ${http_proxy} + MAKE_JOBS: "2" # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy @@ -50,8 +51,6 @@ build-container:ubuntu: .build-gridpack: stage: build-gridpack - variables: - MAKE_JOBS: "2" script: - ./build/install_gridpack.sh From f0b7ca4d794748b2ccf14adb4e382fe6da7ffac3 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:47:53 -0600 Subject: [PATCH 141/195] Move some variables to the devops.pnnl.gov GitLab CI/CD vars - https://devops.pnnl.gov/groups/gridpack-code/-/settings/ci_cd - https://devops.pnnl.gov/gridpack-code/GridPACK/-/settings/ci_cd --- .gitlab-ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 91dcc6d4b..43cc4364e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,12 +9,6 @@ stages: [ build-container, build-gridpack, test-gridpack ] variables: - KUBERNETES_CPU_REQUEST: "1" - KUBERNETES_CPU_LIMIT: "2" - KUBERNETES_MEMORY_REQUEST: "4Gi" - KUBERNETES_MEMORY_LIMIT: "8Gi" - http_proxy: "http://proxy01.pnl.gov:3128" - https_proxy: ${http_proxy} MAKE_JOBS: "2" From 548fdf42699e58ed951bf352482d3452173fc946 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:50:24 -0600 Subject: [PATCH 142/195] Point kaniko args to new build dir --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 43cc4364e..e68fa31df 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,11 +25,11 @@ variables: - chmod +x *.sh - > /kaniko/executor - --context "${CI_PROJECT_DIR}" + --context "${CI_PROJECT_DIR}/build" --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" --build-arg "http_proxy=$http_proxy" --build-arg "https_proxy=$https_proxy" - --dockerfile "${CI_PROJECT_DIR}/dockerfile" + --dockerfile "${CI_PROJECT_DIR}/build/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 rules: - if: $CI_PIPELINE_SOURCE == "push" From 5b36074ad7263a7b16598bbfb3581d2b42b3c700 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:51:52 -0600 Subject: [PATCH 143/195] Add open MPI vars to silence warnings --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e68fa31df..bb167beff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,6 +10,8 @@ stages: [ build-container, build-gridpack, test-gridpack ] variables: MAKE_JOBS: "2" + OMPI_ALLOW_RUN_AS_ROOT: "1" + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: "1" # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy From dd58702aa5124aca1c056da31442ae5b79a2a7a3 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:56:35 -0600 Subject: [PATCH 144/195] Script renames --- build/dockerfile | 8 ++++---- ...stall_gridpack_deps.sh => install_from_source_deps.sh} | 0 ...ll_environment_packages.sh => install_package_deps.sh} | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename build/{install_gridpack_deps.sh => install_from_source_deps.sh} (100%) rename build/{install_environment_packages.sh => install_package_deps.sh} (100%) diff --git a/build/dockerfile b/build/dockerfile index b2a790ea2..7991a198b 100644 --- a/build/dockerfile +++ b/build/dockerfile @@ -14,8 +14,8 @@ ENV https_proxy=${https_proxy} WORKDIR ${GP_EXT_DEPS} -COPY install_environment_packages.sh . -RUN ./install_environment_packages.sh && rm ./install_environment_packages.sh +COPY install_package_deps.sh . +RUN ./install_package_deps.sh && rm ./install_package_deps.sh -COPY install_gridpack_deps.sh . -RUN ./install_gridpack_deps.sh && rm ./install_gridpack_deps.sh +COPY install_from_source_deps.sh . +RUN ./install_from_source_deps.sh && rm ./install_from_source_deps.sh diff --git a/build/install_gridpack_deps.sh b/build/install_from_source_deps.sh similarity index 100% rename from build/install_gridpack_deps.sh rename to build/install_from_source_deps.sh diff --git a/build/install_environment_packages.sh b/build/install_package_deps.sh similarity index 100% rename from build/install_environment_packages.sh rename to build/install_package_deps.sh From af832bc0f235acc6a0d86adb554ab4159a09c8f5 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 07:59:14 -0600 Subject: [PATCH 145/195] Load mpi module via script before generating make files --- build/install_gridpack.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/install_gridpack.sh b/build/install_gridpack.sh index 1ad7ec4f9..14b590a30 100644 --- a/build/install_gridpack.sh +++ b/build/install_gridpack.sh @@ -29,6 +29,9 @@ function install_gridpack { # remove existing cmake output rm -rf CMake* + # load mpi module + source load_mpi_module.sh + # generate make files echo "Generating GridPACK make files" cmake \ From ecd4c7c83a2a275b4aaa378a608159368bd16eda Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 08:04:05 -0600 Subject: [PATCH 146/195] Update paths for kaniko args --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bb167beff..f87395661 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,14 +24,14 @@ variables: variables: IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env script: - - chmod +x *.sh + - chmod +x build/*.sh - > /kaniko/executor - --context "${CI_PROJECT_DIR}/build" + --context build --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" --build-arg "http_proxy=$http_proxy" --build-arg "https_proxy=$https_proxy" - --dockerfile "${CI_PROJECT_DIR}/build/dockerfile" + --dockerfile build/dockerfile --destination "${IMAGE_PATH}" > build.log 2>&1 rules: - if: $CI_PIPELINE_SOURCE == "push" From 4728088271b7cae1076ca97d7f2ac09c9090799f Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 08:09:37 -0600 Subject: [PATCH 147/195] Test --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f87395661..29436bbc8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,6 +48,8 @@ build-container:ubuntu: .build-gridpack: stage: build-gridpack script: + - ls + - chmod +x build/install_gridpack.sh - ./build/install_gridpack.sh From 061f1212b7db62e786c7b2e9f2f6c2b2ffa49493 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 08:29:13 -0600 Subject: [PATCH 148/195] Rework module load --- build/dockerfile | 10 +++++----- build/install_from_source_deps.sh | 3 --- build/install_gridpack.sh | 3 --- build/install_package_deps.sh | 4 ---- 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/build/dockerfile b/build/dockerfile index 7991a198b..1a8dc8ba5 100644 --- a/build/dockerfile +++ b/build/dockerfile @@ -14,8 +14,8 @@ ENV https_proxy=${https_proxy} WORKDIR ${GP_EXT_DEPS} -COPY install_package_deps.sh . -RUN ./install_package_deps.sh && rm ./install_package_deps.sh - -COPY install_from_source_deps.sh . -RUN ./install_from_source_deps.sh && rm ./install_from_source_deps.sh +COPY load_mpi_module.sh install_package_deps.sh install_from_source_deps.sh ./ +RUN source load_mpi_module.sh \ + && ./install_package_deps.sh \ + && ./install_from_source_deps.sh \ + && rm *.sh diff --git a/build/install_from_source_deps.sh b/build/install_from_source_deps.sh index a0d2203bb..c69d10770 100644 --- a/build/install_from_source_deps.sh +++ b/build/install_from_source_deps.sh @@ -31,9 +31,6 @@ function install_boost { pushd boost || exit - # load the mpi module - source load_mpi_module.sh - # bootstrap echo "Bootstrapping Boost" ./bootstrap.sh \ diff --git a/build/install_gridpack.sh b/build/install_gridpack.sh index 14b590a30..1ad7ec4f9 100644 --- a/build/install_gridpack.sh +++ b/build/install_gridpack.sh @@ -29,9 +29,6 @@ function install_gridpack { # remove existing cmake output rm -rf CMake* - # load mpi module - source load_mpi_module.sh - # generate make files echo "Generating GridPACK make files" cmake \ diff --git a/build/install_package_deps.sh b/build/install_package_deps.sh index 839fa58e2..eb2e5f9fb 100644 --- a/build/install_package_deps.sh +++ b/build/install_package_deps.sh @@ -34,10 +34,6 @@ case $distribution in dnf install --assumeyes \ wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ python3-mpi4py-openmpi - - # load the mpi module - source /etc/profile.d/modules.sh - module load "mpi/openmpi-$(arch)" ;; *) From f5e1ce8dc7d7637b1755e5e3584636c284b6cf78 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 08:35:58 -0600 Subject: [PATCH 149/195] Avoid extra chmod step --- .gitlab-ci.yml | 5 +---- build/dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29436bbc8..5641b86f9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,6 @@ variables: variables: IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env script: - - chmod +x build/*.sh - > /kaniko/executor --context build @@ -48,9 +47,7 @@ build-container:ubuntu: .build-gridpack: stage: build-gridpack script: - - ls - - chmod +x build/install_gridpack.sh - - ./build/install_gridpack.sh + - /bin/bash ./build/install_gridpack.sh build-gridpack:ubuntu: diff --git a/build/dockerfile b/build/dockerfile index 1a8dc8ba5..98f06a97e 100644 --- a/build/dockerfile +++ b/build/dockerfile @@ -16,6 +16,6 @@ WORKDIR ${GP_EXT_DEPS} COPY load_mpi_module.sh install_package_deps.sh install_from_source_deps.sh ./ RUN source load_mpi_module.sh \ - && ./install_package_deps.sh \ - && ./install_from_source_deps.sh \ + && /bin/bash ./install_package_deps.sh \ + && /bin/bash ./install_from_source_deps.sh \ && rm *.sh From b25748a85b4efe3194eba5ef811de8d43b51e631 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 08:43:53 -0600 Subject: [PATCH 150/195] Dot source since using sh not bash --- build/dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/dockerfile b/build/dockerfile index 98f06a97e..11df8c246 100644 --- a/build/dockerfile +++ b/build/dockerfile @@ -15,7 +15,7 @@ ENV https_proxy=${https_proxy} WORKDIR ${GP_EXT_DEPS} COPY load_mpi_module.sh install_package_deps.sh install_from_source_deps.sh ./ -RUN source load_mpi_module.sh \ +RUN . ./load_mpi_module.sh \ && /bin/bash ./install_package_deps.sh \ && /bin/bash ./install_from_source_deps.sh \ && rm *.sh From 64df1297679113dfc7cc44ebbbc5340e110b7141 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 31 Jan 2024 09:36:25 -0600 Subject: [PATCH 151/195] Put open MPI env vars in right spot to suppress warnings --- .gitlab-ci.yml | 2 -- build/dockerfile | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5641b86f9..daa2febef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,8 +10,6 @@ stages: [ build-container, build-gridpack, test-gridpack ] variables: MAKE_JOBS: "2" - OMPI_ALLOW_RUN_AS_ROOT: "1" - OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: "1" # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy diff --git a/build/dockerfile b/build/dockerfile index 11df8c246..68395ec26 100644 --- a/build/dockerfile +++ b/build/dockerfile @@ -6,6 +6,8 @@ ENV BOOST_VERSION="1.78.0" ENV GA_VERSION="5.8" ENV PETSC_VERSION="3.16.4" ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost/install_for_gridpack/lib:${GP_EXT_DEPS}/ga/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} +ENV OMPI_ALLOW_RUN_AS_ROOT="1" +ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM="1" ARG http_proxy ARG https_proxy From 4a645f7d2b96e0ad519d05afb9e04135936a8f19 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 5 Feb 2024 23:08:45 -0600 Subject: [PATCH 152/195] Add back script to check if container tag exists --- .gitlab-ci.yml | 5 +++ build/check_container_tag_exists.sh | 58 +++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 build/check_container_tag_exists.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index daa2febef..58939648c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,11 @@ stages: [ build-container, build-gridpack, test-gridpack ] variables: MAKE_JOBS: "2" +check-container-tag-exists: + image: alpine + script: + - apk add --no-cache curl jq + - /bin/bash ./check_container_tag_exists.sh # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy .build-container: diff --git a/build/check_container_tag_exists.sh b/build/check_container_tag_exists.sh new file mode 100644 index 000000000..8078586f4 --- /dev/null +++ b/build/check_container_tag_exists.sh @@ -0,0 +1,58 @@ +#! /bin/bash + +# creates a file named container-tag-not-found if a tag does not exist in the container registry +# needs env vars CI_JOB_TOKEN, CI_API_V4_URL, CI_PROJECT_ID, IMAGE_TAG, CI_PROJECT_PATH_SLUG +# needs curl and jq + +# bash options: +# - errexit: exit on error +# - nounset: treat unset variables as errors +# - pipefail: treat whole pipeline as errored if any commands within error +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +set -o errexit -o nounset -o pipefail + +check_installed() { + if ! command -v "$1" &>/dev/null; then + echo "please install '$1'". >&2 + exit 1 + fi +} + +check_installed jq +check_installed curl + +# submit a GET request to some endpoint on the project api +function get_container_registry_repos { + # get endpoint or use empty string if not provided + # https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html + local endpoint=${1:-} + + # curl options: + # - no-progress-meter: don't show progress bar but show errors + # - location: follow redirects + # - fail: return 22 on server errors + # - header: send private token for authentication + curl \ + --no-progress-meter \ + --fail \ + --location \ + --header "PRIVATE-TOKEN: ${CI_JOB_TOKEN:?}" \ + "${CI_API_V4_URL:?}/projects/${CI_PROJECT_ID:?}/registry/repositories${endpoint}" +} + +# query the registry repo id +repo_id=$( + # jq query: + # - raw-output: output raw strings instead of json + # - exit-status: return 1 if no results + # - for each elements of the array + # - where the registry repo path is the one we care about + # - select the id prop + # https://jqlang.github.io/jq/manual/ + get_container_registry_repos | jq --raw-output --exit-status ".[] | select(.path == \"${CI_PROJECT_PATH_SLUG}\") | .id" +) + +# using that repo id check for a particular tag +# if command errors, create file named container-tag-not-found +get_container_registry_repos "/${repo_id}/tags/${IMAGE_TAG:?}" > /dev/null || + touch container-tag-not-found From b1b87c761d9236507cb87699ec672a64384a663e Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 5 Feb 2024 23:11:00 -0600 Subject: [PATCH 153/195] Add comment on bash options --- build/install_from_source_deps.sh | 8 +++++++- build/install_gridpack.sh | 8 +++++++- build/install_package_deps.sh | 9 ++++++++- build/load_mpi_module.sh | 8 ++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/build/install_from_source_deps.sh b/build/install_from_source_deps.sh index c69d10770..bad11119d 100644 --- a/build/install_from_source_deps.sh +++ b/build/install_from_source_deps.sh @@ -2,7 +2,13 @@ # installs GridPACK dependencies to the current directory -set -xeuo pipefail +# bash options: +# - xtrace: print each command before executing it +# - errexit: exit on error +# - nounset: treat unset variables as errors +# - pipefail: treat whole pipeline as errored if any commands within error +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +set -o xtrace -o errexit -o nounset -o pipefail function install_boost { local boost_version=$1 diff --git a/build/install_gridpack.sh b/build/install_gridpack.sh index 1ad7ec4f9..d610e6f8d 100644 --- a/build/install_gridpack.sh +++ b/build/install_gridpack.sh @@ -4,7 +4,13 @@ # gridPACK is built in src/build and installed to src/install # run this from the top level GridPACK directory -set -xeuo pipefail +# bash options: +# - xtrace: print each command before executing it +# - errexit: exit on error +# - nounset: treat unset variables as errors +# - pipefail: treat whole pipeline as errored if any commands within error +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +set -o xtrace -o errexit -o nounset -o pipefail function install_gridpack { echo "--- GridPACK ---" diff --git a/build/install_package_deps.sh b/build/install_package_deps.sh index eb2e5f9fb..4d5ea3035 100644 --- a/build/install_package_deps.sh +++ b/build/install_package_deps.sh @@ -2,7 +2,13 @@ # installs necessary packages for a given distribution -set -xeuo pipefail +# bash options: +# - xtrace: print each command before executing it +# - errexit: exit on error +# - nounset: treat unset variables as errors +# - pipefail: treat whole pipeline as errored if any commands within error +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +set -o xtrace -o errexit -o nounset -o pipefail distribution=$( source /etc/os-release @@ -37,6 +43,7 @@ case $distribution in ;; *) + echo "$distribution not supported" exit 1 ;; diff --git a/build/load_mpi_module.sh b/build/load_mpi_module.sh index 843901462..f03c90fac 100644 --- a/build/load_mpi_module.sh +++ b/build/load_mpi_module.sh @@ -2,6 +2,14 @@ # source this script to load the mpi module on RHEL distributions +# bash options: +# - xtrace: print each command before executing it +# - errexit: exit on error +# - nounset: treat unset variables as errors +# - pipefail: treat whole pipeline as errored if any commands within error +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +set -o xtrace -o errexit -o nounset -o pipefail + distribution=$( source /etc/os-release echo "$ID" From 84a4a5f21a71feb42dd5f054624c26e509efdcf0 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 6 Feb 2024 10:13:24 -0600 Subject: [PATCH 154/195] Reorder things --- build/check_container_tag_exists.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/build/check_container_tag_exists.sh b/build/check_container_tag_exists.sh index 8078586f4..a4614334f 100644 --- a/build/check_container_tag_exists.sh +++ b/build/check_container_tag_exists.sh @@ -4,13 +4,17 @@ # needs env vars CI_JOB_TOKEN, CI_API_V4_URL, CI_PROJECT_ID, IMAGE_TAG, CI_PROJECT_PATH_SLUG # needs curl and jq + # bash options: +# - xtrace: print each command before executing it # - errexit: exit on error # - nounset: treat unset variables as errors # - pipefail: treat whole pipeline as errored if any commands within error # https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html -set -o errexit -o nounset -o pipefail +set -o xtrace -o errexit -o nounset -o pipefail + +# check that a command is available check_installed() { if ! command -v "$1" &>/dev/null; then echo "please install '$1'". >&2 @@ -18,8 +22,6 @@ check_installed() { fi } -check_installed jq -check_installed curl # submit a GET request to some endpoint on the project api function get_container_registry_repos { @@ -32,6 +34,7 @@ function get_container_registry_repos { # - location: follow redirects # - fail: return 22 on server errors # - header: send private token for authentication + # https://docs.gitlab.com/ee/api/container_registry.html curl \ --no-progress-meter \ --fail \ @@ -40,6 +43,10 @@ function get_container_registry_repos { "${CI_API_V4_URL:?}/projects/${CI_PROJECT_ID:?}/registry/repositories${endpoint}" } + +check_installed jq +check_installed curl + # query the registry repo id repo_id=$( # jq query: @@ -54,5 +61,5 @@ repo_id=$( # using that repo id check for a particular tag # if command errors, create file named container-tag-not-found -get_container_registry_repos "/${repo_id}/tags/${IMAGE_TAG:?}" > /dev/null || +get_container_registry_repos "/${repo_id}/tags/${IMAGE_TAG:?}" >/dev/null || touch container-tag-not-found From 60b8558b9f43a7195a40fadc1b1af115b1602db3 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 7 Feb 2024 08:24:16 -0600 Subject: [PATCH 155/195] Simplify arg checks --- build/install_from_source_deps.sh | 15 +++------------ build/install_gridpack.sh | 12 +++++------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/build/install_from_source_deps.sh b/build/install_from_source_deps.sh index bad11119d..2c7fd067f 100644 --- a/build/install_from_source_deps.sh +++ b/build/install_from_source_deps.sh @@ -11,10 +11,7 @@ set -o xtrace -o errexit -o nounset -o pipefail function install_boost { - local boost_version=$1 - - # check args - : "${boost_version:?}" + local boost_version=${1:?} echo "--- Installing Boost ${boost_version} ---" @@ -58,10 +55,7 @@ function install_boost { } function install_ga { - local ga_version=$1 - - # check args - : "${ga_version:?}" + local ga_version=${1:?} echo "--- Installing Global Arrays ${ga_version} ---" @@ -102,10 +96,7 @@ function install_ga { } function install_petsc { - local petsc_version=$1 - - # check args - : "${petsc_version:?}" + local petsc_version=${1:?} echo "--- Installing PETSc ${petsc_version} ---" diff --git a/build/install_gridpack.sh b/build/install_gridpack.sh index d610e6f8d..40552a381 100644 --- a/build/install_gridpack.sh +++ b/build/install_gridpack.sh @@ -16,10 +16,9 @@ function install_gridpack { echo "--- GridPACK ---" # args - local gridpack_deps_dir=$1 - local gridpack_build_dir=$2 - local gridpack_install_dir=$3 - : "${gridpack_deps_dir:?}" "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" + local gridpack_deps_dir=${1:?} + local gridpack_build_dir=${2:?} + local gridpack_install_dir=${3:?} # remove existing build and install dir rm -rf "$gridpack_build_dir" @@ -69,9 +68,8 @@ function install_gridpack_python { echo "--- GridPACK python wrapper ---" # args - local gridpack_build_dir=$1 - local gridpack_install_dir=$2 - : "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" + local gridpack_build_dir=${1:?} + local gridpack_install_dir=${2:?} # update submodules echo "Updating GridPACK submodules" From e2580771f4ae0c7a7359a65af6f0302872f5a6c8 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 7 Feb 2024 08:24:49 -0600 Subject: [PATCH 156/195] Roll functions into lib.sh --- .gitlab-ci.yml | 31 ++-- build/check_container_tag_exists.sh | 65 ------- build/dockerfile | 7 +- build/install_package_deps.sh | 51 ------ build/lib.sh | 252 ++++++++++++++++++++++++++++ build/load_mpi_module.sh | 24 --- 6 files changed, 269 insertions(+), 161 deletions(-) delete mode 100644 build/check_container_tag_exists.sh delete mode 100644 build/install_package_deps.sh create mode 100644 build/lib.sh delete mode 100644 build/load_mpi_module.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58939648c..bd074f131 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,11 +11,15 @@ stages: [ build-container, build-gridpack, test-gridpack ] variables: MAKE_JOBS: "2" -check-container-tag-exists: + +check-container-definition-changed: image: alpine script: - - apk add --no-cache curl jq - - /bin/bash ./check_container_tag_exists.sh + - touch container-definition-changed + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: [ "build/**/*" ] + # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy .build-container: @@ -24,27 +28,18 @@ check-container-tag-exists: image: name: gcr.io/kaniko-project/executor:v1.9.0-debug entrypoint: [""] - variables: - IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env + needs: [ check-container-definition-changed ] script: - - > - /kaniko/executor - --context build - --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" - --build-arg "http_proxy=$http_proxy" - --build-arg "https_proxy=$https_proxy" - --dockerfile build/dockerfile - --destination "${IMAGE_PATH}" > build.log 2>&1 - rules: - - if: $CI_PIPELINE_SOURCE == "push" - changes: [ "build/**/*" ] + - local force_build="false" + - test -f container-definition-changed && force_build="true" + - . ./build/lib.sh + - check_build_container "${BASE_IMAGE}" "${force_build}" > build.log 2>&1 build-container:ubuntu: extends: .build-container variables: - BASE_IMAGE: ubuntu - BASE_IMAGE_TAG: "22.04" + BASE_IMAGE: ubuntu:22.04 .build-gridpack: diff --git a/build/check_container_tag_exists.sh b/build/check_container_tag_exists.sh deleted file mode 100644 index a4614334f..000000000 --- a/build/check_container_tag_exists.sh +++ /dev/null @@ -1,65 +0,0 @@ -#! /bin/bash - -# creates a file named container-tag-not-found if a tag does not exist in the container registry -# needs env vars CI_JOB_TOKEN, CI_API_V4_URL, CI_PROJECT_ID, IMAGE_TAG, CI_PROJECT_PATH_SLUG -# needs curl and jq - - -# bash options: -# - xtrace: print each command before executing it -# - errexit: exit on error -# - nounset: treat unset variables as errors -# - pipefail: treat whole pipeline as errored if any commands within error -# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html -set -o xtrace -o errexit -o nounset -o pipefail - - -# check that a command is available -check_installed() { - if ! command -v "$1" &>/dev/null; then - echo "please install '$1'". >&2 - exit 1 - fi -} - - -# submit a GET request to some endpoint on the project api -function get_container_registry_repos { - # get endpoint or use empty string if not provided - # https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html - local endpoint=${1:-} - - # curl options: - # - no-progress-meter: don't show progress bar but show errors - # - location: follow redirects - # - fail: return 22 on server errors - # - header: send private token for authentication - # https://docs.gitlab.com/ee/api/container_registry.html - curl \ - --no-progress-meter \ - --fail \ - --location \ - --header "PRIVATE-TOKEN: ${CI_JOB_TOKEN:?}" \ - "${CI_API_V4_URL:?}/projects/${CI_PROJECT_ID:?}/registry/repositories${endpoint}" -} - - -check_installed jq -check_installed curl - -# query the registry repo id -repo_id=$( - # jq query: - # - raw-output: output raw strings instead of json - # - exit-status: return 1 if no results - # - for each elements of the array - # - where the registry repo path is the one we care about - # - select the id prop - # https://jqlang.github.io/jq/manual/ - get_container_registry_repos | jq --raw-output --exit-status ".[] | select(.path == \"${CI_PROJECT_PATH_SLUG}\") | .id" -) - -# using that repo id check for a particular tag -# if command errors, create file named container-tag-not-found -get_container_registry_repos "/${repo_id}/tags/${IMAGE_TAG:?}" >/dev/null || - touch container-tag-not-found diff --git a/build/dockerfile b/build/dockerfile index 68395ec26..5fe7c2bcb 100644 --- a/build/dockerfile +++ b/build/dockerfile @@ -16,8 +16,9 @@ ENV https_proxy=${https_proxy} WORKDIR ${GP_EXT_DEPS} -COPY load_mpi_module.sh install_package_deps.sh install_from_source_deps.sh ./ -RUN . ./load_mpi_module.sh \ - && /bin/bash ./install_package_deps.sh \ +COPY lib.sh install_from_source_deps.sh ./ +RUN . ./lib.sh \ + && install_packages \ + && load_mpi_module \ && /bin/bash ./install_from_source_deps.sh \ && rm *.sh diff --git a/build/install_package_deps.sh b/build/install_package_deps.sh deleted file mode 100644 index 4d5ea3035..000000000 --- a/build/install_package_deps.sh +++ /dev/null @@ -1,51 +0,0 @@ -#! /bin/bash - -# installs necessary packages for a given distribution - -# bash options: -# - xtrace: print each command before executing it -# - errexit: exit on error -# - nounset: treat unset variables as errors -# - pipefail: treat whole pipeline as errored if any commands within error -# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html -set -o xtrace -o errexit -o nounset -o pipefail - -distribution=$( - source /etc/os-release - echo "$ID" -) - -case $distribution in - - debian | ubuntu) - - apt-get update - apt-get upgrade --yes - - # install required packages - apt-get install --yes \ - wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py - ;; - - fedora | rhel | centos | rocky) - - dnf upgrade --assumeyes --verbose - - # use epel and crb repos - # https://wiki.rockylinux.org/rocky/repo/#notes-on-epel - dnf install epel-release --assumeyes - crb enable - - # install required packages - dnf install --assumeyes \ - wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ - python3-mpi4py-openmpi - ;; - - *) - - echo "$distribution not supported" - exit 1 - ;; - -esac diff --git a/build/lib.sh b/build/lib.sh new file mode 100644 index 000000000..9deeaddb6 --- /dev/null +++ b/build/lib.sh @@ -0,0 +1,252 @@ +#! /bin/bash + +# contains functions to use across multiple scripts +# source these functions from the same dir using: +# script_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)" +# source "$script_root/lib.sh" +# https://www.gnu.org/software/bash/manual/html_node/Bash-Variables.html + +# reference line to copy-paste, not necessarily to source +# bash options: +# - xtrace: print each command before executing it +# - errexit: exit on error +# - nounset: treat unset variables as errors +# - pipefail: treat whole pipeline as errored if any commands within error +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace -o errexit -o nounset -o pipefail + +# print a set of args as a comma-space seperated list +# usage: +# print_array item1 item2 item3 +# usage: +# array=('item1' 'item2' 'item3') +# print_array "${array[@]}" +# https://www.gnu.org/software/bash/manual/html_node/Arrays.html +print_array() { + local list + list=$(printf "'%s', " "${@}") + echo "${list%, }" +} + +# check that a command is available +# usage: +# check_installed jq curl +check_installed() { + # gather a list of missing commands + missing_commands=() + for cmd in "${@}"; do + if ! command -v "$cmd" &>/dev/null; then + missing_commands+=("$cmd") + fi + done + + # if any commands are missing, print an error and exit + if [[ ${#missing_commands[@]} -gt 0 ]]; then + local list + list=$(print_array "${missing_commands[@]}") + echo "Please install $list" >&2 + exit 1 + fi +} + +# load the mpi module on RHEL distributions +function load_mpi_module { + if ! on_rhel_distro; then return; fi + + echo "Loading mpi module" + # shellcheck disable=SC1091 + source /etc/profile.d/modules.sh + module load "mpi/openmpi-$(arch)" +} + +# detect if this is a "debian" or "rhel" based linux distribution +# usage: +# if test "$(get_base_distro)" = "debian"; then +# apt-get update +# apt-get upgrade --yes +# fi +function get_base_distro { + distribution=$( + # shellcheck disable=SC1091 + source /etc/os-release + echo "$ID" + ) + + case $distribution in + debian | ubuntu) + echo "debian" + ;; + fedora | rhel | centos | rocky) + echo "rhel" + ;; + *) + echo "unknown" + ;; + esac +} + +# install packages for RHEL +function install_rhel_packages { + dnf upgrade --assumeyes --verbose + + # use epel and crb repos + # https://wiki.rockylinux.org/rocky/repo/#notes-on-epel + dnf install epel-release --assumeyes + crb enable + + # install required packages + dnf install --assumeyes \ + wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ + python3-mpi4py-openmpi +} + +# install packages for debian +function install_debian_packages { + apt-get update + apt-get upgrade --yes + + # install required packages + apt-get install --yes \ + wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py +} + +# install packages for either debian or rhel +function install_packages { + local base_distro + base_distro=$(get_base_distro) + + if test "$base_distro" = "debian"; then + install_debian_packages + elif test "$base_distro" = "rhel"; then + install_rhel_packages + fi +} + +# make a gitlab api call +# usage: +# glab_api GET /projects/123/registry/repositories +# glab_api POST /projects/123/registry/repositories "name=foo" "path=bar" +# needs: curl, $CI_API_V4_URL, $CI_JOB_TOKEN +function glab_api { + local method=${1:?} + local endpoint=${2:?} + local form_items=("${@:3}") + + local api_url=${CI_API_V4_URL:?} + local api_token=${CI_JOB_TOKEN:?} + + # construct the form data + local form_data="" + for item in "${form_items[@]}"; do + form_data+="--form ${item} " + done + + # curl options: + # - no-progress-meter: don't show progress bar but show errors + # - location: follow redirects + # - fail: return 22 on server errors + # - header: send private token for authentication + # - request: specify the HTTP method + # https://docs.gitlab.com/ee/api/container_registry.html + curl \ + --no-progress-meter \ + --fail \ + --location \ + --header "PRIVATE-TOKEN: ${api_token}" \ + --request "${method}" \ + "${form_data}" \ + "${api_url}${endpoint}" +} + +# get the id of a container registry repo for a project by path +# usage: +# get_container_registry_repo_id +# needs: jq, curl, $CI_API_V4_URL, $CI_JOB_TOKEN, $CI_PROJECT_ID, $CI_PROJECT_PATH_SLUG +function get_container_registry_repo_id { + local project_id=${CI_PROJECT_ID:?} + local project_path_slug=${CI_PROJECT_PATH_SLUG:?} + + # jq query: + # - raw-output: output raw strings instead of json + # - exit-status: return 1 if no results + # - for each elements of the array + # - where the registry repo path is the one we care about + # - select the id prop + # https://jqlang.github.io/jq/manual/ + glab_api GET "/projects/${project_id}/registry/repositories" | + jq --raw-output --exit-status ".[] | select(.path == \"${project_path_slug}\") | .id" +} + +# check if a tag exists in a project's container registry +# usage: +# container_tag_exists 123 "my-tag" +# needs: curl, $CI_API_V4_URL, $CI_JOB_TOKEN, $CI_PROJECT_ID +function container_tag_exists { + local reg_repo_id=${1:?} + local image_tag=${2:?} + + local project_id=${CI_PROJECT_ID:?} + + glab_api GET "/projects/${project_id}/registry/repositories/${reg_repo_id}/tags/${image_tag}" >/dev/null +} + +# delete tag from a project's container registry +# usage: +# delete_container_tag 123 "my-tag" +# needs: curl, $CI_API_V4_URL, $CI_JOB_TOKEN, $CI_PROJECT_ID +function delete_container_tag { + local reg_repo_id=${1:?} + local image_tag=${2:?} + + local project_id=${CI_PROJECT_ID:?} + + glab_api DELETE "/projects/${project_id}/registry/repositories/${reg_repo_id}/tags/${image_tag}" >/dev/null +} + +# build the container image with kaniko +# usage: +# build_container "ubuntu:22.04" +# needs: /kaniko/executor, $HTTP_PROXY, $HTTPS_PROXY, $CI_REGISTRY_IMAGE +function build_container { + local base_image=${1:?} + + local project_registry=${CI_REGISTRY_IMAGE:?} + + # strip tag from base image + local base_image_no_tag=${base_image%%:*} + + # build with kaniko + # https://github.com/GoogleContainerTools/kaniko + /kaniko/executor \ + --context build \ + --build-arg "BASE_IMAGE=${base_image}" \ + --build-arg "http_proxy=${HTTP_PROXY}" \ + --build-arg "https_proxy=${HTTPS_PROXY}" \ + --dockerfile build/dockerfile \ + --destination "${project_registry}:${base_image_no_tag}-gridpack-env" +} + +# build container image if not available in registry or force_rebuild = 'true' +# usage: +# check_build_container "ubuntu:22.04" "true" +# needs: /kaniko/executor, jq, curl, $CI_API_V4_URL, $CI_JOB_TOKEN, $CI_PROJECT_ID, +# $CI_PROJECT_PATH_SLUG, $HTTP_PROXY, $HTTPS_PROXY, $CI_REGISTRY_IMAGE +function check_build_container { + local tag=${1:?} + local force_rebuild=${2:-false} + + local repo_id + repo_id=$(get_container_registry_repo_id) + + if [[ "$force_rebuild" == "true" ]] || ! container_tag_exists "$repo_id" "$tag"; then + # in case the build fails, remove the tag from the registry + # this will signal to the next pipeline that the container image still needs rebuilt + # even if relevant files have not changed + # the alternative is to unknowingly use an outdated image in subsequent pipelines + # and there are cases where a rebuild can succeed without any changes + delete_container_tag "$repo_id" "$tag" + + # build container image and push to registry + build_container "$tag" + fi +} diff --git a/build/load_mpi_module.sh b/build/load_mpi_module.sh deleted file mode 100644 index f03c90fac..000000000 --- a/build/load_mpi_module.sh +++ /dev/null @@ -1,24 +0,0 @@ -#! /bin/bash - -# source this script to load the mpi module on RHEL distributions - -# bash options: -# - xtrace: print each command before executing it -# - errexit: exit on error -# - nounset: treat unset variables as errors -# - pipefail: treat whole pipeline as errored if any commands within error -# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html -set -o xtrace -o errexit -o nounset -o pipefail - -distribution=$( - source /etc/os-release - echo "$ID" -) - -case $distribution in - fedora | rhel | centos | rocky) - echo "Loading mpi module" - source /etc/profile.d/modules.sh - module load "mpi/openmpi-$(arch)" - ;; -esac From a247e3386b4f220754d3fb868f918a693b943ea1 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 7 Feb 2024 09:09:29 -0600 Subject: [PATCH 157/195] Check commands are installed --- build/lib.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/lib.sh b/build/lib.sh index 9deeaddb6..991a7c754 100644 --- a/build/lib.sh +++ b/build/lib.sh @@ -135,6 +135,8 @@ function glab_api { local api_url=${CI_API_V4_URL:?} local api_token=${CI_JOB_TOKEN:?} + check_installed curl + # construct the form data local form_data="" for item in "${form_items[@]}"; do @@ -166,6 +168,8 @@ function get_container_registry_repo_id { local project_id=${CI_PROJECT_ID:?} local project_path_slug=${CI_PROJECT_PATH_SLUG:?} + check_installed jq + # jq query: # - raw-output: output raw strings instead of json # - exit-status: return 1 if no results From fe8402ea71c8567ac50f582652dc4329b49fd4e3 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 7 Feb 2024 11:33:44 -0600 Subject: [PATCH 158/195] Split apart check_build_container into two jobs We don't easily have access to curl and jq in the kaniko container. --- .gitlab-ci.yml | 43 +++++++++++++++++++++++++++++-------------- build/lib.sh | 25 ++++++++++++------------- 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bd074f131..5bc5184b8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ default: artifacts: when: always untracked: true + image: alpine stages: [ build-container, build-gridpack, test-gridpack ] @@ -10,10 +11,13 @@ stages: [ build-container, build-gridpack, test-gridpack ] variables: MAKE_JOBS: "2" + UBUNTU_BASE_IMAGE: "ubuntu:22.04" + UBUNTU_ENV_IMAGE: "ubuntu-gridpack-env" + ROCKY_BASE_IMAGE: "rockylinux:9" + ROCKY_ENV_IMAGE: "rocky-gridpack-env" check-container-definition-changed: - image: alpine script: - touch container-definition-changed rules: @@ -21,6 +25,22 @@ check-container-definition-changed: changes: [ "build/**/*" ] +.check-container-needs-built: + needs: + - job: check-container-definition-changed + optional: true + script: + - apk add --no-cache curl jq + - . ./build/lib.sh + - check_container_needs_built "${BASE_IMAGE}" + + +check-container-needs-built:ubuntu: + extends: .check-container-needs-built + variables: + BASE_IMAGE: ${UBUNTU_BASE_IMAGE} + + # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy .build-container: stage: build-container @@ -28,18 +48,17 @@ check-container-definition-changed: image: name: gcr.io/kaniko-project/executor:v1.9.0-debug entrypoint: [""] - needs: [ check-container-definition-changed ] script: - - local force_build="false" - - test -f container-definition-changed && force_build="true" - . ./build/lib.sh - - check_build_container "${BASE_IMAGE}" "${force_build}" > build.log 2>&1 + - test -f container-definition-changed && + build_container "${BASE_IMAGE}" > build.log 2>&1 build-container:ubuntu: extends: .build-container + needs: [ check-container-needs-built:ubuntu ] variables: - BASE_IMAGE: ubuntu:22.04 + BASE_IMAGE: ${UBUNTU_BASE_IMAGE} .build-gridpack: @@ -50,23 +69,19 @@ build-container:ubuntu: build-gridpack:ubuntu: extends: .build-gridpack - image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env - needs: - - job: build-container:ubuntu - optional: true + image: ${CI_REGISTRY_IMAGE}:${UBUNTU_ENV_IMAGE} + needs: [ build-container:ubuntu ] .test-gridpack: stage: test-gridpack script: - - useradd gridpack - - chown -R gridpack:gridpack . - - su gridpack -c 'ctest --test-dir src/build --output-on-failure' + - ctest --test-dir src/build --output-on-failure artifacts: paths: [ src/build/Testing/Temporary ] test-gridpack:ubuntu: extends: .test-gridpack - image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env + image: ${CI_REGISTRY_IMAGE}:${UBUNTU_ENV_IMAGE} needs: [ build-gridpack:ubuntu ] diff --git a/build/lib.sh b/build/lib.sh index 991a7c754..907e42a21 100644 --- a/build/lib.sh +++ b/build/lib.sh @@ -232,25 +232,24 @@ function build_container { # build container image if not available in registry or force_rebuild = 'true' # usage: -# check_build_container "ubuntu:22.04" "true" -# needs: /kaniko/executor, jq, curl, $CI_API_V4_URL, $CI_JOB_TOKEN, $CI_PROJECT_ID, -# $CI_PROJECT_PATH_SLUG, $HTTP_PROXY, $HTTPS_PROXY, $CI_REGISTRY_IMAGE -function check_build_container { +# check_container_needs_built "ubuntu:22.04" +# needs: jq, curl, $CI_API_V4_URL, $CI_JOB_TOKEN, $CI_PROJECT_ID, $CI_PROJECT_PATH_SLUG +function check_container_needs_built { local tag=${1:?} - local force_rebuild=${2:-false} local repo_id repo_id=$(get_container_registry_repo_id) - if [[ "$force_rebuild" == "true" ]] || ! container_tag_exists "$repo_id" "$tag"; then - # in case the build fails, remove the tag from the registry - # this will signal to the next pipeline that the container image still needs rebuilt - # even if relevant files have not changed - # the alternative is to unknowingly use an outdated image in subsequent pipelines - # and there are cases where a rebuild can succeed without any changes + # if we see a file in the current directory called "container-definition-changed" + # or the tag does not exist in the registry, then the container needs rebuilt + if test -f ./container-definition-changed || ! container_tag_exists "$repo_id" "$tag"; then + # in case the build fails, delete the image associated with this tag from the registry + # this will signal to potential subsequent pipelines that the container image still needs + # rebuilt even if relevant files have not changed + # this provents unknowingly using an outdated image in subsequent pipelines delete_container_tag "$repo_id" "$tag" - # build container image and push to registry - build_container "$tag" + # signal to the next job via an empty file artifact that the container needs built + touch ./container-needs-built fi } From 971c9723a8662347e5a37e4d195b352ecf42066f Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 7 Feb 2024 11:39:36 -0600 Subject: [PATCH 159/195] Remove doc comments --- build/lib.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/build/lib.sh b/build/lib.sh index 907e42a21..08dea802e 100644 --- a/build/lib.sh +++ b/build/lib.sh @@ -1,19 +1,6 @@ #! /bin/bash # contains functions to use across multiple scripts -# source these functions from the same dir using: -# script_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)" -# source "$script_root/lib.sh" -# https://www.gnu.org/software/bash/manual/html_node/Bash-Variables.html - -# reference line to copy-paste, not necessarily to source -# bash options: -# - xtrace: print each command before executing it -# - errexit: exit on error -# - nounset: treat unset variables as errors -# - pipefail: treat whole pipeline as errored if any commands within error -# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html -# set -o xtrace -o errexit -o nounset -o pipefail # print a set of args as a comma-space seperated list # usage: From 88217a22e32560a032bdc6a4c5b88c3dc385d5e3 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 7 Feb 2024 11:39:59 -0600 Subject: [PATCH 160/195] Use bash in dockerfile --- build/dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build/dockerfile b/build/dockerfile index 5fe7c2bcb..ab4265df3 100644 --- a/build/dockerfile +++ b/build/dockerfile @@ -17,8 +17,9 @@ ENV https_proxy=${https_proxy} WORKDIR ${GP_EXT_DEPS} COPY lib.sh install_from_source_deps.sh ./ -RUN . ./lib.sh \ +SHELL ["/bin/bash", "-c"] +RUN source ./lib.sh \ && install_packages \ && load_mpi_module \ - && /bin/bash ./install_from_source_deps.sh \ + && source ./install_from_source_deps.sh \ && rm *.sh From 2096c5c61fa9f21b1c2e596c72919095bd6ce266 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 16 Feb 2024 09:24:19 -0600 Subject: [PATCH 161/195] Reorg scripts --- .gitlab-ci.yml | 29 ++---- build/{lib.sh => container_lib.sh} | 75 +--------------- build/dockerfile | 47 ++++++---- build/install_boost.sh | 43 +++++++++ build/install_from_source_deps.sh | 138 ++--------------------------- build/install_ga.sh | 40 +++++++++ build/install_gridpack.sh | 10 ++- build/install_package_deps_lib.sh | 92 +++++++++++++++++++ build/install_petsc.sh | 45 ++++++++++ 9 files changed, 273 insertions(+), 246 deletions(-) rename build/{lib.sh => container_lib.sh} (75%) create mode 100644 build/install_boost.sh create mode 100644 build/install_ga.sh create mode 100644 build/install_package_deps_lib.sh create mode 100644 build/install_petsc.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5bc5184b8..354f0ed3c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,13 +1,11 @@ default: - tags: [ basic, gridpack, ikp, k8s ] + tags: [basic, gridpack, ikp, k8s] artifacts: when: always untracked: true image: alpine - -stages: [ build-container, build-gridpack, test-gridpack ] - +stages: [build-container, build-gridpack, test-gridpack] variables: MAKE_JOBS: "2" @@ -16,14 +14,12 @@ variables: ROCKY_BASE_IMAGE: "rockylinux:9" ROCKY_ENV_IMAGE: "rocky-gridpack-env" - check-container-definition-changed: script: - touch container-definition-changed rules: - if: $CI_PIPELINE_SOURCE == "push" - changes: [ "build/**/*" ] - + changes: ["build/**/*"] .check-container-needs-built: needs: @@ -31,16 +27,14 @@ check-container-definition-changed: optional: true script: - apk add --no-cache curl jq - - . ./build/lib.sh + - . ./build/container_lib.sh - check_container_needs_built "${BASE_IMAGE}" - check-container-needs-built:ubuntu: extends: .check-container-needs-built variables: BASE_IMAGE: ${UBUNTU_BASE_IMAGE} - # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy .build-container: stage: build-container @@ -49,39 +43,34 @@ check-container-needs-built:ubuntu: name: gcr.io/kaniko-project/executor:v1.9.0-debug entrypoint: [""] script: - - . ./build/lib.sh + - . ./build/container_lib.sh - test -f container-definition-changed && build_container "${BASE_IMAGE}" > build.log 2>&1 - build-container:ubuntu: extends: .build-container - needs: [ check-container-needs-built:ubuntu ] + needs: [check-container-needs-built:ubuntu] variables: BASE_IMAGE: ${UBUNTU_BASE_IMAGE} - .build-gridpack: stage: build-gridpack script: - /bin/bash ./build/install_gridpack.sh - build-gridpack:ubuntu: extends: .build-gridpack image: ${CI_REGISTRY_IMAGE}:${UBUNTU_ENV_IMAGE} - needs: [ build-container:ubuntu ] - + needs: [build-container:ubuntu] .test-gridpack: stage: test-gridpack script: - ctest --test-dir src/build --output-on-failure artifacts: - paths: [ src/build/Testing/Temporary ] - + paths: [src/build/Testing/Temporary] test-gridpack:ubuntu: extends: .test-gridpack image: ${CI_REGISTRY_IMAGE}:${UBUNTU_ENV_IMAGE} - needs: [ build-gridpack:ubuntu ] + needs: [build-gridpack:ubuntu] diff --git a/build/lib.sh b/build/container_lib.sh similarity index 75% rename from build/lib.sh rename to build/container_lib.sh index 08dea802e..d52a1f399 100644 --- a/build/lib.sh +++ b/build/container_lib.sh @@ -1,6 +1,6 @@ #! /bin/bash -# contains functions to use across multiple scripts +# library of functions related to building the container image # print a set of args as a comma-space seperated list # usage: @@ -36,79 +36,6 @@ check_installed() { fi } -# load the mpi module on RHEL distributions -function load_mpi_module { - if ! on_rhel_distro; then return; fi - - echo "Loading mpi module" - # shellcheck disable=SC1091 - source /etc/profile.d/modules.sh - module load "mpi/openmpi-$(arch)" -} - -# detect if this is a "debian" or "rhel" based linux distribution -# usage: -# if test "$(get_base_distro)" = "debian"; then -# apt-get update -# apt-get upgrade --yes -# fi -function get_base_distro { - distribution=$( - # shellcheck disable=SC1091 - source /etc/os-release - echo "$ID" - ) - - case $distribution in - debian | ubuntu) - echo "debian" - ;; - fedora | rhel | centos | rocky) - echo "rhel" - ;; - *) - echo "unknown" - ;; - esac -} - -# install packages for RHEL -function install_rhel_packages { - dnf upgrade --assumeyes --verbose - - # use epel and crb repos - # https://wiki.rockylinux.org/rocky/repo/#notes-on-epel - dnf install epel-release --assumeyes - crb enable - - # install required packages - dnf install --assumeyes \ - wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ - python3-mpi4py-openmpi -} - -# install packages for debian -function install_debian_packages { - apt-get update - apt-get upgrade --yes - - # install required packages - apt-get install --yes \ - wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py -} - -# install packages for either debian or rhel -function install_packages { - local base_distro - base_distro=$(get_base_distro) - - if test "$base_distro" = "debian"; then - install_debian_packages - elif test "$base_distro" = "rhel"; then - install_rhel_packages - fi -} - # make a gitlab api call # usage: # glab_api GET /projects/123/registry/repositories diff --git a/build/dockerfile b/build/dockerfile index ab4265df3..1b75eab50 100644 --- a/build/dockerfile +++ b/build/dockerfile @@ -1,25 +1,42 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -ENV GP_EXT_DEPS=/gridpack-dependencies -ENV BOOST_VERSION="1.78.0" -ENV GA_VERSION="5.8" -ENV PETSC_VERSION="3.16.4" -ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost/install_for_gridpack/lib:${GP_EXT_DEPS}/ga/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} -ENV OMPI_ALLOW_RUN_AS_ROOT="1" -ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM="1" +SHELL ["/bin/bash", "-c"] ARG http_proxy ARG https_proxy -ENV http_proxy=${http_proxy} -ENV https_proxy=${https_proxy} + +ENV GP_EXT_DEPS=/gridpack-dependencies \ + HTTP_PROXY=${http_proxy} \ + HTTPS_PROXY=${https_proxy} \ + OMPI_ALLOW_RUN_AS_ROOT="1" \ + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM="1" \ + LD_LIBRARY_PATH="${GP_EXT_DEPS}/boost/install_for_gridpack/lib:${GP_EXT_DEPS}/ga/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib" WORKDIR ${GP_EXT_DEPS} +RUN mkdir logs -COPY lib.sh install_from_source_deps.sh ./ -SHELL ["/bin/bash", "-c"] -RUN source ./lib.sh \ - && install_packages \ +COPY install_package_deps_lib.sh ./ +RUN set -xeuo pipefail \ + && source ./install_package_deps_lib.sh \ + && install_packages 2>&1 | tee logs/install_packages.log + +COPY install_boost.sh ./ +RUN set -xeuo pipefail \ + && source ./install_package_deps_lib.sh \ && load_mpi_module \ - && source ./install_from_source_deps.sh \ - && rm *.sh + && bash ./install_boost.sh 2>&1 | tee logs/install_boost.log + +COPY install_ga.sh ./ +RUN set -xeuo pipefail \ + && source ./install_package_deps_lib.sh \ + && load_mpi_module \ + && bash ./install_ga.sh 2>&1 | tee logs/install_ga.log + +COPY install_petsc.sh ./ +RUN set -xeuo pipefail \ + && source ./install_package_deps_lib.sh \ + && load_mpi_module \ + && bash ./install_petsc.sh 2>&1 | tee logs/install_petsc.log + +RUN rm *.sh diff --git a/build/install_boost.sh b/build/install_boost.sh new file mode 100644 index 000000000..2bf406150 --- /dev/null +++ b/build/install_boost.sh @@ -0,0 +1,43 @@ +#! /bin/bash + +boost_version="${BOOST_VERSION:-1.78.0}" + +echo "--- Installing Boost ${boost_version} ---" + +# remove existing +rm -rf boost* + +# download +echo "Downloading Boost" +wget \ + "https://boostorg.jfrog.io/artifactory/main/release/${boost_version}/source/boost_${boost_version//./_}.tar.gz" \ + -O boost.tar.gz \ + --quiet + +# unpack +echo "Unpacking Boost" +tar -xf boost.tar.gz && rm -f boost.tar.gz + +# remove version from dir name +mv "boost_${boost_version//./_}" boost + +pushd boost || exit + +# bootstrap +echo "Bootstrapping Boost" +./bootstrap.sh \ + --prefix=install_for_gridpack \ + --with-libraries=mpi,serialization,random,filesystem,system +echo 'using mpi ;' >>project-config.jam + +# build +echo "Building Boost" +./b2 -a -d+2 link=shared stage + +# install +echo "Installing Boost" +./b2 -a -d+2 link=shared install + +popd || exit + +echo "Boost installation complete" diff --git a/build/install_from_source_deps.sh b/build/install_from_source_deps.sh index 2c7fd067f..f160c0ddb 100644 --- a/build/install_from_source_deps.sh +++ b/build/install_from_source_deps.sh @@ -10,142 +10,14 @@ # https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html set -o xtrace -o errexit -o nounset -o pipefail -function install_boost { - local boost_version=${1:?} - - echo "--- Installing Boost ${boost_version} ---" - - # remove existing - rm -rf boost* - - # download - echo "Downloading Boost" - wget \ - "https://boostorg.jfrog.io/artifactory/main/release/${boost_version}/source/boost_${boost_version//./_}.tar.gz" \ - -O boost.tar.gz \ - --quiet - - # unpack - echo "Unpacking Boost" - tar -xf boost.tar.gz && rm -f boost.tar.gz - - # remove version from dir name - mv "boost_${boost_version//./_}" boost - - pushd boost || exit - - # bootstrap - echo "Bootstrapping Boost" - ./bootstrap.sh \ - --prefix=install_for_gridpack \ - --with-libraries=mpi,serialization,random,filesystem,system - echo 'using mpi ;' >>project-config.jam - - # build - echo "Building Boost" - ./b2 -a -d+2 link=shared stage - - # install - echo "Installing Boost" - ./b2 -a -d+2 link=shared install - - popd || exit - - echo "Boost installation complete" -} - -function install_ga { - local ga_version=${1:?} - - echo "--- Installing Global Arrays ${ga_version} ---" - - # download - echo "Downloading Global Arrays" - wget \ - "https://github.com/GlobalArrays/ga/releases/download/v${ga_version}/ga-${ga_version}.tar.gz" \ - -O ga.tar.gz \ - --quiet - - # unpack - echo "Unpacking Global Arrays" - tar -xf ga.tar.gz && rm -f ga.tar.gz - - # remove version from dir name - mv "ga-${ga_version}" ga - - pushd ga || exit - - # build - echo "Configuring Global Arrays" - ./configure \ - --with-mpi-ts \ - --disable-f77 \ - --without-blas \ - --enable-cxx \ - --enable-i4 \ - --prefix="${PWD}/install_for_gridpack" \ - --enable-shared - - # install - echo "Installing Global Arrays" - make -j "${MAKE_JOBS:-$(nproc)}" install - - popd || exit - - echo "Global Arrays installation complete" -} - -function install_petsc { - local petsc_version=${1:?} - - echo "--- Installing PETSc ${petsc_version} ---" - - # clone - echo "Cloning PETSc repository" - git clone https://gitlab.com/petsc/petsc.git - - pushd petsc || exit - - git checkout "tags/v${petsc_version}" -b "v${petsc_version}" - - export PETSC_DIR=${PWD} - export PETSC_ARCH=build-dir - - # install - echo "Configuring PETSc" - ./configure \ - --download-mumps \ - --download-scalapack \ - --download-metis \ - --download-parmetis \ - --download-suitesparse \ - --download-f2cblaslapack \ - --prefix="${PWD}"/install_for_gridpack \ - --scalar-type=complex \ - --with-shared-libraries=1 - - # build - echo "Building PETSc" - make - - # install - echo "Installing PETSc" - make install - - # check - echo "Checking PETSc" - make check - - popd || exit - - echo "PETSc installation complete" -} +# get the parent directory of this script +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd) echo "Installing GridPACK dependencies" date -install_boost "${BOOST_VERSION:?}" -install_ga "${GA_VERSION:?}" -install_petsc "${PETSC_VERSION:?}" +bash "${script_dir}/install_boost.sh" +bash "${script_dir}/install_ga.sh" +bash "${script_dir}/install_petsc.sh" echo "GridPACK dependency installation complete" diff --git a/build/install_ga.sh b/build/install_ga.sh new file mode 100644 index 000000000..34dfa0e16 --- /dev/null +++ b/build/install_ga.sh @@ -0,0 +1,40 @@ +#! /bin/bash + +ga_version="${GA_VERSION:-5.8}" + +echo "--- Installing Global Arrays ${ga_version} ---" + +# download +echo "Downloading Global Arrays" +wget \ + "https://github.com/GlobalArrays/ga/releases/download/v${ga_version}/ga-${ga_version}.tar.gz" \ + -O ga.tar.gz \ + --quiet + +# unpack +echo "Unpacking Global Arrays" +tar -xf ga.tar.gz && rm -f ga.tar.gz + +# remove version from dir name +mv "ga-${ga_version}" ga + +pushd ga || exit + +# build +echo "Configuring Global Arrays" +./configure \ + --with-mpi-ts \ + --disable-f77 \ + --without-blas \ + --enable-cxx \ + --enable-i4 \ + --prefix="${PWD}/install_for_gridpack" \ + --enable-shared + +# install +echo "Installing Global Arrays" +make -j "${MAKE_JOBS:-$(nproc)}" install + +popd || exit + +echo "Global Arrays installation complete" diff --git a/build/install_gridpack.sh b/build/install_gridpack.sh index 40552a381..f450ced3f 100644 --- a/build/install_gridpack.sh +++ b/build/install_gridpack.sh @@ -85,9 +85,9 @@ function install_gridpack_python { # set an env var if we are running on RHEL case $distribution in - fedora | rhel | centos | rocky) - export RHEL_OPENMPI_HACK=yes - ;; + fedora | rhel | centos | rocky) + export RHEL_OPENMPI_HACK=yes + ;; esac # set python executable path @@ -126,7 +126,9 @@ date build_dir=${PWD}/src/build install_dir=${PWD}/src/install -install_gridpack "${GP_EXT_DEPS:?}" "$build_dir" "$install_dir" +gp_ext_deps=${GP_EXT_DEPS:-/gridpack-dependencies} + +install_gridpack "$gp_ext_deps" "$build_dir" "$install_dir" install_gridpack_python "$build_dir" "$install_dir" echo "Completed GridPACK installation" diff --git a/build/install_package_deps_lib.sh b/build/install_package_deps_lib.sh new file mode 100644 index 000000000..29d3e02f9 --- /dev/null +++ b/build/install_package_deps_lib.sh @@ -0,0 +1,92 @@ +#! /bin/bash + +# library to install packages required to build GridPACK on debian or RHEL based distros +# meant to be sourced + +# load the mpi module on RHEL distributions +function load_mpi_module { + local base_distro + base_distro=$(get_base_distro) + + if ! test "$base_distro" = "rhel"; then + return + fi + + echo "Loading mpi module" + + # clear unbound variable check temporarily + option_state=$([[ $- == *u* ]] && echo "-u" || echo "+u") + set +u + + # source the script which contains the module command + # shellcheck disable=SC1091 + source /etc/profile.d/modules.sh + + # reset the unbound variable check setting + set "$option_state" + + # load the mpi module + module load "mpi/openmpi-$(arch)" +} + +# detect if this is a "debian" or "rhel" based linux distribution +# usage: +# if test "$(get_base_distro)" = "debian"; then +# apt-get update +# apt-get upgrade --yes +# fi +function get_base_distro { + distribution=$( + # shellcheck disable=SC1091 + source /etc/os-release + echo "$ID" + ) + + case $distribution in + debian | ubuntu) + echo "debian" + ;; + fedora | rhel | centos | rocky) + echo "rhel" + ;; + *) + echo "unknown" + ;; + esac +} + +# install packages for RHEL +function install_rhel_packages { + dnf upgrade --assumeyes --verbose + + # use epel and crb repos + # https://wiki.rockylinux.org/rocky/repo/#notes-on-epel + dnf install epel-release --assumeyes + crb enable + + # install required packages + dnf install --assumeyes \ + wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ + python3-mpi4py-openmpi +} + +# install packages for debian +function install_debian_packages { + apt-get update + apt-get upgrade --yes + + # install required packages + apt-get install --yes \ + wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py +} + +function install_packages { + local base_distro + base_distro=$(get_base_distro) + + if test "$base_distro" = "debian"; then + install_debian_packages + elif test "$base_distro" = "rhel"; then + install_rhel_packages + fi +} diff --git a/build/install_petsc.sh b/build/install_petsc.sh new file mode 100644 index 000000000..e3abc3f01 --- /dev/null +++ b/build/install_petsc.sh @@ -0,0 +1,45 @@ +#! /bin/bash + +petsc_version="${PETSC_VERSION:-3.16.4}" + +echo "--- Installing PETSc ${petsc_version} ---" + +# clone +echo "Cloning PETSc repository" +git clone https://gitlab.com/petsc/petsc.git + +pushd petsc || exit + +git checkout "tags/v${petsc_version}" -b "v${petsc_version}" + +export PETSC_DIR=${PWD} +export PETSC_ARCH=build-dir + +# install +echo "Configuring PETSc" +./configure \ + --download-mumps \ + --download-scalapack \ + --download-metis \ + --download-parmetis \ + --download-suitesparse \ + --download-f2cblaslapack \ + --prefix="${PWD}"/install_for_gridpack \ + --scalar-type=complex \ + --with-shared-libraries=1 + +# build +echo "Building PETSc" +make + +# install +echo "Installing PETSc" +make install + +# check +echo "Checking PETSc" +make check + +popd || exit + +echo "PETSc installation complete" From 41be3a4ee1c0a1680061b792c3576c48ae13b8a6 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sun, 18 Feb 2024 13:15:10 -0600 Subject: [PATCH 162/195] Use lib function to load mpi module and test distro --- build/install_gridpack.sh | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/build/install_gridpack.sh b/build/install_gridpack.sh index f450ced3f..5eb106b2f 100644 --- a/build/install_gridpack.sh +++ b/build/install_gridpack.sh @@ -34,6 +34,9 @@ function install_gridpack { # remove existing cmake output rm -rf CMake* + # load mpi module for RHEL + load_mpi_module + # generate make files echo "Generating GridPACK make files" cmake \ @@ -77,18 +80,11 @@ function install_gridpack_python { pushd python || exit - # detect distribution - distribution=$( - source /etc/os-release - echo "$ID" - ) - - # set an env var if we are running on RHEL - case $distribution in - fedora | rhel | centos | rocky) + # export RHEL_OPENMPI_HACK for RHEL + distribution=$(get_base_distro) + if test "$distribution" = "rhel"; then export RHEL_OPENMPI_HACK=yes - ;; - esac + fi # set python executable path python_exe=$(which python || which python3) @@ -120,6 +116,12 @@ function install_gridpack_python { echo "GridPACK python wrapper installation complete" } +# get the parent directory of this script +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +# load the `load_mpi_module`, `get_base_distro` functions +source "$script_dir/install_package_deps_lib.sh" + echo "Installing GridPACK" date From 8c59fcdde208cac66c6c24446438dc8f79910e5f Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sun, 18 Feb 2024 13:16:11 -0600 Subject: [PATCH 163/195] Load mpi module before running tests --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 354f0ed3c..89849266d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -66,6 +66,11 @@ build-gridpack:ubuntu: .test-gridpack: stage: test-gridpack script: + # load mpi module if on RHEL + - . ./build/install_package_deps_lib.sh + - load_mpi_module + + # run tests - ctest --test-dir src/build --output-on-failure artifacts: paths: [src/build/Testing/Temporary] From 66c080ed345f6dd43410977b70461cdfc19747bf Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sun, 18 Feb 2024 13:25:25 -0600 Subject: [PATCH 164/195] Load mpi module from the boost script directly Remove redundancy from dockerfile --- build/dockerfile | 15 +++------------ build/install_boost.sh | 7 +++++++ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/build/dockerfile b/build/dockerfile index 1b75eab50..ca1257042 100644 --- a/build/dockerfile +++ b/build/dockerfile @@ -22,21 +22,12 @@ RUN set -xeuo pipefail \ && install_packages 2>&1 | tee logs/install_packages.log COPY install_boost.sh ./ -RUN set -xeuo pipefail \ - && source ./install_package_deps_lib.sh \ - && load_mpi_module \ - && bash ./install_boost.sh 2>&1 | tee logs/install_boost.log +RUN set -xeuo pipefail && bash ./install_boost.sh 2>&1 | tee logs/install_boost.log COPY install_ga.sh ./ -RUN set -xeuo pipefail \ - && source ./install_package_deps_lib.sh \ - && load_mpi_module \ - && bash ./install_ga.sh 2>&1 | tee logs/install_ga.log +RUN set -xeuo pipefail && bash ./install_ga.sh 2>&1 | tee logs/install_ga.log COPY install_petsc.sh ./ -RUN set -xeuo pipefail \ - && source ./install_package_deps_lib.sh \ - && load_mpi_module \ - && bash ./install_petsc.sh 2>&1 | tee logs/install_petsc.log +RUN set -xeuo pipefail && bash ./install_petsc.sh 2>&1 | tee logs/install_petsc.log RUN rm *.sh diff --git a/build/install_boost.sh b/build/install_boost.sh index 2bf406150..0d9ae934e 100644 --- a/build/install_boost.sh +++ b/build/install_boost.sh @@ -1,5 +1,12 @@ #! /bin/bash +# get the parent directory of this script +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +# load mpi module for RHEL +source "$script_dir/install_package_deps_lib.sh" +load_mpi_module + boost_version="${BOOST_VERSION:-1.78.0}" echo "--- Installing Boost ${boost_version} ---" From 5b209f93ce00227ca639631c2feb8e529e94a3f0 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sun, 18 Feb 2024 13:29:13 -0600 Subject: [PATCH 165/195] Add container check jobs to a stage --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 89849266d..93bbf5f16 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ default: untracked: true image: alpine -stages: [build-container, build-gridpack, test-gridpack] +stages: [check-container, build-container, build-gridpack, test-gridpack] variables: MAKE_JOBS: "2" @@ -15,6 +15,7 @@ variables: ROCKY_ENV_IMAGE: "rocky-gridpack-env" check-container-definition-changed: + stage: check-container script: - touch container-definition-changed rules: @@ -22,6 +23,7 @@ check-container-definition-changed: changes: ["build/**/*"] .check-container-needs-built: + stage: check-container needs: - job: check-container-definition-changed optional: true From cab51198aeab4791d677e10c3a6f9ef0c0b920f7 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sun, 18 Feb 2024 22:40:12 -0600 Subject: [PATCH 166/195] Use image with bash to run container image checks --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 93bbf5f16..5703db506 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ default: artifacts: when: always untracked: true - image: alpine + image: ubuntu:22.04 stages: [check-container, build-container, build-gridpack, test-gridpack] @@ -28,8 +28,7 @@ check-container-definition-changed: - job: check-container-definition-changed optional: true script: - - apk add --no-cache curl jq - - . ./build/container_lib.sh + - source ./build/container_lib.sh - check_container_needs_built "${BASE_IMAGE}" check-container-needs-built:ubuntu: From 8b3c60a0b6525678a37b93ec3fd4280e06ed5a8b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sun, 18 Feb 2024 22:43:29 -0600 Subject: [PATCH 167/195] Install jq for container image checks --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5703db506..7c49db385 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,7 @@ check-container-definition-changed: - job: check-container-definition-changed optional: true script: + - apt-get update && apt-get install -y jq - source ./build/container_lib.sh - check_container_needs_built "${BASE_IMAGE}" @@ -44,7 +45,7 @@ check-container-needs-built:ubuntu: name: gcr.io/kaniko-project/executor:v1.9.0-debug entrypoint: [""] script: - - . ./build/container_lib.sh + - source ./build/container_lib.sh - test -f container-definition-changed && build_container "${BASE_IMAGE}" > build.log 2>&1 @@ -68,7 +69,7 @@ build-gridpack:ubuntu: stage: test-gridpack script: # load mpi module if on RHEL - - . ./build/install_package_deps_lib.sh + - source ./build/install_package_deps_lib.sh - load_mpi_module # run tests From 65ff2e9484605c7acc490920bf058e062e2f04ea Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sun, 18 Feb 2024 23:00:14 -0600 Subject: [PATCH 168/195] Need curl too --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7c49db385..8838066ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,7 +28,7 @@ check-container-definition-changed: - job: check-container-definition-changed optional: true script: - - apt-get update && apt-get install -y jq + - apt-get update && apt-get install -y curl jq - source ./build/container_lib.sh - check_container_needs_built "${BASE_IMAGE}" From aacf18261094abbefd3dcfa3accf7ddf0db54e4f Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sun, 18 Feb 2024 23:06:29 -0600 Subject: [PATCH 169/195] Troubleshoot url issue --- .gitlab-ci.yml | 1 + build/container_lib.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8838066ae..1ffd556f7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,7 @@ check-container-definition-changed: - job: check-container-definition-changed optional: true script: + - set -o xtrace -o errexit -o nounset -o pipefail - apt-get update && apt-get install -y curl jq - source ./build/container_lib.sh - check_container_needs_built "${BASE_IMAGE}" diff --git a/build/container_lib.sh b/build/container_lib.sh index d52a1f399..76c5c21ba 100644 --- a/build/container_lib.sh +++ b/build/container_lib.sh @@ -64,6 +64,8 @@ function glab_api { # - header: send private token for authentication # - request: specify the HTTP method # https://docs.gitlab.com/ee/api/container_registry.html + echo "${api_url}${endpoint}" + echo "${form_data}" curl \ --no-progress-meter \ --fail \ From 18751025f8d5adaf3851e70ce4f84855510dc6f4 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sun, 18 Feb 2024 23:15:56 -0600 Subject: [PATCH 170/195] Merge form data with url so theres not a false arg --- build/container_lib.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/container_lib.sh b/build/container_lib.sh index 76c5c21ba..bca4cf2c8 100644 --- a/build/container_lib.sh +++ b/build/container_lib.sh @@ -72,8 +72,7 @@ function glab_api { --location \ --header "PRIVATE-TOKEN: ${api_token}" \ --request "${method}" \ - "${form_data}" \ - "${api_url}${endpoint}" + "${form_data}${api_url}${endpoint}" } # get the id of a container registry repo for a project by path From ac71d3950cf148111032776474efcb006cfedd1a Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sun, 18 Feb 2024 23:45:33 -0600 Subject: [PATCH 171/195] Try different header name --- build/container_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/container_lib.sh b/build/container_lib.sh index bca4cf2c8..2ce0cccff 100644 --- a/build/container_lib.sh +++ b/build/container_lib.sh @@ -70,7 +70,7 @@ function glab_api { --no-progress-meter \ --fail \ --location \ - --header "PRIVATE-TOKEN: ${api_token}" \ + --header "JOB-TOKEN: ${api_token}" \ --request "${method}" \ "${form_data}${api_url}${endpoint}" } From 4483102b327aa21e6ec4d34b5c0a11e7a538c878 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 10:36:11 -0600 Subject: [PATCH 172/195] Use PRIVATE_TOKEN until gitlab updated to allow JOB_TOKEN by default --- build/container_lib.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/build/container_lib.sh b/build/container_lib.sh index 2ce0cccff..1b180c885 100644 --- a/build/container_lib.sh +++ b/build/container_lib.sh @@ -40,14 +40,14 @@ check_installed() { # usage: # glab_api GET /projects/123/registry/repositories # glab_api POST /projects/123/registry/repositories "name=foo" "path=bar" -# needs: curl, $CI_API_V4_URL, $CI_JOB_TOKEN +# needs: curl, $CI_API_V4_URL, $API_TOKEN function glab_api { local method=${1:?} local endpoint=${2:?} local form_items=("${@:3}") local api_url=${CI_API_V4_URL:?} - local api_token=${CI_JOB_TOKEN:?} + local api_token=${API_TOKEN:?} check_installed curl @@ -69,8 +69,7 @@ function glab_api { curl \ --no-progress-meter \ --fail \ - --location \ - --header "JOB-TOKEN: ${api_token}" \ + --location \ --header "PRIVATE-TOKEN: ${api_token}" \ --request "${method}" \ "${form_data}${api_url}${endpoint}" } @@ -78,7 +77,7 @@ function glab_api { # get the id of a container registry repo for a project by path # usage: # get_container_registry_repo_id -# needs: jq, curl, $CI_API_V4_URL, $CI_JOB_TOKEN, $CI_PROJECT_ID, $CI_PROJECT_PATH_SLUG +# needs: jq, curl, $CI_API_V4_URL, $API_TOKEN, $CI_PROJECT_ID, $CI_PROJECT_PATH_SLUG function get_container_registry_repo_id { local project_id=${CI_PROJECT_ID:?} local project_path_slug=${CI_PROJECT_PATH_SLUG:?} @@ -99,7 +98,7 @@ function get_container_registry_repo_id { # check if a tag exists in a project's container registry # usage: # container_tag_exists 123 "my-tag" -# needs: curl, $CI_API_V4_URL, $CI_JOB_TOKEN, $CI_PROJECT_ID +# needs: curl, $CI_API_V4_URL, $API_TOKEN, $CI_PROJECT_ID function container_tag_exists { local reg_repo_id=${1:?} local image_tag=${2:?} @@ -112,7 +111,7 @@ function container_tag_exists { # delete tag from a project's container registry # usage: # delete_container_tag 123 "my-tag" -# needs: curl, $CI_API_V4_URL, $CI_JOB_TOKEN, $CI_PROJECT_ID +# needs: curl, $CI_API_V4_URL, $API_TOKEN, $CI_PROJECT_ID function delete_container_tag { local reg_repo_id=${1:?} local image_tag=${2:?} @@ -148,7 +147,7 @@ function build_container { # build container image if not available in registry or force_rebuild = 'true' # usage: # check_container_needs_built "ubuntu:22.04" -# needs: jq, curl, $CI_API_V4_URL, $CI_JOB_TOKEN, $CI_PROJECT_ID, $CI_PROJECT_PATH_SLUG +# needs: jq, curl, $CI_API_V4_URL, $API_TOKEN, $CI_PROJECT_ID, $CI_PROJECT_PATH_SLUG function check_container_needs_built { local tag=${1:?} From fc607c61d3c5f0c7f22dacb7bcaac36b62c94a77 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 10:42:46 -0600 Subject: [PATCH 173/195] Fix newline --- build/container_lib.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/container_lib.sh b/build/container_lib.sh index 1b180c885..c836aad32 100644 --- a/build/container_lib.sh +++ b/build/container_lib.sh @@ -69,7 +69,8 @@ function glab_api { curl \ --no-progress-meter \ --fail \ - --location \ --header "PRIVATE-TOKEN: ${api_token}" \ + --location \ + --header "PRIVATE-TOKEN: ${api_token}" \ --request "${method}" \ "${form_data}${api_url}${endpoint}" } From d328502dab70a0d37bce057ac0730508f528ef36 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 11:09:08 -0600 Subject: [PATCH 174/195] Use intermediate variable between curl and jq --- build/container_lib.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build/container_lib.sh b/build/container_lib.sh index c836aad32..bc61b81c8 100644 --- a/build/container_lib.sh +++ b/build/container_lib.sh @@ -85,6 +85,9 @@ function get_container_registry_repo_id { check_installed jq + response=$(glab_api GET "/projects/${project_id}/registry/repositories") + echo "${response}" + # jq query: # - raw-output: output raw strings instead of json # - exit-status: return 1 if no results @@ -92,8 +95,7 @@ function get_container_registry_repo_id { # - where the registry repo path is the one we care about # - select the id prop # https://jqlang.github.io/jq/manual/ - glab_api GET "/projects/${project_id}/registry/repositories" | - jq --raw-output --exit-status ".[] | select(.path == \"${project_path_slug}\") | .id" + jq --raw-output --exit-status ".[] | select(.path == \"${project_path_slug}\") | .id" <<< "${response}" } # check if a tag exists in a project's container registry From f275d8cae372ef495f962223732b8ce0b63d1ffa Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 11:38:33 -0600 Subject: [PATCH 175/195] Try echoing to jq --- build/container_lib.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/container_lib.sh b/build/container_lib.sh index bc61b81c8..460838b34 100644 --- a/build/container_lib.sh +++ b/build/container_lib.sh @@ -86,7 +86,6 @@ function get_container_registry_repo_id { check_installed jq response=$(glab_api GET "/projects/${project_id}/registry/repositories") - echo "${response}" # jq query: # - raw-output: output raw strings instead of json @@ -95,7 +94,7 @@ function get_container_registry_repo_id { # - where the registry repo path is the one we care about # - select the id prop # https://jqlang.github.io/jq/manual/ - jq --raw-output --exit-status ".[] | select(.path == \"${project_path_slug}\") | .id" <<< "${response}" + echo "${response}" | jq --raw-output --exit-status ".[] | select(.path == \"${project_path_slug}\") | .id" } # check if a tag exists in a project's container registry From cd133fb2deb000ac9a3474c28a3895746578ad9e Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 11:40:08 -0600 Subject: [PATCH 176/195] Remove the troubleshooting from earlier... --- build/container_lib.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/build/container_lib.sh b/build/container_lib.sh index 460838b34..1ff8719e9 100644 --- a/build/container_lib.sh +++ b/build/container_lib.sh @@ -64,8 +64,6 @@ function glab_api { # - header: send private token for authentication # - request: specify the HTTP method # https://docs.gitlab.com/ee/api/container_registry.html - echo "${api_url}${endpoint}" - echo "${form_data}" curl \ --no-progress-meter \ --fail \ @@ -85,8 +83,6 @@ function get_container_registry_repo_id { check_installed jq - response=$(glab_api GET "/projects/${project_id}/registry/repositories") - # jq query: # - raw-output: output raw strings instead of json # - exit-status: return 1 if no results @@ -94,7 +90,8 @@ function get_container_registry_repo_id { # - where the registry repo path is the one we care about # - select the id prop # https://jqlang.github.io/jq/manual/ - echo "${response}" | jq --raw-output --exit-status ".[] | select(.path == \"${project_path_slug}\") | .id" + glab_api GET "/projects/${project_id}/registry/repositories" \ + | jq --raw-output --exit-status ".[] | select(.path == \"${project_path_slug}\") | .id" } # check if a tag exists in a project's container registry From 55bfa713454c32d2328713488e25655c8ba0125a Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 12:06:39 -0600 Subject: [PATCH 177/195] Use proper tag when checking registry --- .gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1ffd556f7..9447518e0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ check-container-definition-changed: - set -o xtrace -o errexit -o nounset -o pipefail - apt-get update && apt-get install -y curl jq - source ./build/container_lib.sh - - check_container_needs_built "${BASE_IMAGE}" + - check_container_needs_built "${ENV_IMAGE}" check-container-needs-built:ubuntu: extends: .check-container-needs-built variables: - BASE_IMAGE: ${UBUNTU_BASE_IMAGE} + ENV_IMAGE: ${UBUNTU_ENV_IMAGE} # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy .build-container: @@ -47,8 +47,7 @@ check-container-needs-built:ubuntu: entrypoint: [""] script: - source ./build/container_lib.sh - - test -f container-definition-changed && - build_container "${BASE_IMAGE}" > build.log 2>&1 + - (test -f container-definition-changed || test -f container-needs-built) && build_container "${BASE_IMAGE}" > build.log 2>&1 build-container:ubuntu: extends: .build-container From 1b88fbab70489aa1cf331a896e8b8dc020746d26 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 12:08:22 -0600 Subject: [PATCH 178/195] Only depend on container-needs-built file in .build-container --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9447518e0..93fce85e6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,7 +47,7 @@ check-container-needs-built:ubuntu: entrypoint: [""] script: - source ./build/container_lib.sh - - (test -f container-definition-changed || test -f container-needs-built) && build_container "${BASE_IMAGE}" > build.log 2>&1 + - test -f container-needs-built && build_container "${BASE_IMAGE}" > build.log 2>&1 build-container:ubuntu: extends: .build-container From c9aa739728d905d97c140f6a49f72fac7ad4aa7d Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 12:17:57 -0600 Subject: [PATCH 179/195] Try project path, not project path slug --- build/container_lib.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build/container_lib.sh b/build/container_lib.sh index 1ff8719e9..68485476a 100644 --- a/build/container_lib.sh +++ b/build/container_lib.sh @@ -76,10 +76,11 @@ function glab_api { # get the id of a container registry repo for a project by path # usage: # get_container_registry_repo_id -# needs: jq, curl, $CI_API_V4_URL, $API_TOKEN, $CI_PROJECT_ID, $CI_PROJECT_PATH_SLUG +# needs: jq, curl, $CI_API_V4_URL, $API_TOKEN, $CI_PROJECT_ID, $CI_PROJECT_PATH function get_container_registry_repo_id { local project_id=${CI_PROJECT_ID:?} - local project_path_slug=${CI_PROJECT_PATH_SLUG:?} + : "${CI_PROJECT_PATH:?}" + local project_path=${CI_PROJECT_PATH,,} check_installed jq @@ -91,7 +92,7 @@ function get_container_registry_repo_id { # - select the id prop # https://jqlang.github.io/jq/manual/ glab_api GET "/projects/${project_id}/registry/repositories" \ - | jq --raw-output --exit-status ".[] | select(.path == \"${project_path_slug}\") | .id" + | jq --raw-output --exit-status ".[] | select(.path == \"${project_path}\") | .id" } # check if a tag exists in a project's container registry From f3421d67c060d0190f9e63efaf76e1ddee412c8e Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 12:31:07 -0600 Subject: [PATCH 180/195] Inline build_container function Missing bash causes issues sourcing lib script --- .gitlab-ci.yml | 12 ++++++++++-- build/container_lib.sh | 23 ----------------------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 93fce85e6..728cda66e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,14 +46,22 @@ check-container-needs-built:ubuntu: name: gcr.io/kaniko-project/executor:v1.9.0-debug entrypoint: [""] script: - - source ./build/container_lib.sh - - test -f container-needs-built && build_container "${BASE_IMAGE}" > build.log 2>&1 + - | + test -f container-needs-built \ + && /kaniko/executor \ + --context build \ + --build-arg "BASE_IMAGE=${base_image}" \ + --build-arg "http_proxy=${HTTP_PROXY}" \ + --build-arg "https_proxy=${HTTPS_PROXY}" \ + --dockerfile build/dockerfile \ + --destination "${CI_REGISTRY_IMAGE}:${ENV_IMAGE}" > build.log 2>&1 build-container:ubuntu: extends: .build-container needs: [check-container-needs-built:ubuntu] variables: BASE_IMAGE: ${UBUNTU_BASE_IMAGE} + ENV_IMAGE: ${UBUNTU_ENV_IMAGE} .build-gridpack: stage: build-gridpack diff --git a/build/container_lib.sh b/build/container_lib.sh index 68485476a..febdf0a63 100644 --- a/build/container_lib.sh +++ b/build/container_lib.sh @@ -121,29 +121,6 @@ function delete_container_tag { glab_api DELETE "/projects/${project_id}/registry/repositories/${reg_repo_id}/tags/${image_tag}" >/dev/null } -# build the container image with kaniko -# usage: -# build_container "ubuntu:22.04" -# needs: /kaniko/executor, $HTTP_PROXY, $HTTPS_PROXY, $CI_REGISTRY_IMAGE -function build_container { - local base_image=${1:?} - - local project_registry=${CI_REGISTRY_IMAGE:?} - - # strip tag from base image - local base_image_no_tag=${base_image%%:*} - - # build with kaniko - # https://github.com/GoogleContainerTools/kaniko - /kaniko/executor \ - --context build \ - --build-arg "BASE_IMAGE=${base_image}" \ - --build-arg "http_proxy=${HTTP_PROXY}" \ - --build-arg "https_proxy=${HTTPS_PROXY}" \ - --dockerfile build/dockerfile \ - --destination "${project_registry}:${base_image_no_tag}-gridpack-env" -} - # build container image if not available in registry or force_rebuild = 'true' # usage: # check_container_needs_built "ubuntu:22.04" From dc82d3e293d7433c3a40b037bcce3814b19ef79c Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 13:22:47 -0600 Subject: [PATCH 181/195] Uppercase variable --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 728cda66e..eb8c29044 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,7 @@ check-container-needs-built:ubuntu: test -f container-needs-built \ && /kaniko/executor \ --context build \ - --build-arg "BASE_IMAGE=${base_image}" \ + --build-arg "BASE_IMAGE=${BASE_IMAGE}" \ --build-arg "http_proxy=${HTTP_PROXY}" \ --build-arg "https_proxy=${HTTPS_PROXY}" \ --dockerfile build/dockerfile \ From 6e9646f57260dd23e7879be534042766c3d44162 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 13:24:19 -0600 Subject: [PATCH 182/195] Break script into multiple array items --- .gitlab-ci.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eb8c29044..2bc8fc6e9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,15 +46,14 @@ check-container-needs-built:ubuntu: name: gcr.io/kaniko-project/executor:v1.9.0-debug entrypoint: [""] script: - - | - test -f container-needs-built \ - && /kaniko/executor \ - --context build \ - --build-arg "BASE_IMAGE=${BASE_IMAGE}" \ - --build-arg "http_proxy=${HTTP_PROXY}" \ - --build-arg "https_proxy=${HTTPS_PROXY}" \ - --dockerfile build/dockerfile \ - --destination "${CI_REGISTRY_IMAGE}:${ENV_IMAGE}" > build.log 2>&1 + - test -f container-needs-built || exit 0 + - /kaniko/executor \ + --context build \ + --build-arg "BASE_IMAGE=${BASE_IMAGE}" \ + --build-arg "http_proxy=${HTTP_PROXY}" \ + --build-arg "https_proxy=${HTTPS_PROXY}" \ + --dockerfile build/dockerfile \ + --destination "${CI_REGISTRY_IMAGE}:${ENV_IMAGE}" > build.log 2>&1 build-container:ubuntu: extends: .build-container From b5ea6aefc247c87cd070fde94d47a209e854646c Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 14:01:47 -0600 Subject: [PATCH 183/195] Try a different yaml string format for kaniko command --- .gitlab-ci.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2bc8fc6e9..dc6a56ec2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,12 +47,13 @@ check-container-needs-built:ubuntu: entrypoint: [""] script: - test -f container-needs-built || exit 0 - - /kaniko/executor \ - --context build \ - --build-arg "BASE_IMAGE=${BASE_IMAGE}" \ - --build-arg "http_proxy=${HTTP_PROXY}" \ - --build-arg "https_proxy=${HTTPS_PROXY}" \ - --dockerfile build/dockerfile \ + - >- + /kaniko/executor + --context build + --build-arg "BASE_IMAGE=${BASE_IMAGE}" + --build-arg "http_proxy=${HTTP_PROXY}" + --build-arg "https_proxy=${HTTPS_PROXY}" + --dockerfile build/dockerfile --destination "${CI_REGISTRY_IMAGE}:${ENV_IMAGE}" > build.log 2>&1 build-container:ubuntu: From 7edf6dbb7954cd6f360ec3068fc6c738a85b287f Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 14:59:16 -0600 Subject: [PATCH 184/195] Add a brief readme --- build/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 build/README.md diff --git a/build/README.md b/build/README.md new file mode 100644 index 000000000..d061c64a5 --- /dev/null +++ b/build/README.md @@ -0,0 +1,9 @@ +The shell scripts here are used to automate installation of GridPACK on Debian/RHEL based systems. These are leveraged by a build pipeline running on a PNNL GitLab instance that hosts a mirror of this GitHub repo. The build pipeline is defined by `/.gitlab-ci.yml` and used to automate software testing. An integration was configured to report the build result back to this GitHub repo for any commit triggering a build. `dockerfile` defines a container image used to create an environment with all necessary GridPACK dependencies, including those installed from packages and those installed from source. + +The build pipeline can be broken down into these steps: + +1. Check if container image definition has changed +2. Check if container image exists in the PNNL GitLab registry +3. If definition changed or image not in registry, build/push the image +3. Install GridPACK to a container created with the latest image +4. Execute GridPACK tests From 3aa1ca3158ff9afd266409e0587fda4a819d9f21 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 15:23:08 -0600 Subject: [PATCH 185/195] Add rules for when GridPACK build/test jobs should run --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dc6a56ec2..b04e4c891 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -67,6 +67,9 @@ build-container:ubuntu: stage: build-gridpack script: - /bin/bash ./build/install_gridpack.sh + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: ["build/**/*", "src/**/*", "python/**/*"] build-gridpack:ubuntu: extends: .build-gridpack @@ -84,6 +87,9 @@ build-gridpack:ubuntu: - ctest --test-dir src/build --output-on-failure artifacts: paths: [src/build/Testing/Temporary] + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: ["build/**/*", "src/**/*", "python/**/*"] test-gridpack:ubuntu: extends: .test-gridpack From 063d9548af8ca61260a18712ac705fd248373c95 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 22:38:29 -0600 Subject: [PATCH 186/195] Separate GP_EXT_DEPS for ref by LD_LIBRARY_PATH --- build/dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/dockerfile b/build/dockerfile index ca1257042..8526695a6 100644 --- a/build/dockerfile +++ b/build/dockerfile @@ -6,8 +6,8 @@ SHELL ["/bin/bash", "-c"] ARG http_proxy ARG https_proxy -ENV GP_EXT_DEPS=/gridpack-dependencies \ - HTTP_PROXY=${http_proxy} \ +ENV GP_EXT_DEPS=/gridpack-dependencies +ENV HTTP_PROXY=${http_proxy} \ HTTPS_PROXY=${https_proxy} \ OMPI_ALLOW_RUN_AS_ROOT="1" \ OMPI_ALLOW_RUN_AS_ROOT_CONFIRM="1" \ From c2fa80777273a5402a4bbe0fbd3a0841146dc6f7 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 19 Feb 2024 22:41:42 -0600 Subject: [PATCH 187/195] Add rocky linux jobs back --- .gitlab-ci.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b04e4c891..08a55f0cc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,6 +38,11 @@ check-container-needs-built:ubuntu: variables: ENV_IMAGE: ${UBUNTU_ENV_IMAGE} +check-container-needs-built:rocky: + extends: .check-container-needs-built + variables: + ENV_IMAGE: ${ROCKY_ENV_IMAGE} + # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy .build-container: stage: build-container @@ -63,6 +68,13 @@ build-container:ubuntu: BASE_IMAGE: ${UBUNTU_BASE_IMAGE} ENV_IMAGE: ${UBUNTU_ENV_IMAGE} +build-container:rocky: + extends: .build-container + needs: [check-container-needs-built:rocky] + variables: + BASE_IMAGE: ${ROCKY_BASE_IMAGE} + ENV_IMAGE: ${ROCKY_ENV_IMAGE} + .build-gridpack: stage: build-gridpack script: @@ -76,6 +88,11 @@ build-gridpack:ubuntu: image: ${CI_REGISTRY_IMAGE}:${UBUNTU_ENV_IMAGE} needs: [build-container:ubuntu] +build-gridpack:rocky: + extends: .build-gridpack + image: ${CI_REGISTRY_IMAGE}:${ROCKY_ENV_IMAGE} + needs: [build-container:rocky] + .test-gridpack: stage: test-gridpack script: @@ -95,3 +112,8 @@ test-gridpack:ubuntu: extends: .test-gridpack image: ${CI_REGISTRY_IMAGE}:${UBUNTU_ENV_IMAGE} needs: [build-gridpack:ubuntu] + +test-gridpack:rocky: + extends: .test-gridpack + image: ${CI_REGISTRY_IMAGE}:${ROCKY_ENV_IMAGE} + needs: [build-gridpack:rocky] From 9cebab7ba315aecf3581c599d127f99e6a7ff9ac Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 20 Feb 2024 11:52:06 -0600 Subject: [PATCH 188/195] Load mpi for GA and PETSC, fix docker RUN error handling --- build/dockerfile | 8 ++++---- build/install_ga.sh | 7 +++++++ build/install_petsc.sh | 7 +++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/build/dockerfile b/build/dockerfile index 8526695a6..a55ff6330 100644 --- a/build/dockerfile +++ b/build/dockerfile @@ -19,15 +19,15 @@ RUN mkdir logs COPY install_package_deps_lib.sh ./ RUN set -xeuo pipefail \ && source ./install_package_deps_lib.sh \ - && install_packages 2>&1 | tee logs/install_packages.log + && install_packages >logs/install_packages.log 2>&1 COPY install_boost.sh ./ -RUN set -xeuo pipefail && bash ./install_boost.sh 2>&1 | tee logs/install_boost.log +RUN bash -o xtrace -o errexit -o nounset -o pipefail ./install_boost.sh >logs/install_boost.log 2>&1 COPY install_ga.sh ./ -RUN set -xeuo pipefail && bash ./install_ga.sh 2>&1 | tee logs/install_ga.log +RUN bash -o xtrace -o errexit -o nounset -o pipefail ./install_ga.sh >logs/install_ga.log 2>&1 COPY install_petsc.sh ./ -RUN set -xeuo pipefail && bash ./install_petsc.sh 2>&1 | tee logs/install_petsc.log +RUN bash -o xtrace -o errexit -o nounset -o pipefail ./install_petsc.sh >logs/install_petsc.log 2>&1 RUN rm *.sh diff --git a/build/install_ga.sh b/build/install_ga.sh index 34dfa0e16..5b5d1721d 100644 --- a/build/install_ga.sh +++ b/build/install_ga.sh @@ -1,5 +1,8 @@ #! /bin/bash +# get the parent directory of this script +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + ga_version="${GA_VERSION:-5.8}" echo "--- Installing Global Arrays ${ga_version} ---" @@ -20,6 +23,10 @@ mv "ga-${ga_version}" ga pushd ga || exit +# load mpi module for RHEL +source "$script_dir/install_package_deps_lib.sh" +load_mpi_module + # build echo "Configuring Global Arrays" ./configure \ diff --git a/build/install_petsc.sh b/build/install_petsc.sh index e3abc3f01..4edddeba2 100644 --- a/build/install_petsc.sh +++ b/build/install_petsc.sh @@ -1,5 +1,8 @@ #! /bin/bash +# get the parent directory of this script +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + petsc_version="${PETSC_VERSION:-3.16.4}" echo "--- Installing PETSc ${petsc_version} ---" @@ -15,6 +18,10 @@ git checkout "tags/v${petsc_version}" -b "v${petsc_version}" export PETSC_DIR=${PWD} export PETSC_ARCH=build-dir +# load mpi module for RHEL +source "$script_dir/install_package_deps_lib.sh" +load_mpi_module + # install echo "Configuring PETSc" ./configure \ From 7bbb045ac65966b682aa6767cd93a46108e1ab2a Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 20 Feb 2024 12:26:20 -0600 Subject: [PATCH 189/195] Tee to build log with pipefail reporting nonzero exit from kaniko --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 08a55f0cc..a35240d58 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,6 +51,7 @@ check-container-needs-built:rocky: name: gcr.io/kaniko-project/executor:v1.9.0-debug entrypoint: [""] script: + - set -o errexit -o nounset -o pipefail - test -f container-needs-built || exit 0 - >- /kaniko/executor @@ -59,7 +60,7 @@ check-container-needs-built:rocky: --build-arg "http_proxy=${HTTP_PROXY}" --build-arg "https_proxy=${HTTPS_PROXY}" --dockerfile build/dockerfile - --destination "${CI_REGISTRY_IMAGE}:${ENV_IMAGE}" > build.log 2>&1 + --destination "${CI_REGISTRY_IMAGE}:${ENV_IMAGE}" 2>&1 | tee build.log build-container:ubuntu: extends: .build-container From 1a7f3aba86626e35bffb5f39c1c15b24851436ee Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 21 Feb 2024 09:41:19 -0600 Subject: [PATCH 190/195] Provide option to run jobs from web UI --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a35240d58..236311e40 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,7 +52,7 @@ check-container-needs-built:rocky: entrypoint: [""] script: - set -o errexit -o nounset -o pipefail - - test -f container-needs-built || exit 0 + - test -f container-needs-built || test "$FORCE_CONTAINER_BUILD" = "true" || exit 0 - >- /kaniko/executor --context build @@ -83,6 +83,7 @@ build-container:rocky: rules: - if: $CI_PIPELINE_SOURCE == "push" changes: ["build/**/*", "src/**/*", "python/**/*"] + - if: $CI_PIPELINE_SOURCE == "web" build-gridpack:ubuntu: extends: .build-gridpack @@ -108,6 +109,7 @@ build-gridpack:rocky: rules: - if: $CI_PIPELINE_SOURCE == "push" changes: ["build/**/*", "src/**/*", "python/**/*"] + - if: $CI_PIPELINE_SOURCE == "web" test-gridpack:ubuntu: extends: .test-gridpack From 99007f28e857f3b58c6835e9f58b8c990d504f33 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 21 Feb 2024 09:50:40 -0600 Subject: [PATCH 191/195] Remove nounset for kaniko job --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 236311e40..12486d5ce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,7 +51,7 @@ check-container-needs-built:rocky: name: gcr.io/kaniko-project/executor:v1.9.0-debug entrypoint: [""] script: - - set -o errexit -o nounset -o pipefail + - set -o errexit -o pipefail - test -f container-needs-built || test "$FORCE_CONTAINER_BUILD" = "true" || exit 0 - >- /kaniko/executor From ec163b11f0588c48b2778b12af34abfabad46cd2 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Thu, 22 Feb 2024 08:16:06 -0600 Subject: [PATCH 192/195] Add more specific file rules for check-container-definition-changed --- .gitlab-ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 12486d5ce..c1e607944 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,12 @@ check-container-definition-changed: - touch container-definition-changed rules: - if: $CI_PIPELINE_SOURCE == "push" - changes: ["build/**/*"] + changes: + - "build/dockerfile" + - "build/install_package_deps_lib.sh" + - "build/install_boost.sh" + - "build/install_ga.sh" + - "build/install_petsc.sh" .check-container-needs-built: stage: check-container From ad3edd05d8a8b4e644cec87b81e7af28213f6805 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Thu, 22 Feb 2024 08:21:34 -0600 Subject: [PATCH 193/195] Rename build to build-system, update readme contents --- .gitlab-ci.yml | 36 ++++++++++++------- build-system/README.md | 19 ++++++++++ {build => build-system}/container_lib.sh | 0 {build => build-system}/dockerfile | 0 {build => build-system}/install_boost.sh | 7 ++-- .../install_from_source_deps.sh | 0 {build => build-system}/install_ga.sh | 0 {build => build-system}/install_gridpack.sh | 0 .../install_package_deps_lib.sh | 0 {build => build-system}/install_petsc.sh | 0 build/README.md | 9 ----- 11 files changed, 46 insertions(+), 25 deletions(-) create mode 100644 build-system/README.md rename {build => build-system}/container_lib.sh (100%) rename {build => build-system}/dockerfile (100%) rename {build => build-system}/install_boost.sh (99%) rename {build => build-system}/install_from_source_deps.sh (100%) rename {build => build-system}/install_ga.sh (100%) rename {build => build-system}/install_gridpack.sh (100%) rename {build => build-system}/install_package_deps_lib.sh (100%) rename {build => build-system}/install_petsc.sh (100%) delete mode 100644 build/README.md diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c1e607944..89dd68d38 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,11 @@ default: untracked: true image: ubuntu:22.04 -stages: [check-container, build-container, build-gridpack, test-gridpack] +stages: +- check-container +- build-container +- build-gridpack +- test-gridpack variables: MAKE_JOBS: "2" @@ -21,11 +25,11 @@ check-container-definition-changed: rules: - if: $CI_PIPELINE_SOURCE == "push" changes: - - "build/dockerfile" - - "build/install_package_deps_lib.sh" - - "build/install_boost.sh" - - "build/install_ga.sh" - - "build/install_petsc.sh" + - "build-system/dockerfile" + - "build-system/install_package_deps_lib.sh" + - "build-system/install_boost.sh" + - "build-system/install_ga.sh" + - "build-system/install_petsc.sh" .check-container-needs-built: stage: check-container @@ -35,7 +39,7 @@ check-container-definition-changed: script: - set -o xtrace -o errexit -o nounset -o pipefail - apt-get update && apt-get install -y curl jq - - source ./build/container_lib.sh + - source ./build-system/container_lib.sh - check_container_needs_built "${ENV_IMAGE}" check-container-needs-built:ubuntu: @@ -60,11 +64,11 @@ check-container-needs-built:rocky: - test -f container-needs-built || test "$FORCE_CONTAINER_BUILD" = "true" || exit 0 - >- /kaniko/executor - --context build + --context build-system --build-arg "BASE_IMAGE=${BASE_IMAGE}" --build-arg "http_proxy=${HTTP_PROXY}" --build-arg "https_proxy=${HTTPS_PROXY}" - --dockerfile build/dockerfile + --dockerfile build-system/dockerfile --destination "${CI_REGISTRY_IMAGE}:${ENV_IMAGE}" 2>&1 | tee build.log build-container:ubuntu: @@ -84,10 +88,13 @@ build-container:rocky: .build-gridpack: stage: build-gridpack script: - - /bin/bash ./build/install_gridpack.sh + - /bin/bash ./build-system/install_gridpack.sh rules: - if: $CI_PIPELINE_SOURCE == "push" - changes: ["build/**/*", "src/**/*", "python/**/*"] + changes: + - "build-system/**/*" + - "src/**/*" + - "python/**/*" - if: $CI_PIPELINE_SOURCE == "web" build-gridpack:ubuntu: @@ -104,7 +111,7 @@ build-gridpack:rocky: stage: test-gridpack script: # load mpi module if on RHEL - - source ./build/install_package_deps_lib.sh + - source ./build-system/install_package_deps_lib.sh - load_mpi_module # run tests @@ -113,7 +120,10 @@ build-gridpack:rocky: paths: [src/build/Testing/Temporary] rules: - if: $CI_PIPELINE_SOURCE == "push" - changes: ["build/**/*", "src/**/*", "python/**/*"] + changes: + - "build-system/**/*" + - "src/**/*" + - "python/**/*" - if: $CI_PIPELINE_SOURCE == "web" test-gridpack:ubuntu: diff --git a/build-system/README.md b/build-system/README.md new file mode 100644 index 000000000..f73c9986f --- /dev/null +++ b/build-system/README.md @@ -0,0 +1,19 @@ +The shell scripts in this directory are used to automate the installation of GridPACK on Debian/RHEL based systems. + +# GitLab Mirror + +A [PNNL GitLab instance](https://devops.pnnl.gov/gridpack-code/GridPACK) hosting a [pull-mirror](https://docs.gitlab.com/ee/user/project/repository/mirror/pull.html) of this [GitHub repo](https://github.com/GridOPTICS/GridPACK) uses these scripts to execute tests whenever commits are pushed to GitHub. GitLab polls the GitHub repo to check for new commits every 30 minutes or every 5 minutes when triggered by the GitLab UI/API. An [integration](https://docs.gitlab.com/ee/ci/ci_cd_for_external_repos/github_integration.html) was configured to report the result back to GitHub for any commit associated with a build ["pipeline"](https://docs.gitlab.com/ee/ci/pipelines/). + +# GitLab Jobs + +The test automation is defined by the following set of ["jobs"](https://docs.gitlab.com/ee/ci/jobs/) in `/.gitlab-ci.yml`, which may be added to a pipeline by GitLab when a build is triggered: + +1. `check-container-definition-changed`: Creates a file called `container-definition-changed` if the scripts or `dockerfile` used to build the container image have been modified. The file is passed to subsequent jobs as an ["artifact"](https://docs.gitlab.com/ee/ci/jobs/job_artifacts.html). +2. `.check-container-needs-built`: A template job for multiple distributions. The leading `.` on this job name tells GitLab it is not meant to be added to the pipeline. This job runs `check_container_needs_built`, a function in `container_lib.sh` which creates another signal file called `container-needs-built` if the definition has changed or the container image tag does not exist in the GitLab project [container registry](https://docs.gitlab.com/ee/user/packages/container_registry/). Additionally, if the container image does need to be rebuilt, this function deletes the image tag from the registry to avoid unintentional use by subsequent pipelines in the event that the container image build fails to complete successfully and overwrite the old image. +3. `.build-container`: A template job for multiple distributions. Uses [kaniko](https://github.com/GoogleContainerTools/kaniko) to build a container image based on a given distribution if the file `container-needs-built` is passed as an artifact from a prior job or an environment variable called `FORCE_CONTAINER_BUILD` is set to `true`. The environment variable is intended to be used when [manually triggering a pipeline from the GitLab UI](https://docs.gitlab.com/ee/ci/pipelines/#run-a-pipeline-manually). +4. `.build-gridpack`: A template job for multiple distributions. Builds GridPACK in a container using an image from the GitLab project container registry based on a given distribution whenever files in `/src`, `/python`, or `/build` are modified using `install_gridpack.sh`. All files created by the build are saved as artifacts. +5. `.test-gridpack`: A template job for multiple distributions. Tests GridPACK in a new container using the artifacts from the associated `.build-container` job and with the same container image. These tests could have been performed at the end of the `.build-container` job but were separated to provide a distinct status for each concern and to maintain organized logs/artifacts. + +# Container image + +`dockerfile` defines a container image used to create an environment with all necessary GridPACK dependencies, including those installed from packages and those installed from source. diff --git a/build/container_lib.sh b/build-system/container_lib.sh similarity index 100% rename from build/container_lib.sh rename to build-system/container_lib.sh diff --git a/build/dockerfile b/build-system/dockerfile similarity index 100% rename from build/dockerfile rename to build-system/dockerfile diff --git a/build/install_boost.sh b/build-system/install_boost.sh similarity index 99% rename from build/install_boost.sh rename to build-system/install_boost.sh index 0d9ae934e..8ae3190c0 100644 --- a/build/install_boost.sh +++ b/build-system/install_boost.sh @@ -3,9 +3,6 @@ # get the parent directory of this script script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -# load mpi module for RHEL -source "$script_dir/install_package_deps_lib.sh" -load_mpi_module boost_version="${BOOST_VERSION:-1.78.0}" @@ -30,6 +27,10 @@ mv "boost_${boost_version//./_}" boost pushd boost || exit +# load mpi module for RHEL +source "$script_dir/install_package_deps_lib.sh" +load_mpi_module + # bootstrap echo "Bootstrapping Boost" ./bootstrap.sh \ diff --git a/build/install_from_source_deps.sh b/build-system/install_from_source_deps.sh similarity index 100% rename from build/install_from_source_deps.sh rename to build-system/install_from_source_deps.sh diff --git a/build/install_ga.sh b/build-system/install_ga.sh similarity index 100% rename from build/install_ga.sh rename to build-system/install_ga.sh diff --git a/build/install_gridpack.sh b/build-system/install_gridpack.sh similarity index 100% rename from build/install_gridpack.sh rename to build-system/install_gridpack.sh diff --git a/build/install_package_deps_lib.sh b/build-system/install_package_deps_lib.sh similarity index 100% rename from build/install_package_deps_lib.sh rename to build-system/install_package_deps_lib.sh diff --git a/build/install_petsc.sh b/build-system/install_petsc.sh similarity index 100% rename from build/install_petsc.sh rename to build-system/install_petsc.sh diff --git a/build/README.md b/build/README.md deleted file mode 100644 index d061c64a5..000000000 --- a/build/README.md +++ /dev/null @@ -1,9 +0,0 @@ -The shell scripts here are used to automate installation of GridPACK on Debian/RHEL based systems. These are leveraged by a build pipeline running on a PNNL GitLab instance that hosts a mirror of this GitHub repo. The build pipeline is defined by `/.gitlab-ci.yml` and used to automate software testing. An integration was configured to report the build result back to this GitHub repo for any commit triggering a build. `dockerfile` defines a container image used to create an environment with all necessary GridPACK dependencies, including those installed from packages and those installed from source. - -The build pipeline can be broken down into these steps: - -1. Check if container image definition has changed -2. Check if container image exists in the PNNL GitLab registry -3. If definition changed or image not in registry, build/push the image -3. Install GridPACK to a container created with the latest image -4. Execute GridPACK tests From 3a910642f77a42c863b6f3b59f2ca1821bc4f00c Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Thu, 22 Feb 2024 08:28:37 -0600 Subject: [PATCH 194/195] Disable rockylinux jobs from automatically running for now --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 89dd68d38..e95e8e818 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,6 +51,7 @@ check-container-needs-built:rocky: extends: .check-container-needs-built variables: ENV_IMAGE: ${ROCKY_ENV_IMAGE} + when: manual # todo remove when ready to test with rockylinux # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy .build-container: @@ -84,6 +85,7 @@ build-container:rocky: variables: BASE_IMAGE: ${ROCKY_BASE_IMAGE} ENV_IMAGE: ${ROCKY_ENV_IMAGE} + when: manual # todo remove when ready to test with rockylinux .build-gridpack: stage: build-gridpack @@ -106,6 +108,7 @@ build-gridpack:rocky: extends: .build-gridpack image: ${CI_REGISTRY_IMAGE}:${ROCKY_ENV_IMAGE} needs: [build-container:rocky] + when: manual # todo remove when ready to test with rockylinux .test-gridpack: stage: test-gridpack @@ -135,3 +138,4 @@ test-gridpack:rocky: extends: .test-gridpack image: ${CI_REGISTRY_IMAGE}:${ROCKY_ENV_IMAGE} needs: [build-gridpack:rocky] + when: manual # todo remove when ready to test with rockylinux From 0e4f663e848a5a07426bdd4159e84f79e9d4abd3 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Thu, 22 Feb 2024 16:12:26 -0600 Subject: [PATCH 195/195] Update readme --- build-system/README.md | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/build-system/README.md b/build-system/README.md index f73c9986f..7228ce051 100644 --- a/build-system/README.md +++ b/build-system/README.md @@ -1,19 +1,25 @@ The shell scripts in this directory are used to automate the installation of GridPACK on Debian/RHEL based systems. -# GitLab Mirror +# GitLab Build Automation -A [PNNL GitLab instance](https://devops.pnnl.gov/gridpack-code/GridPACK) hosting a [pull-mirror](https://docs.gitlab.com/ee/user/project/repository/mirror/pull.html) of this [GitHub repo](https://github.com/GridOPTICS/GridPACK) uses these scripts to execute tests whenever commits are pushed to GitHub. GitLab polls the GitHub repo to check for new commits every 30 minutes or every 5 minutes when triggered by the GitLab UI/API. An [integration](https://docs.gitlab.com/ee/ci/ci_cd_for_external_repos/github_integration.html) was configured to report the result back to GitHub for any commit associated with a build ["pipeline"](https://docs.gitlab.com/ee/ci/pipelines/). +## Repository Mirror -# GitLab Jobs +A [PNNL GitLab instance](https://devops.pnnl.gov/gridpack-code/GridPACK) hosting a [pull-mirror](https://docs.gitlab.com/ee/user/project/repository/mirror/pull.html) of this [GitHub repo](https://github.com/GridOPTICS/GridPACK) uses these scripts to execute tests whenever commits are pushed to GitHub. GitLab polls the GitHub repo to check for new commits every 30 minutes or every 5 minutes when triggered by the GitLab UI/API. An [integration](https://docs.gitlab.com/ee/ci/ci_cd_for_external_repos/github_integration.html) is configured to report a pass/fail status back to GitHub for any commit associated with a build ["pipeline"](https://docs.gitlab.com/ee/ci/pipelines/). -The test automation is defined by the following set of ["jobs"](https://docs.gitlab.com/ee/ci/jobs/) in `/.gitlab-ci.yml`, which may be added to a pipeline by GitLab when a build is triggered: +## Build Jobs -1. `check-container-definition-changed`: Creates a file called `container-definition-changed` if the scripts or `dockerfile` used to build the container image have been modified. The file is passed to subsequent jobs as an ["artifact"](https://docs.gitlab.com/ee/ci/jobs/job_artifacts.html). -2. `.check-container-needs-built`: A template job for multiple distributions. The leading `.` on this job name tells GitLab it is not meant to be added to the pipeline. This job runs `check_container_needs_built`, a function in `container_lib.sh` which creates another signal file called `container-needs-built` if the definition has changed or the container image tag does not exist in the GitLab project [container registry](https://docs.gitlab.com/ee/user/packages/container_registry/). Additionally, if the container image does need to be rebuilt, this function deletes the image tag from the registry to avoid unintentional use by subsequent pipelines in the event that the container image build fails to complete successfully and overwrite the old image. -3. `.build-container`: A template job for multiple distributions. Uses [kaniko](https://github.com/GoogleContainerTools/kaniko) to build a container image based on a given distribution if the file `container-needs-built` is passed as an artifact from a prior job or an environment variable called `FORCE_CONTAINER_BUILD` is set to `true`. The environment variable is intended to be used when [manually triggering a pipeline from the GitLab UI](https://docs.gitlab.com/ee/ci/pipelines/#run-a-pipeline-manually). -4. `.build-gridpack`: A template job for multiple distributions. Builds GridPACK in a container using an image from the GitLab project container registry based on a given distribution whenever files in `/src`, `/python`, or `/build` are modified using `install_gridpack.sh`. All files created by the build are saved as artifacts. -5. `.test-gridpack`: A template job for multiple distributions. Tests GridPACK in a new container using the artifacts from the associated `.build-container` job and with the same container image. These tests could have been performed at the end of the `.build-container` job but were separated to provide a distinct status for each concern and to maintain organized logs/artifacts. +The build/test automation is defined by the following set of ["jobs"](https://docs.gitlab.com/ee/ci/jobs/) in `/.gitlab-ci.yml`, which are conditionally added to a pipeline by GitLab when a build is triggered: -# Container image +1. `check-container-definition-changed`: Creates a file called `container-definition-changed` if the scripts or `./dockerfile` used to build the container image have been modified. The file is passed to subsequent jobs as an ["artifact"](https://docs.gitlab.com/ee/ci/jobs/job_artifacts.html). +2. `.check-container-needs-built`: A template (note the leading `.`) to define jobs for multiple distributions via [extension](https://docs.gitlab.com/ee/ci/yaml/#extends). Runs `check_container_needs_built`, a function in `./container_lib.sh` which creates another file called `container-needs-built` if `container-definition-changed` exists or the container image tag does not exist in the GitLab project [container registry](https://docs.gitlab.com/ee/user/packages/container_registry/). Additionally, if the container image does need to be rebuilt, this function [deletes](https://docs.gitlab.com/ee/api/container_registry.html#delete-a-registry-repository-tag) the image tag from the registry to avoid unintentional use by subsequent pipelines in the event that the container image build fails and does not replace the old image. +3. `.build-container`: A template job for multiple distributions. Uses [kaniko](https://github.com/GoogleContainerTools/kaniko) to build a container image based on a given distribution if the file `container-needs-built` exists or an environment variable called `FORCE_CONTAINER_BUILD` is set to `true`. The environment variable is intended to be used when [manually triggering a pipeline from the GitLab UI](https://docs.gitlab.com/ee/ci/pipelines/#run-a-pipeline-manually). +4. `.build-gridpack`: A template job for multiple distributions. Builds GridPACK in a container using an image from the GitLab project container registry based on a given distribution whenever files in `/src`, `/python`, or `/build` are modified using `./install_gridpack.sh`. All files created by the build are saved as artifacts. +5. `.test-gridpack`: A template job for multiple distributions. Tests GridPACK in a new container using the artifacts from the associated `.build-container` job and with the same container image that was used in that job. These tests could have been performed at the end of the `.build-container` job but were separated to provide a distinct status for each of these concerns and to maintain organized logs/artifacts. -`dockerfile` defines a container image used to create an environment with all necessary GridPACK dependencies, including those installed from packages and those installed from source. +`check-container-definition-changed` and `.check-container-needs-built` are very slim, taking about a minute each to run. Considering the source code and GitLab runner resources as of writing, `.build_container` takes about 35 minutes, `.build-gridpack` about 25 minutes, and `.test-gridpack` about 15 minutes for the `ubuntu:22.04` base image. Note again that the `.build_container` job only needs to run if the container image definition has changed. This mainly means changes to package dependencies, from-source dependencies, or environment variables. + +## Variables + +From the [project settings](https://devops.pnnl.gov/gridpack-code/GridPACK/-/settings/ci_cd) in GitLab a few variables are defined that are injected into each build pipeline. The difference between this type of variable and those in `/.gitlab-ci.yml` is that they can contain secrets and do not require source code modification to change. `API_TOKEN` is a [project access token](https://devops.pnnl.gov/gridpack-code/GridPACK/-/settings/access_tokens) which allows jobs to interact with the container registry API. When PNNL's GitLab instance is upgraded to `v16.8`, this variable should no longer be necessary. At that point it can be removed and curl can use the `JOB-TOKEN: ${CI_JOB_TOKEN}` header instead. That is desirable because the token will not need to be rotated. + +Another set of variables are used to manage resources available to the [GitLab runner](https://docs.gitlab.com/runner/executors/kubernetes/) executing each job in Kubernetes: `KUBERNETES_CPU_LIMIT`, `KUBERNETES_CPU_REQUEST`, `KUBERNETES_MEMORY_LIMIT`, and `KUBERNETES_MEMORY_REQUEST`. These mainly need to be considered if tests are performed for multiple distributions because that makes jobs available for concurrent processing. In this situation active jobs must share the resources from the namespace quota. Currently the runner is configured to allow two concurrent jobs, but the performance of this hasn't been compared to a serial approach with full resources.