From 99ca16c832653d2fad4ed94505e9d9e87e43e527 Mon Sep 17 00:00:00 2001 From: Alexander Richards Date: Thu, 18 Jul 2019 16:47:19 +0100 Subject: [PATCH 01/13] Add Docker image build files. This necessitated removing build/ from the .gitignore. modified: .gitignore new file: build/Dockerfile new file: build/build.sh new file: build/startup.sh --- .gitignore | 1 - build/Dockerfile | 35 +++++++++++++++++++++++++++++++++++ build/build.sh | 2 ++ build/startup.sh | 30 ++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 build/Dockerfile create mode 100755 build/build.sh create mode 100755 build/startup.sh diff --git a/.gitignore b/.gitignore index eaec4cc..a882870 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ __pycache__/ # Distribution / packaging .Python env/ -build/ develop-eggs/ dist/ downloads/ diff --git a/build/Dockerfile b/build/Dockerfile new file mode 100644 index 0000000..66b2b47 --- /dev/null +++ b/build/Dockerfile @@ -0,0 +1,35 @@ +# syntax = docker/dockerfile:1.0-experimental +FROM centos:7 +ARG dirac_version=v6r21p7 +ARG lcgtools_version=v14r1 +ARG python_version=27 +ARG productionsystem_version=master + +RUN yum install -y centos-release-scl +RUN yum install -y rh-python36 +RUN yum install -y wget git gcc +RUN yum clean all + +RUN mkdir -p /root/dirac_ui + +WORKDIR /root/dirac_ui + +RUN wget -np -O dirac-install https://raw.githubusercontent.com/DIRACGrid/DIRAC/integration/Core/scripts/dirac-install.py +RUN chmod u+x dirac-install +RUN ./dirac-install -r $dirac_version -i $python_version -g $lcgtools_version +RUN --mount=type=secret,id=proxy,dst=/tmp/x509up_u0 . /root/dirac_ui/bashrc && dirac-configure -F -S GridPP -C dips://dirac01.grid.hep.ph.ic.ac.uk:9135/Configuration/Server -I +RUN rm -f dirac-install + +WORKDIR /root + +RUN echo "virtualenv venv3" | scl enable rh-python36 - +RUN . venv3/bin/activate && pip install --upgrade pip setuptools wheel +RUN --mount=type=ssh . venv3/bin/activate && pip install productionsystem@git+git://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version +RUN . dirac_ui/bashrc && python -m pip install --upgrade pip setuptools wheel +RUN --mount=type=ssh . dirac_ui/bashrc && python -m pip install --upgrade productionsystem@git+git://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version + +COPY startup.sh /root/startup.sh + +CMD ["all"] + +ENTRYPOINT ["/root/startup.sh"] diff --git a/build/build.sh b/build/build.sh new file mode 100755 index 0000000..a94369a --- /dev/null +++ b/build/build.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +DOCKER_BUILDKIT=1 docker build --ssh default --no-cache --progress=plain --secret id=proxy,src=/tmp/x509up_u$ID -t alexanderrichards/productionsystem:latest . diff --git a/build/startup.sh b/build/startup.sh new file mode 100755 index 0000000..42d2cbe --- /dev/null +++ b/build/startup.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +function start_daemon { + case "$1" in + dirac) + echo "*** Starting DIRAC daemon. ***" + (. /root/dirac_ui/bashrc && dirac-daemon.py start) + ;; + monitoring) + echo "*** Starting monitoring daemon. ***" + (. /root/venv3/bin/activate && monitoring-daemon.py start) + ;; + webapp) + echo "*** Starting web app daemon. ***" + (. /root/venv3/bin/activate && webapp-daemon.py start) + ;; + all) + start_daemon dirac + start_daemon monitoring + start_daemon webapp + ;; + *) + echo "Unknown parameter: $1" + echo "Expected: {dirac|monitoring|webapp|all}" + exit 1 + esac +} + +start_daemon $1 +/bin/bash From 568895b30dd5574693c6c92bdde483d06a8d2b4b Mon Sep 17 00:00:00 2001 From: Alexander Richards Date: Mon, 9 Dec 2019 16:24:59 +0000 Subject: [PATCH 02/13] Updating for CentOS8. --- build/Dockerfile | 33 ++++++++++++++++++--------------- build/build.sh | 6 ++++-- build/startup.sh | 46 +++++++++++++++++++++++++++++++++++++++------- 3 files changed, 61 insertions(+), 24 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 66b2b47..4231ed9 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,32 +1,35 @@ # syntax = docker/dockerfile:1.0-experimental -FROM centos:7 -ARG dirac_version=v6r21p7 +from centos:8 +ARG dirac_version=v6r22p6 ARG lcgtools_version=v14r1 ARG python_version=27 ARG productionsystem_version=master -RUN yum install -y centos-release-scl -RUN yum install -y rh-python36 -RUN yum install -y wget git gcc -RUN yum clean all +RUN dnf swap -y coreutils-single coreutils # Fix for wrapper coreutils +RUN dnf install -y gcc git wget python2 python3 python3-devel +RUN alternatives --set python /usr/bin/python2 # Set unversioned python command to python2 for DIRAC scripts +RUN python3 -m pip install --upgrade pip setuptools wheel +RUN python3 -m pip install --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version +RUN dnf clean all +# DIRAC setup RUN mkdir -p /root/dirac_ui - WORKDIR /root/dirac_ui - RUN wget -np -O dirac-install https://raw.githubusercontent.com/DIRACGrid/DIRAC/integration/Core/scripts/dirac-install.py RUN chmod u+x dirac-install -RUN ./dirac-install -r $dirac_version -i $python_version -g $lcgtools_version +RUN ./dirac-install --dirac-os -r $dirac_version -i $python_version -g $lcgtools_version RUN --mount=type=secret,id=proxy,dst=/tmp/x509up_u0 . /root/dirac_ui/bashrc && dirac-configure -F -S GridPP -C dips://dirac01.grid.hep.ph.ic.ac.uk:9135/Configuration/Server -I +# These two lines are necessary as git commands broken in DIRACOS so need to check out first then pip install else could do commented out line below +RUN git clone -b $productionsystem_version https://github.com/alexanderrichards/ProductionSystem.git +RUN . /root/dirac_ui/bashrc && python -m pip install --upgrade ProductionSystem/ +#RUN . /root/dirac_ui/bashrc && python -m pip install -v --trusted-host github.com --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version RUN rm -f dirac-install -WORKDIR /root -RUN echo "virtualenv venv3" | scl enable rh-python36 - -RUN . venv3/bin/activate && pip install --upgrade pip setuptools wheel -RUN --mount=type=ssh . venv3/bin/activate && pip install productionsystem@git+git://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version -RUN . dirac_ui/bashrc && python -m pip install --upgrade pip setuptools wheel -RUN --mount=type=ssh . dirac_ui/bashrc && python -m pip install --upgrade productionsystem@git+git://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version +## setup cron jobs - maybe not needed if mounting from outside. + + +WORKDIR /root COPY startup.sh /root/startup.sh diff --git a/build/build.sh b/build/build.sh index a94369a..7758e90 100755 --- a/build/build.sh +++ b/build/build.sh @@ -1,2 +1,4 @@ -#!/usr/bin/env bash -DOCKER_BUILDKIT=1 docker build --ssh default --no-cache --progress=plain --secret id=proxy,src=/tmp/x509up_u$ID -t alexanderrichards/productionsystem:latest . +#!/bin/bash +(. ~/dirac_ui/bashrc && dirac-proxy-init -x) +#DOCKER_BUILDKIT=1 docker build --no-cache --progress=plain --secret id=proxy,src=/tmp/x509up_u`id -u` -t alexanderrichards/productionsystem:latest . +DOCKER_BUILDKIT=1 docker build --progress=plain --secret id=proxy,src=/tmp/x509up_u`id -u` -t alexanderrichards/productionsystem:latest . diff --git a/build/startup.sh b/build/startup.sh index 42d2cbe..1c51d7f 100755 --- a/build/startup.sh +++ b/build/startup.sh @@ -1,4 +1,32 @@ -#!/usr/bin/env bash +#!/bin/bash + +trap "stop_daemon $1 && exit 0" SIGTERM + +function stop_daemon { + case "$1" in + dirac) + echo "*** Stopping DIRAC daemon. ***" + (. /root/dirac_ui/bashrc && dirac-daemon.py stop) + ;; + monitoring) + echo "*** Stopping monitoring daemon. ***" + monitoring-daemon.py stop + ;; + webapp) + echo "*** Stopping web app daemon. ***" + webapp-daemon.py stop + ;; + all) + stop_daemon dirac + stop_daemon monitoring + stop_daemon webapp + ;; + *) + echo "Unknown parameter: $1" + echo "Expected: {dirac|monitoring|webapp|all}" + exit 1 + esac +} function start_daemon { case "$1" in @@ -8,16 +36,16 @@ function start_daemon { ;; monitoring) echo "*** Starting monitoring daemon. ***" - (. /root/venv3/bin/activate && monitoring-daemon.py start) + monitoring-daemon.py start ;; webapp) echo "*** Starting web app daemon. ***" - (. /root/venv3/bin/activate && webapp-daemon.py start) + webapp-daemon.py start ;; all) - start_daemon dirac - start_daemon monitoring - start_daemon webapp + start_daemon dirac + start_daemon monitoring + start_daemon webapp ;; *) echo "Unknown parameter: $1" @@ -27,4 +55,8 @@ function start_daemon { } start_daemon $1 -/bin/bash +# Allow time to receive and process SIGTERM handler +while true +do +sleep 5 +done From cad23bb691f1f417d7a95a55fc6a33cea03f9adc Mon Sep 17 00:00:00 2001 From: Alexander Richards Date: Tue, 10 Dec 2019 16:30:15 +0000 Subject: [PATCH 03/13] Adding cron job for updating users. --- build/Dockerfile | 4 ++-- build/startup.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 4231ed9..473ae71 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -6,7 +6,7 @@ ARG python_version=27 ARG productionsystem_version=master RUN dnf swap -y coreutils-single coreutils # Fix for wrapper coreutils -RUN dnf install -y gcc git wget python2 python3 python3-devel +RUN dnf install -y cronie gcc git wget python2 python3 python3-devel RUN alternatives --set python /usr/bin/python2 # Set unversioned python command to python2 for DIRAC scripts RUN python3 -m pip install --upgrade pip setuptools wheel RUN python3 -m pip install --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version @@ -27,7 +27,7 @@ RUN rm -f dirac-install ## setup cron jobs - maybe not needed if mounting from outside. - +RUN echo "@daily userdb-update.py &>> /root/log/userdb-update.log" | crontab WORKDIR /root diff --git a/build/startup.sh b/build/startup.sh index 1c51d7f..1ba1da9 100755 --- a/build/startup.sh +++ b/build/startup.sh @@ -54,6 +54,7 @@ function start_daemon { esac } +crond start_daemon $1 # Allow time to receive and process SIGTERM handler while true From 59d5895318dc8d632262439481fbe5e2df0d19b3 Mon Sep 17 00:00:00 2001 From: Alexander Richards Date: Thu, 27 Aug 2020 11:29:16 +0100 Subject: [PATCH 04/13] Move old Dockerfile --- build/{Dockerfile => Dockerfileold} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build/{Dockerfile => Dockerfileold} (100%) diff --git a/build/Dockerfile b/build/Dockerfileold similarity index 100% rename from build/Dockerfile rename to build/Dockerfileold From d07b6714afc6032b19aefa7223787189bb9b9f35 Mon Sep 17 00:00:00 2001 From: Alexander Richards Date: Thu, 27 Aug 2020 11:29:52 +0100 Subject: [PATCH 05/13] Add new dockerfile --- build/Dockerfile | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 build/Dockerfile diff --git a/build/Dockerfile b/build/Dockerfile new file mode 100644 index 0000000..8c2c477 --- /dev/null +++ b/build/Dockerfile @@ -0,0 +1,20 @@ +# syntax = docker/dockerfile:1.0-experimental +ARG dirac_version=v6r22p26 +FROM alexanderrichards/dirac_ui:${dirac_version} +ARG productionsystem_version=master + +RUN yum install -y python3 +RUN python3 -m pip install --upgrade pip setuptools wheel +RUN python3 -m pip install --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version +RUN yum clean all + +## setup cron jobs - maybe not needed if mounting from outside. +RUN echo "@daily userdb-update.py &>> /root/log/userdb-update.log" | crontab + +WORKDIR /root + +COPY startup.sh /root/startup.sh + +CMD ["all"] + +ENTRYPOINT ["/root/startup.sh"] From 7a7efe802f8eeb4b06a61e54686f7962eaf7e72d Mon Sep 17 00:00:00 2001 From: Alexander Richards Date: Thu, 27 Aug 2020 15:45:00 +0100 Subject: [PATCH 06/13] Add git to Dockerfile --- build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index 8c2c477..62b8a58 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -3,7 +3,7 @@ ARG dirac_version=v6r22p26 FROM alexanderrichards/dirac_ui:${dirac_version} ARG productionsystem_version=master -RUN yum install -y python3 +RUN yum install -y git python3 RUN python3 -m pip install --upgrade pip setuptools wheel RUN python3 -m pip install --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version RUN yum clean all From c4a62ff322997a24bfe8aaa6712743f73959eb84 Mon Sep 17 00:00:00 2001 From: Alexander Richards Date: Thu, 27 Aug 2020 16:00:16 +0100 Subject: [PATCH 07/13] Add gcc to Dockerfile --- build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index 62b8a58..5891e24 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -3,7 +3,7 @@ ARG dirac_version=v6r22p26 FROM alexanderrichards/dirac_ui:${dirac_version} ARG productionsystem_version=master -RUN yum install -y git python3 +RUN yum install -y git gcc python3 RUN python3 -m pip install --upgrade pip setuptools wheel RUN python3 -m pip install --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version RUN yum clean all From 07dc9a8a48d1da5e528dc1daeb4fc8207288f629 Mon Sep 17 00:00:00 2001 From: Alexander Richards Date: Thu, 27 Aug 2020 16:25:39 +0100 Subject: [PATCH 08/13] Add Python devel to Dockerfile --- build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index 5891e24..762e4e8 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -3,7 +3,7 @@ ARG dirac_version=v6r22p26 FROM alexanderrichards/dirac_ui:${dirac_version} ARG productionsystem_version=master -RUN yum install -y git gcc python3 +RUN yum install -y git gcc python3 python3-devel RUN python3 -m pip install --upgrade pip setuptools wheel RUN python3 -m pip install --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version RUN yum clean all From 8d89ecaf42cdf27816e2de11776df2ddd45c50b8 Mon Sep 17 00:00:00 2001 From: Alexander Richards Date: Thu, 27 Aug 2020 16:48:07 +0100 Subject: [PATCH 09/13] Add cron to the install list --- build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index 762e4e8..04b0844 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -3,7 +3,7 @@ ARG dirac_version=v6r22p26 FROM alexanderrichards/dirac_ui:${dirac_version} ARG productionsystem_version=master -RUN yum install -y git gcc python3 python3-devel +RUN yum install -y git gcc cronie python3 python3-devel RUN python3 -m pip install --upgrade pip setuptools wheel RUN python3 -m pip install --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version RUN yum clean all From 0648fb94fac871540035230863efc65e796fc3a4 Mon Sep 17 00:00:00 2001 From: Alexander Richards Date: Fri, 28 Aug 2020 09:53:59 +0100 Subject: [PATCH 10/13] Explicitly setting python version --- scripts/dirac-daemon.py | 2 +- scripts/monitoring-daemon.py | 2 +- scripts/userdb-update.py | 2 +- scripts/webapp-daemon.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/dirac-daemon.py b/scripts/dirac-daemon.py index bbc5030..8d7d7c2 100755 --- a/scripts/dirac-daemon.py +++ b/scripts/dirac-daemon.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # pylint: disable=invalid-name """Dirac daemon run script.""" # Py2/3 compatibility layer diff --git a/scripts/monitoring-daemon.py b/scripts/monitoring-daemon.py index 0a13774..49c0bd8 100755 --- a/scripts/monitoring-daemon.py +++ b/scripts/monitoring-daemon.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # pylint: disable=invalid-name """ DB monitoring daemon. diff --git a/scripts/userdb-update.py b/scripts/userdb-update.py index 86b1254..08aa725 100755 --- a/scripts/userdb-update.py +++ b/scripts/userdb-update.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # pylint: disable=invalid-name """Script to read users info from VOMS and update locat SQL table.""" # Py2/3 compatibility layer diff --git a/scripts/webapp-daemon.py b/scripts/webapp-daemon.py index fec5e13..581ee41 100755 --- a/scripts/webapp-daemon.py +++ b/scripts/webapp-daemon.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # pylint: disable=invalid-name """Script to start the Production web server.""" # Py2/3 compatibility layer From edaebc8e17eea2d146156c342b2e3245f188ca2f Mon Sep 17 00:00:00 2001 From: Alexander Richards Date: Fri, 28 Aug 2020 10:28:50 +0100 Subject: [PATCH 11/13] installing production system in DIRAC environment --- build/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/build/Dockerfile b/build/Dockerfile index 04b0844..88b95a1 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -6,6 +6,7 @@ ARG productionsystem_version=master RUN yum install -y git gcc cronie python3 python3-devel RUN python3 -m pip install --upgrade pip setuptools wheel RUN python3 -m pip install --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version +RUN . /root/dirac_ui/bashrc && python -m pip install --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version RUN yum clean all ## setup cron jobs - maybe not needed if mounting from outside. From 5b9114ab9575e29041b12f4a4d7010afa7509a58 Mon Sep 17 00:00:00 2001 From: Alexander Richards Date: Fri, 28 Aug 2020 10:38:58 +0100 Subject: [PATCH 12/13] upgrade pip in DIRAC environment --- build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index 88b95a1..f521151 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -6,7 +6,7 @@ ARG productionsystem_version=master RUN yum install -y git gcc cronie python3 python3-devel RUN python3 -m pip install --upgrade pip setuptools wheel RUN python3 -m pip install --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version -RUN . /root/dirac_ui/bashrc && python -m pip install --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version +RUN . /root/dirac_ui/bashrc && python -m pip install --upgrade pip setuptools wheel && python -m pip install --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version RUN yum clean all ## setup cron jobs - maybe not needed if mounting from outside. From d33f0ac434dd9d4ca4548bb5b14067edeaba7888 Mon Sep 17 00:00:00 2001 From: Alexander Richards Date: Mon, 11 Jan 2021 14:13:18 +0000 Subject: [PATCH 13/13] something to check --- build/Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index f521151..33aa28b 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -3,7 +3,11 @@ ARG dirac_version=v6r22p26 FROM alexanderrichards/dirac_ui:${dirac_version} ARG productionsystem_version=master -RUN yum install -y git gcc cronie python3 python3-devel +RUN yum install -y git gcc cronie python3 python3-devel yum-priorities +RUN yum install -y http://repository.egi.eu/sw/production/umd/4/centos7/x86_64/updates/umd-release-4.1.3-1.el7.centos.noarch.rpm +RUN yum install -y ca-policy-egi-core fetch-crl +RUN systemctl enable fetch-crl-cron; systemctl start fetch-crl-cron +RUN fetch-crl -v RUN python3 -m pip install --upgrade pip setuptools wheel RUN python3 -m pip install --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version RUN . /root/dirac_ui/bashrc && python -m pip install --upgrade pip setuptools wheel && python -m pip install --upgrade productionsystem@git+https://github.com/alexanderrichards/ProductionSystem.git@$productionsystem_version @@ -11,7 +15,7 @@ RUN yum clean all ## setup cron jobs - maybe not needed if mounting from outside. RUN echo "@daily userdb-update.py &>> /root/log/userdb-update.log" | crontab - +RUN echo -e "[userdb]\ntrusted_cas='/etc/grid-security/certificates'" > /root/.config/productionsystem/productionsystem.conf WORKDIR /root COPY startup.sh /root/startup.sh