From e4702b72db242ace35c3a7e87d245bcd6f784743 Mon Sep 17 00:00:00 2001 From: David Manthey Date: Thu, 22 Jul 2021 14:24:28 -0400 Subject: [PATCH] Refactor docker-compose. This allows running girder_worker jobs at the expense of a more complicated deployment. The previous docker-compose used docker volumes for the database and assetstore. Because of docker permission weirdness, docker volumes don't work once the container is run with a specific user, but running with a specific user is needed for a docker to start another docker container, so the assetstore and database are now stored in subdirectories in the devops/annotation_tracker directory. Migrating a previous deployment would involve copying these files from the docker volume to these subdirectories and ensuring that all files are owned by the current user. --- Dockerfile | 68 +------- .../web_client/static/worker.js | 2 +- devops/README.rst | 2 +- .../annotation_tracker/assetstore/.gitignore | 2 + devops/annotation_tracker/db/.gitignore | 2 + devops/annotation_tracker/docker-compose.yml | 159 ++++++++++++++---- devops/annotation_tracker/logs/.gitignore | 2 + 7 files changed, 137 insertions(+), 100 deletions(-) create mode 100644 devops/annotation_tracker/assetstore/.gitignore create mode 100644 devops/annotation_tracker/db/.gitignore create mode 100644 devops/annotation_tracker/logs/.gitignore diff --git a/Dockerfile b/Dockerfile index 892ccda..2f2a2f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,52 +1,6 @@ -FROM ubuntu:18.04 -LABEL maintainer="Kitware, Inc. " +FROM dsarchive/dsa_common -# See logs faster; don't write pyc or pyo files -ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 - -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -qy tzdata && \ - apt-get install --no-install-recommends --yes \ - software-properties-common \ - gpg-agent \ - fonts-dejavu \ - libmagic-dev \ - git \ - libldap2-dev \ - libsasl2-dev \ - curl \ - ca-certificates \ - fuse \ - vim && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -RUN curl -LJ https://github.com/krallin/tini/releases/download/v0.19.0/tini -o /usr/bin/tini && \ - chmod +x /usr/bin/tini - -RUN add-apt-repository ppa:deadsnakes/ppa && \ - apt-get update && \ - apt-get install --no-install-recommends --yes \ - python3.9 \ - python3.9-distutils && \ - curl --silent https://bootstrap.pypa.io/get-pip.py -O && \ - python3.9 get-pip.py && \ - rm get-pip.py && \ - rm /usr/bin/python3 && \ - ln -s /usr/bin/python3.9 /usr/bin/python3 && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -RUN curl -sL https://deb.nodesource.com/setup_12.x | bash && \ - apt-get update && \ - apt-get install --no-install-recommends --yes \ - nodejs && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# add a directory for girder mount -RUN mkdir -p /fuse --mode=a+rwx - -RUN mkdir -p annotation-tracker && \ - mkdir -p /conf +RUN mkdir -p annotation-tracker WORKDIR annotation-tracker @@ -55,24 +9,10 @@ COPY . . # By using --no-cache-dir the Docker image is smaller RUN python3.9 -m pip install --pre --no-cache-dir \ # git+https://github.com/arclamp/annotation-tracker.git \ - . \ - # girder[mount] adds dependencies to show tiles from S3 assets \ - girder[mount] \ - # Add additional girder plugins here \ - # girder-homepage \ - # We use girder_client for provisioning \ - girder_client \ - # Use prebuilt wheels whenever possible \ - --find-links https://girder.github.io/large_image_wheels + . # Build the girder web client RUN girder build && \ # Git rid of unnecessary files to keep the docker image smaller \ - find /usr/local/lib/python3.9 -name node_modules -exec rm -rf {} \+ && \ + find /opt/venv/lib/python3.9 -name node_modules -exec rm -rf {} \+ && \ rm -rf /tmp/npm* - -COPY ./devops/annotation_tracker/girder.local.conf ./devops/annotation_tracker/provision.py /conf/ - -ENTRYPOINT ["/usr/bin/tini", "--"] - -CMD python3.9 /conf/provision.py && (girder mount /fuse || true) && girder serve diff --git a/annotation_tracker/web_client/static/worker.js b/annotation_tracker/web_client/static/worker.js index b780dbd..af46b7b 100644 --- a/annotation_tracker/web_client/static/worker.js +++ b/annotation_tracker/web_client/static/worker.js @@ -20,7 +20,7 @@ onmessage = function (evt) { sendLogs = function () { sendLogTimeout = null; const logs = logsToSend.slice(0, logsToSend.length); - if (!logs.length) { + if (!logs.length || !api) { return; } sendLogTimeout = 'sending'; diff --git a/devops/README.rst b/devops/README.rst index 5c98c52..4d32f91 100644 --- a/devops/README.rst +++ b/devops/README.rst @@ -16,6 +16,6 @@ Start To start the program, in the ``devops/annotation_tracker`` directory, type:: - docker-compose up + DSA_USER=${id -u):$(id -g) docker-compose up By default, it creates an ``admin`` user with a password of ``password``. Some sample files will be downloaded in the ``Sample Images`` collection. diff --git a/devops/annotation_tracker/assetstore/.gitignore b/devops/annotation_tracker/assetstore/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/devops/annotation_tracker/assetstore/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/devops/annotation_tracker/db/.gitignore b/devops/annotation_tracker/db/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/devops/annotation_tracker/db/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/devops/annotation_tracker/docker-compose.yml b/devops/annotation_tracker/docker-compose.yml index 2ebd91b..9b562f5 100644 --- a/devops/annotation_tracker/docker-compose.yml +++ b/devops/annotation_tracker/docker-compose.yml @@ -4,59 +4,150 @@ services: girder: image: dsarchive/annotation_tracker build: ../.. - # Set CURRENT_UID to your user id (e.g., `CURRENT_UID=$(id -u):$(id -g)`) - # so that local file assetstores and logs are owned by yourself. - # user: ${CURRENT_UID} + # Instead of privileged mode, fuse can use: + # devices: + # - /dev/fuse:/dev/fuse + # security_opt: + # - apparmor:unconfined + # cap_add: + # - SYS_ADMIN + # but these may be somewhat host specific, so we default to privileged. If + # the docker daemon is being run with --no-new-privileges, fuse may not + # work. + privileged: true + # Set DSA_USER to a user id that is part of the docker group (e.g., + # `DSA_USER=$(id -u):$(id -g)`). This makes files in assetstores and logs + # owned by that user and provides permissions to manage docker + environment: + DSA_USER: ${DSA_USER:-} restart: unless-stopped # Set DSA_PORT to expose the interface on another port (default 8080). ports: - "${DSA_PORT:-8080}:8080" - environment: - - GIRDER_CONFIG=/conf/girder.local.conf volumes: + # Needed to use slicer_cli_web to run docker containers + - /usr/bin/docker:/usr/bin/docker + - /var/run/docker.sock:/var/run/docker.sock # Default assetstore - - fsdata:/assetstore - - logs:/logs - # Change for local files: - # - ./assetstore:/assetstore - # - ./logs:/logs - # Location of girder.local.conf and provision.py; add to use local - # versions - # - .:/conf + - ./assetstore:/assetstore + # Location of girder.cfg + - ./girder.local.conf:/etc/girder.cfg + # Location of provision.py + - ./provision.py:/opt/digital_slide_archive/devops/dsa/provision.py + # Location to store logs + - ./logs:/logs + depends_on: - mongodb - memcached - # This is needed to allow fuse to run inside the container. fuse is only - # needed if files are on non-filesystem assetstores or multi-file tile - # sources are used (like mrxs). - privileged: true - # Fuse needs fewer permissions than priviledged mode, such as - # cap_add: - # - SYS_ADMIN - # security_opt: - # - apparmor:unconfined - # devices: - # - /dev/fuse:/dev/fuse - # but these may vary based on the host + - rabbitmq + # The command does: + # - Ensures that the main process runs as the DSA_USER and is part of both + # that group and the docker group. This is done by: + # - adding a user with the DSA_USER's id; this user is named ubuntu if it + # doesn't exist. + # - adds a group with the DSA_USER's group id. + # - adds the user to the user group. + # - adds a group with the docker group id. + # - adds the user to the docker group. + # - Run subsequent commands as the DSA_USER. This sets some paths based on + # what is expected in the Docker so that the current python environment + # and the devops/dsa/utils are available. + # - Provision the Girder instance. This sets values in the database, such + # as creating an admin user if there isn't one. See the provision.py + # script for the details. + # - If possible, set up a girder mount. This allows file-like access of + # girder resources. It requires the host to have fuse installed and + # the docker container to be run with enough permissions to use fuse. + # - Start the main girder process. + command: bash -c ' + if [[ -z "$DSA_USER" ]]; then echo "Set the DSA_USER before starting (e.g, DSA_USER=\$$(id -u):\$$(id -g) "; exit 1; fi; + adduser --uid $${DSA_USER%%:*} --disabled-password --gecos "" ubuntu 2>/dev/null; + addgroup --gid $${DSA_USER#*:} $$(id -ng $${DSA_USER#*:}) 2>/dev/null; + adduser $$(id -nu $${DSA_USER%%:*}) $$(getent group $${DSA_USER#*:} | cut "-d:" -f1) 2>/dev/null; + addgroup --gid $$(stat -c "%g" /var/run/docker.sock) docker 2>/dev/null; + adduser $$(id -nu $${DSA_USER%%:*}) $$(getent group $$(stat -c "%g" /var/run/docker.sock) | cut "-d:" -f1) 2>/dev/null; + su $$(id -nu $${DSA_USER%%:*}) -c " + PATH=\"/opt/digital_slide_archive/devops/dsa/utils:/opt/venv/bin:/.pyenv/bin:/.pyenv/shims:$PATH\"; + python /opt/digital_slide_archive/devops/dsa/provision.py && + (girder mount /fuse || true) && + girder serve --dev + "' mongodb: image: "mongo:latest" - # Set CURRENT_UID to your user id (e.g., `CURRENT_UID=$(id -u):$(id -g)`) - # so that local file database and logs are owned by yourself. - # user: ${CURRENT_UID} + # Set DSA_USER to your user id (e.g., `DSA_USER=$(id -u):$(id -g)`) + # so that database files are owned by yourself. + user: ${DSA_USER:-PLEASE SET DSA_USER} + # Set DSA_USER to your user id (e.g., `DSA_USER=$(id -u):$(id -g)`) + # so that database files are owned by yourself. + # user: ${DSA_USER:-PLEASE SET DSA_USER} restart: unless-stopped + # Using --nojournal means that changes can be lost between the last + # checkpoint and an unexpected shutdown, but can substantially reduce + # writes. command: --nojournal volumes: # Location to store database files - # Change for local files: - # - ./db:/data/db - # - ./logs:/var/log/mongodb - - dbdata:/data/db + - ./db:/data/db memcached: image: memcached command: -m 4096 --max-item-size 8M restart: unless-stopped + # Uncomment to allow access to memcached from outside of the docker network + # ports: + # - "11211" + logging: + options: + max-size: "10M" + max-file: "5" + rabbitmq: + image: "rabbitmq:latest" + restart: unless-stopped + # Uncomment to allow access to rabbitmq from outside of the docker network + # ports: + # - "5672" + logging: + options: + max-size: "10M" + max-file: "5" + worker: + image: dsarchive/dsa_common + build: ../.. + # Set DSA_USER to a user id that is part of the docker group (e.g., + # `DSA_USER=$(id -u):$(id -g)`). This provides permissions to manage + # docker + environment: + DSA_USER: ${DSA_USER:-} + DSA_WORKER_CONCURRENCY: ${DSA_WORKER_CONCURRENCY:-2} + TMPDIR: + restart: unless-stopped + volumes: + # Needed to use slicer_cli_web to run docker containers + - /usr/bin/docker:/usr/bin/docker + - /var/run/docker.sock:/var/run/docker.sock + # Needed to allow transferring data to slicer_cli_web docker containers + - ${TMPDIR:-/tmp}:${TMPDIR:-/tmp} + # Add additional mounts here to get access to existing files on your + # system if they have the same path as on the girder container. + depends_on: + - rabbitmq + # See the girder container for an explanation of most of this. + # The main command is to run girder_worker + command: bash -c ' + if [[ -z "$DSA_USER" ]]; then echo "Set the DSA_USER before starting (e.g, DSA_USER=\$$(id -u):\$$(id -g) "; exit 1; fi; + adduser --uid $${DSA_USER%%:*} --disabled-password --gecos "" ubuntu 2>/dev/null; + addgroup --gid $${DSA_USER#*:} $$(id -ng $${DSA_USER#*:}) 2>/dev/null; + adduser $$(id -nu $${DSA_USER%%:*}) $$(getent group $${DSA_USER#*:} | cut "-d:" -f1) 2>/dev/null; + addgroup --gid $$(stat -c "%g" /var/run/docker.sock) docker 2>/dev/null; + adduser $$(id -nu $${DSA_USER%%:*}) $$(getent group $$(stat -c "%g" /var/run/docker.sock) | cut "-d:" -f1) 2>/dev/null; + su $$(id -nu $${DSA_USER%%:*}) -c " + PATH=\"/opt/digital_slide_archive/devops/dsa/utils:/opt/venv/bin:/.pyenv/bin:/.pyenv/shims:$PATH\"; + DOCKER_CLIENT_TIMEOUT=86400 TMPDIR=${TMPDIR:-/tmp} GW_DIRECT_PATHS=true python -m girder_worker --concurrency=$${DSA_WORKER_CONCURRENCY:-2} -Ofair --prefetch-multiplier=1 + "' + logging: + options: + max-size: "10M" + max-file: "5" volumes: dbdata: - fsdata: - logs: diff --git a/devops/annotation_tracker/logs/.gitignore b/devops/annotation_tracker/logs/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/devops/annotation_tracker/logs/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore