From 6fef38ffd23eb5e771f1413bf8d9354dcbce0975 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Wed, 30 Jul 2025 14:29:37 -0400 Subject: [PATCH 01/26] reorganize folders for gpu jobs --- .../{ => gpu-background-removal}/Dockerfile | 0 .../config/promtail.yaml | 0 .../gpu-paco-classifier/Dockerfile | 189 ++++++++++++++++++ .../gpu-paco-classifier/config/promtail.yaml | 19 ++ .../gpu-sae-binarization/Dockerfile | 189 ++++++++++++++++++ .../gpu-sae-binarization/config/promtail.yaml | 19 ++ .../gpu-workers/gpu-text-alignment/Dockerfile | 189 ++++++++++++++++++ .../gpu-text-alignment/config/promtail.yaml | 19 ++ 8 files changed, 624 insertions(+) rename celery/gpu-workers/{ => gpu-background-removal}/Dockerfile (100%) rename celery/gpu-workers/{ => gpu-background-removal}/config/promtail.yaml (100%) create mode 100644 celery/gpu-workers/gpu-paco-classifier/Dockerfile create mode 100644 celery/gpu-workers/gpu-paco-classifier/config/promtail.yaml create mode 100644 celery/gpu-workers/gpu-sae-binarization/Dockerfile create mode 100644 celery/gpu-workers/gpu-sae-binarization/config/promtail.yaml create mode 100644 celery/gpu-workers/gpu-text-alignment/Dockerfile create mode 100644 celery/gpu-workers/gpu-text-alignment/config/promtail.yaml diff --git a/celery/gpu-workers/Dockerfile b/celery/gpu-workers/gpu-background-removal/Dockerfile similarity index 100% rename from celery/gpu-workers/Dockerfile rename to celery/gpu-workers/gpu-background-removal/Dockerfile diff --git a/celery/gpu-workers/config/promtail.yaml b/celery/gpu-workers/gpu-background-removal/config/promtail.yaml similarity index 100% rename from celery/gpu-workers/config/promtail.yaml rename to celery/gpu-workers/gpu-background-removal/config/promtail.yaml diff --git a/celery/gpu-workers/gpu-paco-classifier/Dockerfile b/celery/gpu-workers/gpu-paco-classifier/Dockerfile new file mode 100644 index 0000000..1674bb2 --- /dev/null +++ b/celery/gpu-workers/gpu-paco-classifier/Dockerfile @@ -0,0 +1,189 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# +# THIS IS A GENERATED DOCKERFILE. +# +# This file was assembled from multiple pieces, whose use is documented +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. + +ARG UBUNTU_VERSION=18.04 +ARG CUDA=11.2 +FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}.2-base-ubuntu${UBUNTU_VERSION} AS base +# ARCH and CUDA are specified again because the FROM directive resets ARGs +# (but their default value is retained if set previously) +ARG ARCH +ARG CUDA +ARG CUDNN=8.1.0.77-1 +ARG CUDNN_MAJOR_VERSION=8 +ARG LIB_DIR_PREFIX=x86_64 +ARG LIBNVINFER=7.2.2-1 +ARG LIBNVINFER_MAJOR_VERSION=7 + +# The following two arguments are rodan-specific +ARG BRANCHES +ARG VERSION + +# Needed for string substitution +SHELL ["/bin/bash", "-c"] +# Pick up some TF dependencies +#RUN apt-get update + +#RUN rm -rf /etc/apt/sources.list.d/cuda.list + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + cuda-command-line-tools-${CUDA/./-} \ + libcublas-${CUDA/./-} \ + cuda-nvrtc-${CUDA/./-} \ + libcufft-${CUDA/./-} \ + libcurand-${CUDA/./-} \ + libcusolver-${CUDA/./-} \ + libcusparse-${CUDA/./-} \ + curl \ + libcudnn8=${CUDNN}+cuda${CUDA} \ + libfreetype6-dev \ + libhdf5-serial-dev \ + libzmq3-dev \ + pkg-config \ + software-properties-common \ + unzip \ + wget +# added wget + +# Install TensorRT if not building for PowerPC +# NOTE: libnvinfer uses cuda11.1 versions +# RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \ +# apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.1 \ +# libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.1 \ +# && apt-get clean \ +# && rm -rf /var/lib/apt/lists/*; } + +# For CUDA profiling, TensorFlow requires CUPTI. +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/cuda-11.1/lib64:$LD_LIBRARY_PATH + +# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure +# dynamic linker run-time bindings +RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \ + && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \ + && ldconfig + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +# This section differs from the default tensorflow2.5.1 Dockerfile, because we specifically add python 3.7; +ARG PYTHON=python3.7 +ARG TF_PACKAGE=tensorflow +ARG TF_PACKAGE_VERSION=2.5.1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.7 \ + python3-pip \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* \ + && wget https://bootstrap.pypa.io/pip/3.7/get-pip.py \ + && ${PYTHON} get-pip.py \ + && ln -sf /usr/bin/${PYTHON} /usr/local/bin/python3 \ + && ln -sf /usr/local/bin/pip /usr/local/bin/pip3 \ + && pip3 --no-cache-dir install --upgrade pip setuptools==57.0.0 \ + # Some TF tools expect a "python" binary + && ln -s $(which python3) /usr/local/bin/python \ + && python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}} + +# RUN ln -s $(which python3) /usr/local/bin/python + +# Options: +# tensorflow +# tensorflow-gpu +# tf-nightly +# tf-nightly-gpu +# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version. +# Installs the latest version by default. + +# COPY bashrc /etc/bash.bashrc +# RUN chmod a+rwx /etc/bash.bashrc + +# This ends the material obtained from TensorFlow's dockerfile. the remainder is rodan-docker-specific setup. + +# FROM base +RUN set -e \ + && apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get install -yqq \ + git \ + # Python lxml dependencies + python3.7-dev \ + python3-opencv \ + libxml2-dev \ + libxslt1-dev \ + zlib1g-dev \ + lib32ncurses5-dev \ + # Psycopg2 dependencies + libpq-dev \ + # For resource identification + libmagic-dev \ + unzip \ + # Remove when done + vim + +RUN rm -rf /var/lib/apt/lists/* + +# Bandaid fix for the cannot import name '_registerMatType' from 'cv2.cv2' issue +#RUN pip uninstall opencv-python-headless && pip install opencv-python-headless==4.1.2.30 + +# Install GPU Rodan Jobs +COPY ../../scripts/install_gpu_rodan_jobs /opt/ + +# Install Rodan +# Runs on both Rodan service, and Rodan-Celery +COPY ../../scripts/entrypoint /opt/ +COPY ../../scripts/start-celery /run/ +COPY ../../scripts/wait-for-app /run/ + +# Copying rodan core from build context into container +# Rodan folder MUST be uppercase, otherwise many unittests fail. +COPY ../../backend/django /code/Rodan + +# necessary for scikit-image > 0.17, or else it will try to make a cache directory +# in a place where the www-data user does not have permissions to do so +ENV SKIMAGE_DATADIR "/tmp/.skimage_cache" + + +RUN set -x \ + # Create Folders + && mkdir -p /code/jobs \ + # Install GPU Jobs + && chmod +x /opt/install_gpu_rodan_jobs \ + && /opt/install_gpu_rodan_jobs \ + # Install Rodan + && sed -i "s/lxml/#lxml/g" /code/Rodan/requirements.txt \ + # && sed -i "s/pybagit==1.5.0/-e git:\/\/github.com\/deepio\/pybagit.git@a27c9e0fc3bdf99dab8bd327f3ce9ea884abd6b4#egg=pybagit/g" /code/Rodan/requirements.txt \ + # Add Entrypoints + && sed -i 's/\r//' /opt/entrypoint \ + && chmod +x /opt/entrypoint \ + # Add Celery script + && chmod +x /run/start-celery \ + # Change the concurency for gpu jobs because Calvo is very expensive + && sed -i "s/=10/=1/g" /run/start-celery \ + # Script to wait for postgres and redis to be running before attempting to connect to them. + && chmod +x /run/wait-for-app + +# Install Rodan. +RUN pip3 install -r /code/Rodan/requirements.txt + +RUN pip3 uninstall -y opencv-python opencv-python-headless +RUN pip3 install opencv-python-headless==4.1.2.30 + +ENTRYPOINT ["/opt/entrypoint"] diff --git a/celery/gpu-workers/gpu-paco-classifier/config/promtail.yaml b/celery/gpu-workers/gpu-paco-classifier/config/promtail.yaml new file mode 100644 index 0000000..6fd7300 --- /dev/null +++ b/celery/gpu-workers/gpu-paco-classifier/config/promtail.yaml @@ -0,0 +1,19 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://192.168.17.170:3100/loki/api/v1/push + +scrape_configs: +- job_name: rodan_celery + static_configs: + - targets: + - localhost + labels: + job: varlogs + host: SERVER_HOST + __path__: /code/Rodan/*.log \ No newline at end of file diff --git a/celery/gpu-workers/gpu-sae-binarization/Dockerfile b/celery/gpu-workers/gpu-sae-binarization/Dockerfile new file mode 100644 index 0000000..1674bb2 --- /dev/null +++ b/celery/gpu-workers/gpu-sae-binarization/Dockerfile @@ -0,0 +1,189 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# +# THIS IS A GENERATED DOCKERFILE. +# +# This file was assembled from multiple pieces, whose use is documented +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. + +ARG UBUNTU_VERSION=18.04 +ARG CUDA=11.2 +FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}.2-base-ubuntu${UBUNTU_VERSION} AS base +# ARCH and CUDA are specified again because the FROM directive resets ARGs +# (but their default value is retained if set previously) +ARG ARCH +ARG CUDA +ARG CUDNN=8.1.0.77-1 +ARG CUDNN_MAJOR_VERSION=8 +ARG LIB_DIR_PREFIX=x86_64 +ARG LIBNVINFER=7.2.2-1 +ARG LIBNVINFER_MAJOR_VERSION=7 + +# The following two arguments are rodan-specific +ARG BRANCHES +ARG VERSION + +# Needed for string substitution +SHELL ["/bin/bash", "-c"] +# Pick up some TF dependencies +#RUN apt-get update + +#RUN rm -rf /etc/apt/sources.list.d/cuda.list + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + cuda-command-line-tools-${CUDA/./-} \ + libcublas-${CUDA/./-} \ + cuda-nvrtc-${CUDA/./-} \ + libcufft-${CUDA/./-} \ + libcurand-${CUDA/./-} \ + libcusolver-${CUDA/./-} \ + libcusparse-${CUDA/./-} \ + curl \ + libcudnn8=${CUDNN}+cuda${CUDA} \ + libfreetype6-dev \ + libhdf5-serial-dev \ + libzmq3-dev \ + pkg-config \ + software-properties-common \ + unzip \ + wget +# added wget + +# Install TensorRT if not building for PowerPC +# NOTE: libnvinfer uses cuda11.1 versions +# RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \ +# apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.1 \ +# libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.1 \ +# && apt-get clean \ +# && rm -rf /var/lib/apt/lists/*; } + +# For CUDA profiling, TensorFlow requires CUPTI. +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/cuda-11.1/lib64:$LD_LIBRARY_PATH + +# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure +# dynamic linker run-time bindings +RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \ + && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \ + && ldconfig + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +# This section differs from the default tensorflow2.5.1 Dockerfile, because we specifically add python 3.7; +ARG PYTHON=python3.7 +ARG TF_PACKAGE=tensorflow +ARG TF_PACKAGE_VERSION=2.5.1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.7 \ + python3-pip \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* \ + && wget https://bootstrap.pypa.io/pip/3.7/get-pip.py \ + && ${PYTHON} get-pip.py \ + && ln -sf /usr/bin/${PYTHON} /usr/local/bin/python3 \ + && ln -sf /usr/local/bin/pip /usr/local/bin/pip3 \ + && pip3 --no-cache-dir install --upgrade pip setuptools==57.0.0 \ + # Some TF tools expect a "python" binary + && ln -s $(which python3) /usr/local/bin/python \ + && python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}} + +# RUN ln -s $(which python3) /usr/local/bin/python + +# Options: +# tensorflow +# tensorflow-gpu +# tf-nightly +# tf-nightly-gpu +# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version. +# Installs the latest version by default. + +# COPY bashrc /etc/bash.bashrc +# RUN chmod a+rwx /etc/bash.bashrc + +# This ends the material obtained from TensorFlow's dockerfile. the remainder is rodan-docker-specific setup. + +# FROM base +RUN set -e \ + && apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get install -yqq \ + git \ + # Python lxml dependencies + python3.7-dev \ + python3-opencv \ + libxml2-dev \ + libxslt1-dev \ + zlib1g-dev \ + lib32ncurses5-dev \ + # Psycopg2 dependencies + libpq-dev \ + # For resource identification + libmagic-dev \ + unzip \ + # Remove when done + vim + +RUN rm -rf /var/lib/apt/lists/* + +# Bandaid fix for the cannot import name '_registerMatType' from 'cv2.cv2' issue +#RUN pip uninstall opencv-python-headless && pip install opencv-python-headless==4.1.2.30 + +# Install GPU Rodan Jobs +COPY ../../scripts/install_gpu_rodan_jobs /opt/ + +# Install Rodan +# Runs on both Rodan service, and Rodan-Celery +COPY ../../scripts/entrypoint /opt/ +COPY ../../scripts/start-celery /run/ +COPY ../../scripts/wait-for-app /run/ + +# Copying rodan core from build context into container +# Rodan folder MUST be uppercase, otherwise many unittests fail. +COPY ../../backend/django /code/Rodan + +# necessary for scikit-image > 0.17, or else it will try to make a cache directory +# in a place where the www-data user does not have permissions to do so +ENV SKIMAGE_DATADIR "/tmp/.skimage_cache" + + +RUN set -x \ + # Create Folders + && mkdir -p /code/jobs \ + # Install GPU Jobs + && chmod +x /opt/install_gpu_rodan_jobs \ + && /opt/install_gpu_rodan_jobs \ + # Install Rodan + && sed -i "s/lxml/#lxml/g" /code/Rodan/requirements.txt \ + # && sed -i "s/pybagit==1.5.0/-e git:\/\/github.com\/deepio\/pybagit.git@a27c9e0fc3bdf99dab8bd327f3ce9ea884abd6b4#egg=pybagit/g" /code/Rodan/requirements.txt \ + # Add Entrypoints + && sed -i 's/\r//' /opt/entrypoint \ + && chmod +x /opt/entrypoint \ + # Add Celery script + && chmod +x /run/start-celery \ + # Change the concurency for gpu jobs because Calvo is very expensive + && sed -i "s/=10/=1/g" /run/start-celery \ + # Script to wait for postgres and redis to be running before attempting to connect to them. + && chmod +x /run/wait-for-app + +# Install Rodan. +RUN pip3 install -r /code/Rodan/requirements.txt + +RUN pip3 uninstall -y opencv-python opencv-python-headless +RUN pip3 install opencv-python-headless==4.1.2.30 + +ENTRYPOINT ["/opt/entrypoint"] diff --git a/celery/gpu-workers/gpu-sae-binarization/config/promtail.yaml b/celery/gpu-workers/gpu-sae-binarization/config/promtail.yaml new file mode 100644 index 0000000..6fd7300 --- /dev/null +++ b/celery/gpu-workers/gpu-sae-binarization/config/promtail.yaml @@ -0,0 +1,19 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://192.168.17.170:3100/loki/api/v1/push + +scrape_configs: +- job_name: rodan_celery + static_configs: + - targets: + - localhost + labels: + job: varlogs + host: SERVER_HOST + __path__: /code/Rodan/*.log \ No newline at end of file diff --git a/celery/gpu-workers/gpu-text-alignment/Dockerfile b/celery/gpu-workers/gpu-text-alignment/Dockerfile new file mode 100644 index 0000000..1674bb2 --- /dev/null +++ b/celery/gpu-workers/gpu-text-alignment/Dockerfile @@ -0,0 +1,189 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# +# THIS IS A GENERATED DOCKERFILE. +# +# This file was assembled from multiple pieces, whose use is documented +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. + +ARG UBUNTU_VERSION=18.04 +ARG CUDA=11.2 +FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}.2-base-ubuntu${UBUNTU_VERSION} AS base +# ARCH and CUDA are specified again because the FROM directive resets ARGs +# (but their default value is retained if set previously) +ARG ARCH +ARG CUDA +ARG CUDNN=8.1.0.77-1 +ARG CUDNN_MAJOR_VERSION=8 +ARG LIB_DIR_PREFIX=x86_64 +ARG LIBNVINFER=7.2.2-1 +ARG LIBNVINFER_MAJOR_VERSION=7 + +# The following two arguments are rodan-specific +ARG BRANCHES +ARG VERSION + +# Needed for string substitution +SHELL ["/bin/bash", "-c"] +# Pick up some TF dependencies +#RUN apt-get update + +#RUN rm -rf /etc/apt/sources.list.d/cuda.list + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + cuda-command-line-tools-${CUDA/./-} \ + libcublas-${CUDA/./-} \ + cuda-nvrtc-${CUDA/./-} \ + libcufft-${CUDA/./-} \ + libcurand-${CUDA/./-} \ + libcusolver-${CUDA/./-} \ + libcusparse-${CUDA/./-} \ + curl \ + libcudnn8=${CUDNN}+cuda${CUDA} \ + libfreetype6-dev \ + libhdf5-serial-dev \ + libzmq3-dev \ + pkg-config \ + software-properties-common \ + unzip \ + wget +# added wget + +# Install TensorRT if not building for PowerPC +# NOTE: libnvinfer uses cuda11.1 versions +# RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \ +# apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.1 \ +# libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.1 \ +# && apt-get clean \ +# && rm -rf /var/lib/apt/lists/*; } + +# For CUDA profiling, TensorFlow requires CUPTI. +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/cuda-11.1/lib64:$LD_LIBRARY_PATH + +# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure +# dynamic linker run-time bindings +RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \ + && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \ + && ldconfig + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +# This section differs from the default tensorflow2.5.1 Dockerfile, because we specifically add python 3.7; +ARG PYTHON=python3.7 +ARG TF_PACKAGE=tensorflow +ARG TF_PACKAGE_VERSION=2.5.1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.7 \ + python3-pip \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* \ + && wget https://bootstrap.pypa.io/pip/3.7/get-pip.py \ + && ${PYTHON} get-pip.py \ + && ln -sf /usr/bin/${PYTHON} /usr/local/bin/python3 \ + && ln -sf /usr/local/bin/pip /usr/local/bin/pip3 \ + && pip3 --no-cache-dir install --upgrade pip setuptools==57.0.0 \ + # Some TF tools expect a "python" binary + && ln -s $(which python3) /usr/local/bin/python \ + && python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}} + +# RUN ln -s $(which python3) /usr/local/bin/python + +# Options: +# tensorflow +# tensorflow-gpu +# tf-nightly +# tf-nightly-gpu +# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version. +# Installs the latest version by default. + +# COPY bashrc /etc/bash.bashrc +# RUN chmod a+rwx /etc/bash.bashrc + +# This ends the material obtained from TensorFlow's dockerfile. the remainder is rodan-docker-specific setup. + +# FROM base +RUN set -e \ + && apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get install -yqq \ + git \ + # Python lxml dependencies + python3.7-dev \ + python3-opencv \ + libxml2-dev \ + libxslt1-dev \ + zlib1g-dev \ + lib32ncurses5-dev \ + # Psycopg2 dependencies + libpq-dev \ + # For resource identification + libmagic-dev \ + unzip \ + # Remove when done + vim + +RUN rm -rf /var/lib/apt/lists/* + +# Bandaid fix for the cannot import name '_registerMatType' from 'cv2.cv2' issue +#RUN pip uninstall opencv-python-headless && pip install opencv-python-headless==4.1.2.30 + +# Install GPU Rodan Jobs +COPY ../../scripts/install_gpu_rodan_jobs /opt/ + +# Install Rodan +# Runs on both Rodan service, and Rodan-Celery +COPY ../../scripts/entrypoint /opt/ +COPY ../../scripts/start-celery /run/ +COPY ../../scripts/wait-for-app /run/ + +# Copying rodan core from build context into container +# Rodan folder MUST be uppercase, otherwise many unittests fail. +COPY ../../backend/django /code/Rodan + +# necessary for scikit-image > 0.17, or else it will try to make a cache directory +# in a place where the www-data user does not have permissions to do so +ENV SKIMAGE_DATADIR "/tmp/.skimage_cache" + + +RUN set -x \ + # Create Folders + && mkdir -p /code/jobs \ + # Install GPU Jobs + && chmod +x /opt/install_gpu_rodan_jobs \ + && /opt/install_gpu_rodan_jobs \ + # Install Rodan + && sed -i "s/lxml/#lxml/g" /code/Rodan/requirements.txt \ + # && sed -i "s/pybagit==1.5.0/-e git:\/\/github.com\/deepio\/pybagit.git@a27c9e0fc3bdf99dab8bd327f3ce9ea884abd6b4#egg=pybagit/g" /code/Rodan/requirements.txt \ + # Add Entrypoints + && sed -i 's/\r//' /opt/entrypoint \ + && chmod +x /opt/entrypoint \ + # Add Celery script + && chmod +x /run/start-celery \ + # Change the concurency for gpu jobs because Calvo is very expensive + && sed -i "s/=10/=1/g" /run/start-celery \ + # Script to wait for postgres and redis to be running before attempting to connect to them. + && chmod +x /run/wait-for-app + +# Install Rodan. +RUN pip3 install -r /code/Rodan/requirements.txt + +RUN pip3 uninstall -y opencv-python opencv-python-headless +RUN pip3 install opencv-python-headless==4.1.2.30 + +ENTRYPOINT ["/opt/entrypoint"] diff --git a/celery/gpu-workers/gpu-text-alignment/config/promtail.yaml b/celery/gpu-workers/gpu-text-alignment/config/promtail.yaml new file mode 100644 index 0000000..6fd7300 --- /dev/null +++ b/celery/gpu-workers/gpu-text-alignment/config/promtail.yaml @@ -0,0 +1,19 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://192.168.17.170:3100/loki/api/v1/push + +scrape_configs: +- job_name: rodan_celery + static_configs: + - targets: + - localhost + labels: + job: varlogs + host: SERVER_HOST + __path__: /code/Rodan/*.log \ No newline at end of file From ac24e18d27d318ec38f7988b3a9077ac1ea2d7ae Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Wed, 30 Jul 2025 14:43:53 -0400 Subject: [PATCH 02/26] modify install job script and make individual dockerfile --- .../gpu-background-removal/Dockerfile | 6 +-- .../gpu-paco-classifier/Dockerfile | 6 +-- .../gpu-sae-binarization/Dockerfile | 6 +-- .../gpu-workers/gpu-text-alignment/Dockerfile | 6 +-- scripts/.DS_Store | Bin 0 -> 6148 bytes .../install_background_removal | 34 +++++++++++++++++ .../install_gpu_jobs/install_paco_classifier | 32 ++++++++++++++++ .../install_gpu_jobs/install_sae_binarization | 35 ++++++++++++++++++ .../install_gpu_jobs/install_text_alignment | 28 ++++++++++++++ 9 files changed, 141 insertions(+), 12 deletions(-) create mode 100644 scripts/.DS_Store create mode 100644 scripts/install_gpu_jobs/install_background_removal create mode 100644 scripts/install_gpu_jobs/install_paco_classifier create mode 100644 scripts/install_gpu_jobs/install_sae_binarization create mode 100644 scripts/install_gpu_jobs/install_text_alignment diff --git a/celery/gpu-workers/gpu-background-removal/Dockerfile b/celery/gpu-workers/gpu-background-removal/Dockerfile index 1674bb2..6f87e60 100644 --- a/celery/gpu-workers/gpu-background-removal/Dockerfile +++ b/celery/gpu-workers/gpu-background-removal/Dockerfile @@ -144,7 +144,7 @@ RUN rm -rf /var/lib/apt/lists/* #RUN pip uninstall opencv-python-headless && pip install opencv-python-headless==4.1.2.30 # Install GPU Rodan Jobs -COPY ../../scripts/install_gpu_rodan_jobs /opt/ +COPY ../../scripts/install_gpu_jobs/install_background_removal /opt/ # Install Rodan # Runs on both Rodan service, and Rodan-Celery @@ -165,8 +165,8 @@ RUN set -x \ # Create Folders && mkdir -p /code/jobs \ # Install GPU Jobs - && chmod +x /opt/install_gpu_rodan_jobs \ - && /opt/install_gpu_rodan_jobs \ + && chmod +x /opt/install_background_removal \ + && /opt/install_background_removal \ # Install Rodan && sed -i "s/lxml/#lxml/g" /code/Rodan/requirements.txt \ # && sed -i "s/pybagit==1.5.0/-e git:\/\/github.com\/deepio\/pybagit.git@a27c9e0fc3bdf99dab8bd327f3ce9ea884abd6b4#egg=pybagit/g" /code/Rodan/requirements.txt \ diff --git a/celery/gpu-workers/gpu-paco-classifier/Dockerfile b/celery/gpu-workers/gpu-paco-classifier/Dockerfile index 1674bb2..90f7151 100644 --- a/celery/gpu-workers/gpu-paco-classifier/Dockerfile +++ b/celery/gpu-workers/gpu-paco-classifier/Dockerfile @@ -144,7 +144,7 @@ RUN rm -rf /var/lib/apt/lists/* #RUN pip uninstall opencv-python-headless && pip install opencv-python-headless==4.1.2.30 # Install GPU Rodan Jobs -COPY ../../scripts/install_gpu_rodan_jobs /opt/ +COPY ../../scripts/install_gpu_jobs/install_paco_classifier /opt/ # Install Rodan # Runs on both Rodan service, and Rodan-Celery @@ -165,8 +165,8 @@ RUN set -x \ # Create Folders && mkdir -p /code/jobs \ # Install GPU Jobs - && chmod +x /opt/install_gpu_rodan_jobs \ - && /opt/install_gpu_rodan_jobs \ + && chmod +x /opt/install_paco_classifier \ + && /opt/install_paco_classifier \ # Install Rodan && sed -i "s/lxml/#lxml/g" /code/Rodan/requirements.txt \ # && sed -i "s/pybagit==1.5.0/-e git:\/\/github.com\/deepio\/pybagit.git@a27c9e0fc3bdf99dab8bd327f3ce9ea884abd6b4#egg=pybagit/g" /code/Rodan/requirements.txt \ diff --git a/celery/gpu-workers/gpu-sae-binarization/Dockerfile b/celery/gpu-workers/gpu-sae-binarization/Dockerfile index 1674bb2..47280ac 100644 --- a/celery/gpu-workers/gpu-sae-binarization/Dockerfile +++ b/celery/gpu-workers/gpu-sae-binarization/Dockerfile @@ -144,7 +144,7 @@ RUN rm -rf /var/lib/apt/lists/* #RUN pip uninstall opencv-python-headless && pip install opencv-python-headless==4.1.2.30 # Install GPU Rodan Jobs -COPY ../../scripts/install_gpu_rodan_jobs /opt/ +COPY ../../scripts/install_gpu_jobs/install_sae_binarization /opt/ # Install Rodan # Runs on both Rodan service, and Rodan-Celery @@ -165,8 +165,8 @@ RUN set -x \ # Create Folders && mkdir -p /code/jobs \ # Install GPU Jobs - && chmod +x /opt/install_gpu_rodan_jobs \ - && /opt/install_gpu_rodan_jobs \ + && chmod +x /opt/install_sae_binarization \ + && /opt/install_sae_binarization \ # Install Rodan && sed -i "s/lxml/#lxml/g" /code/Rodan/requirements.txt \ # && sed -i "s/pybagit==1.5.0/-e git:\/\/github.com\/deepio\/pybagit.git@a27c9e0fc3bdf99dab8bd327f3ce9ea884abd6b4#egg=pybagit/g" /code/Rodan/requirements.txt \ diff --git a/celery/gpu-workers/gpu-text-alignment/Dockerfile b/celery/gpu-workers/gpu-text-alignment/Dockerfile index 1674bb2..ee77600 100644 --- a/celery/gpu-workers/gpu-text-alignment/Dockerfile +++ b/celery/gpu-workers/gpu-text-alignment/Dockerfile @@ -144,7 +144,7 @@ RUN rm -rf /var/lib/apt/lists/* #RUN pip uninstall opencv-python-headless && pip install opencv-python-headless==4.1.2.30 # Install GPU Rodan Jobs -COPY ../../scripts/install_gpu_rodan_jobs /opt/ +COPY ../../scripts/install_gpu_jobs/install_text_alignment /opt/ # Install Rodan # Runs on both Rodan service, and Rodan-Celery @@ -165,8 +165,8 @@ RUN set -x \ # Create Folders && mkdir -p /code/jobs \ # Install GPU Jobs - && chmod +x /opt/install_gpu_rodan_jobs \ - && /opt/install_gpu_rodan_jobs \ + && chmod +x /opt/install_text_alignment \ + && /opt/install_text_alignment \ # Install Rodan && sed -i "s/lxml/#lxml/g" /code/Rodan/requirements.txt \ # && sed -i "s/pybagit==1.5.0/-e git:\/\/github.com\/deepio\/pybagit.git@a27c9e0fc3bdf99dab8bd327f3ce9ea884abd6b4#egg=pybagit/g" /code/Rodan/requirements.txt \ diff --git a/scripts/.DS_Store b/scripts/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..b50faaa2ca8e17e98a55e0a29440888f57cf0cc3 GIT binary patch literal 6148 zcmeHK%}N6?5T4X(w-jLy3VI88E!f3>yezf8fGc`Xsk>~^)pb+4TYD&lz3U74B0i5Z zNh(SeFCtQA%H*3&W)kwHWHtaG+QZlbZ~?$UC9Kr3`9f%%bVe%HLn!ndA*A^0!VUCi zqS^5u8KARsA%P5XDB$Dy4e=cBKtTj~G|Kx?sv?YiiTY`h7p>N-s4P?$mugPUX*gHG ztr`VISPb)aIJltRsZwb)wZrHv>5aPX%8|;7Fv)rYosh&m47of{vRIAUYM8~j&JE0f zQ+MiJcYQqG+TCeNZ~I`7PTPzLgs{_mLQZCeT$_*96=E#715*$`@|3?9sSbA`4&rqCLM%c8RxMp3;RM5dUfjj~QSF=86GP?F5}RZpqfxnaxqHm8jRKBotQ~ md{03`mtu^iQoM$$1^to?MBid*5IrdTA)smCh8g%%2HpS|c2gDr literal 0 HcmV?d00001 diff --git a/scripts/install_gpu_jobs/install_background_removal b/scripts/install_gpu_jobs/install_background_removal new file mode 100644 index 0000000..1cd4899 --- /dev/null +++ b/scripts/install_gpu_jobs/install_background_removal @@ -0,0 +1,34 @@ +#!/bin/sh +set -o errexit # Exit immediately if a command exits with a non-zero status. +set -o nounset # Treat unset variables as an error when substituting. +set -o xtrace # Print commands and their arguments as they are executed. + +PIP=$(which pip3) || PIP=$(which pip) + +cd /code/Rodan/code/jobs + + +cat << EOF | python3 +new_backend = """{ + "image_dim_ordering": "tf", + "epsilon": 1e-07, + "floatx": "float32", + "backend": "tensorflow" +}""" +open("keras.json", "w").write(new_backend) +EOF + +# Install Background Removal +cd / +git clone -b v1.1.1 https://github.com/DDMAL/background_removal.git +mv background_removal .background_removal +cd .background_removal +which pip3 && $PIP install --no-cache-dir . + + + +cd /code/Rodan/rodan +sed -i 's/#gpu //g' /code/Rodan/rodan/settings.py + +# Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. +pip install opencv-python==4.6.0.66 diff --git a/scripts/install_gpu_jobs/install_paco_classifier b/scripts/install_gpu_jobs/install_paco_classifier new file mode 100644 index 0000000..cab30f8 --- /dev/null +++ b/scripts/install_gpu_jobs/install_paco_classifier @@ -0,0 +1,32 @@ +#!/bin/sh +set -o errexit # Exit immediately if a command exits with a non-zero status. +set -o nounset # Treat unset variables as an error when substituting. +set -o xtrace # Print commands and their arguments as they are executed. + +PIP=$(which pip3) || PIP=$(which pip) + +cd /code/Rodan/code/jobs + + +cat << EOF | python3 +new_backend = """{ + "image_dim_ordering": "tf", + "epsilon": 1e-07, + "floatx": "float32", + "backend": "tensorflow" +}""" +open("keras.json", "w").write(new_backend) +EOF + +# Install Paco classifier +cd / +git clone -b v2.0.4 --depth 1 https://github.com/DDMAL/Paco_classifier.git +mv Paco_classifier .Paco_classifier +cd .Paco_classifier +which pip3 && $PIP install --no-cache-dir . + +cd /code/Rodan/rodan +sed -i 's/#gpu //g' /code/Rodan/rodan/settings.py + +# Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. +pip install opencv-python==4.6.0.66 diff --git a/scripts/install_gpu_jobs/install_sae_binarization b/scripts/install_gpu_jobs/install_sae_binarization new file mode 100644 index 0000000..b392cb2 --- /dev/null +++ b/scripts/install_gpu_jobs/install_sae_binarization @@ -0,0 +1,35 @@ +#!/bin/sh +set -o errexit # Exit immediately if a command exits with a non-zero status. +set -o nounset # Treat unset variables as an error when substituting. +set -o xtrace # Print commands and their arguments as they are executed. + +PIP=$(which pip3) || PIP=$(which pip) + +cd /code/Rodan/code/jobs + + +cat << EOF | python3 +new_backend = """{ + "image_dim_ordering": "tf", + "epsilon": 1e-07, + "floatx": "float32", + "backend": "tensorflow" +}""" +open("keras.json", "w").write(new_backend) +EOF + + + +# Install SAE_binarization +cd / +git clone -b v1.1.1 https://github.com/DDMAL/SAE_binarization.git +mv SAE_binarization .SAE_binarization +cd .SAE_binarization +which pip3 && $PIP install --no-cache-dir . + + +cd /code/Rodan/rodan +sed -i 's/#gpu //g' /code/Rodan/rodan/settings.py + +# Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. +pip install opencv-python==4.6.0.66 diff --git a/scripts/install_gpu_jobs/install_text_alignment b/scripts/install_gpu_jobs/install_text_alignment new file mode 100644 index 0000000..b991599 --- /dev/null +++ b/scripts/install_gpu_jobs/install_text_alignment @@ -0,0 +1,28 @@ +#!/bin/sh +set -o errexit # Exit immediately if a command exits with a non-zero status. +set -o nounset # Treat unset variables as an error when substituting. +set -o xtrace # Print commands and their arguments as they are executed. + +PIP=$(which pip3) || PIP=$(which pip) + +cd /code/Rodan/code/jobs +# Install Text Alignment +$PIP install -r ./text_alignment/requirements.txt + +cat << EOF | python3 +new_backend = """{ + "image_dim_ordering": "tf", + "epsilon": 1e-07, + "floatx": "float32", + "backend": "tensorflow" +}""" +open("keras.json", "w").write(new_backend) +EOF + + + +cd /code/Rodan/rodan +sed -i 's/#gpu //g' /code/Rodan/rodan/settings.py + +# Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. +pip install opencv-python==4.6.0.66 From ce47305b121d77563e266a6c5047d62bb9d71b37 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Tue, 5 Aug 2025 18:20:15 -0400 Subject: [PATCH 03/26] use one base block for gpu worker and separate services --- docker-compose.yml | 88 +++++++++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 28 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 265e5db..0475aa8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,30 @@ volumes: rodan-resources: +x-celery-gpu-worker-base: &celery-gpu-worker-base + command: bash -c "tail -f /dev/null" + environment: + TZ: America/Toronto + SERVER_HOST: localhost + depends_on: + - backend-django + - taskqueue-rabbitmq + - database-postgres + - database-redis + - celery-controller + env_file: + - ./scripts/local.env + volumes: + - "rodan-resources:/rodan/data" + - "./backend/django/code:/code/Rodan" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu, compute, utility] + services: frontend-nginx: container_name: "frontend-nginx" @@ -115,36 +139,44 @@ services: - "rodan-resources:/rodan/data" - "./backend/django/code:/code/Rodan" - celery-gpu-workers: + celery-gpu-background-removal: + <<: *celery-gpu-worker-base build: - context: . - dockerfile: ./celery/gpu-workers/Dockerfile - command: bash -c "tail -f /dev/null" + context: ./celery/gpu-workers/gpu-background-removal + dockerfile: ./celery/gpu-workers/gpu-background-removal/Dockerfile environment: - TZ: America/Toronto - SERVER_HOST: localhost - # CELERY_JOB_QUEUE: GPU - depends_on: - - backend-django - - taskqueue-rabbitmq - - database-postgres - - database-redis - - celery-controller - env_file: - - ./scripts/local.env - volumes: - - "rodan-resources:/rodan/data" - - "./backend/django/code:/code/Rodan" - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: all - capabilities: [gpu, compute, utility] - # See more: - # - https://docs.docker.com/compose/how-tos/gpu-support/ - # - https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html + <<: *celery-gpu-worker-base.environment + CELERY_JOB_QUEUE: GPU + + celery-gpu-paco-classifier: + <<: *celery-gpu-worker-base + build: + context: ./celery/gpu-workers/gpu-paco-classifier + dockerfile: ./celery/gpu-workers/gpu-paco-classifier/Dockerfile + environment: + <<: *celery-gpu-worker-base.environment + CELERY_JOB_QUEUE: GPU + + celery-gpu-sae-binarization: + <<: *celery-gpu-worker-base + build: + context: ./celery/gpu-workers/gpu-sae-binarization + dockerfile: ./celery/gpu-workers/gpu-sae-binarization/Dockerfile + environment: + <<: *celery-gpu-worker-base.environment + CELERY_JOB_QUEUE: GPU + + celery-gpu-text-alignment: + <<: *celery-gpu-worker-base + build: + context: ./celery/gpu-workers/gpu-text-alignment + dockerfile: ./celery/gpu-workers/gpu-text-alignment/Dockerfile + environment: + <<: *celery-gpu-worker-base.environment + CELERY_JOB_QUEUE: GPU + # See more: + # - https://docs.docker.com/compose/how-tos/gpu-support/ + # - https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html # Database: PostgreSQL, Redit (messaging) database-postgres: From d059eeb17ca52bdf672b5910cf0125dd9d223dff Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Tue, 5 Aug 2025 18:24:07 -0400 Subject: [PATCH 04/26] add service profiles for gpu jobs --- .github/workflows/CI-celery-build.yml | 2 +- docker-compose.yml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI-celery-build.yml b/.github/workflows/CI-celery-build.yml index 6e276b1..aecf86f 100644 --- a/.github/workflows/CI-celery-build.yml +++ b/.github/workflows/CI-celery-build.yml @@ -49,5 +49,5 @@ jobs: with: version: latest - name: Build GPU-related celery workers - run: docker compose build celery-gpu-workers + run: docker compose --profile celery-gpu-workers build \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 0475aa8..d8dc860 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -141,6 +141,7 @@ services: celery-gpu-background-removal: <<: *celery-gpu-worker-base + profiles: ["celery-gpu-workers"] build: context: ./celery/gpu-workers/gpu-background-removal dockerfile: ./celery/gpu-workers/gpu-background-removal/Dockerfile @@ -150,6 +151,7 @@ services: celery-gpu-paco-classifier: <<: *celery-gpu-worker-base + profiles: ["celery-gpu-workers"] build: context: ./celery/gpu-workers/gpu-paco-classifier dockerfile: ./celery/gpu-workers/gpu-paco-classifier/Dockerfile @@ -159,6 +161,7 @@ services: celery-gpu-sae-binarization: <<: *celery-gpu-worker-base + profiles: ["celery-gpu-workers"] build: context: ./celery/gpu-workers/gpu-sae-binarization dockerfile: ./celery/gpu-workers/gpu-sae-binarization/Dockerfile @@ -168,6 +171,7 @@ services: celery-gpu-text-alignment: <<: *celery-gpu-worker-base + profiles: ["celery-gpu-workers"] build: context: ./celery/gpu-workers/gpu-text-alignment dockerfile: ./celery/gpu-workers/gpu-text-alignment/Dockerfile From fb3377d5bd4614682b6c9d990a16d80fc533058c Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Tue, 5 Aug 2025 18:27:12 -0400 Subject: [PATCH 05/26] remove incorrect environment lines --- docker-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d8dc860..ac18d2b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -146,7 +146,6 @@ services: context: ./celery/gpu-workers/gpu-background-removal dockerfile: ./celery/gpu-workers/gpu-background-removal/Dockerfile environment: - <<: *celery-gpu-worker-base.environment CELERY_JOB_QUEUE: GPU celery-gpu-paco-classifier: @@ -156,7 +155,6 @@ services: context: ./celery/gpu-workers/gpu-paco-classifier dockerfile: ./celery/gpu-workers/gpu-paco-classifier/Dockerfile environment: - <<: *celery-gpu-worker-base.environment CELERY_JOB_QUEUE: GPU celery-gpu-sae-binarization: @@ -166,7 +164,6 @@ services: context: ./celery/gpu-workers/gpu-sae-binarization dockerfile: ./celery/gpu-workers/gpu-sae-binarization/Dockerfile environment: - <<: *celery-gpu-worker-base.environment CELERY_JOB_QUEUE: GPU celery-gpu-text-alignment: @@ -176,7 +173,6 @@ services: context: ./celery/gpu-workers/gpu-text-alignment dockerfile: ./celery/gpu-workers/gpu-text-alignment/Dockerfile environment: - <<: *celery-gpu-worker-base.environment CELERY_JOB_QUEUE: GPU # See more: # - https://docs.docker.com/compose/how-tos/gpu-support/ From 391ff9213b46687542c48819aca4134c48db2344 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Tue, 5 Aug 2025 18:33:13 -0400 Subject: [PATCH 06/26] fix dockerfile path --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ac18d2b..a9d8c8c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -144,7 +144,7 @@ services: profiles: ["celery-gpu-workers"] build: context: ./celery/gpu-workers/gpu-background-removal - dockerfile: ./celery/gpu-workers/gpu-background-removal/Dockerfile + dockerfile: Dockerfile environment: CELERY_JOB_QUEUE: GPU @@ -153,7 +153,7 @@ services: profiles: ["celery-gpu-workers"] build: context: ./celery/gpu-workers/gpu-paco-classifier - dockerfile: ./celery/gpu-workers/gpu-paco-classifier/Dockerfile + dockerfile: Dockerfile environment: CELERY_JOB_QUEUE: GPU @@ -162,7 +162,7 @@ services: profiles: ["celery-gpu-workers"] build: context: ./celery/gpu-workers/gpu-sae-binarization - dockerfile: ./celery/gpu-workers/gpu-sae-binarization/Dockerfile + dockerfile: Dockerfile environment: CELERY_JOB_QUEUE: GPU @@ -171,7 +171,7 @@ services: profiles: ["celery-gpu-workers"] build: context: ./celery/gpu-workers/gpu-text-alignment - dockerfile: ./celery/gpu-workers/gpu-text-alignment/Dockerfile + dockerfile: Dockerfile environment: CELERY_JOB_QUEUE: GPU # See more: From 29a305cbd2ff38a391718bd843b0182c88044195 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Tue, 5 Aug 2025 18:40:05 -0400 Subject: [PATCH 07/26] fix dockerfile path --- docker-compose.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a9d8c8c..0827098 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -143,8 +143,8 @@ services: <<: *celery-gpu-worker-base profiles: ["celery-gpu-workers"] build: - context: ./celery/gpu-workers/gpu-background-removal - dockerfile: Dockerfile + context: . + dockerfile: ./celery/gpu-workers/gpu-background-removal/Dockerfile environment: CELERY_JOB_QUEUE: GPU @@ -152,8 +152,8 @@ services: <<: *celery-gpu-worker-base profiles: ["celery-gpu-workers"] build: - context: ./celery/gpu-workers/gpu-paco-classifier - dockerfile: Dockerfile + context: . + dockerfile: ./celery/gpu-workers/gpu-paco-classifier/Dockerfile environment: CELERY_JOB_QUEUE: GPU @@ -161,8 +161,8 @@ services: <<: *celery-gpu-worker-base profiles: ["celery-gpu-workers"] build: - context: ./celery/gpu-workers/gpu-sae-binarization - dockerfile: Dockerfile + context: . + dockerfile: ./celery/gpu-workers/gpu-sae-binarization/Dockerfile environment: CELERY_JOB_QUEUE: GPU @@ -170,8 +170,8 @@ services: <<: *celery-gpu-worker-base profiles: ["celery-gpu-workers"] build: - context: ./celery/gpu-workers/gpu-text-alignment - dockerfile: Dockerfile + context: . + dockerfile: ./celery/gpu-workers/gpu-text-alignment/Dockerfile environment: CELERY_JOB_QUEUE: GPU # See more: From f01e0adb4d3157852b7b6289407c05afab3a96ad Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Tue, 5 Aug 2025 18:45:02 -0400 Subject: [PATCH 08/26] fix script copy relative path --- celery/gpu-workers/gpu-background-removal/Dockerfile | 10 +++++----- celery/gpu-workers/gpu-paco-classifier/Dockerfile | 10 +++++----- celery/gpu-workers/gpu-sae-binarization/Dockerfile | 10 +++++----- celery/gpu-workers/gpu-text-alignment/Dockerfile | 10 +++++----- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/celery/gpu-workers/gpu-background-removal/Dockerfile b/celery/gpu-workers/gpu-background-removal/Dockerfile index 6f87e60..eb4109d 100644 --- a/celery/gpu-workers/gpu-background-removal/Dockerfile +++ b/celery/gpu-workers/gpu-background-removal/Dockerfile @@ -144,17 +144,17 @@ RUN rm -rf /var/lib/apt/lists/* #RUN pip uninstall opencv-python-headless && pip install opencv-python-headless==4.1.2.30 # Install GPU Rodan Jobs -COPY ../../scripts/install_gpu_jobs/install_background_removal /opt/ +COPY scripts/install_gpu_jobs/install_background_removal /opt/ # Install Rodan # Runs on both Rodan service, and Rodan-Celery -COPY ../../scripts/entrypoint /opt/ -COPY ../../scripts/start-celery /run/ -COPY ../../scripts/wait-for-app /run/ +COPY scripts/entrypoint /opt/ +COPY scripts/start-celery /run/ +COPY scripts/wait-for-app /run/ # Copying rodan core from build context into container # Rodan folder MUST be uppercase, otherwise many unittests fail. -COPY ../../backend/django /code/Rodan +COPY backend/django /code/Rodan # necessary for scikit-image > 0.17, or else it will try to make a cache directory # in a place where the www-data user does not have permissions to do so diff --git a/celery/gpu-workers/gpu-paco-classifier/Dockerfile b/celery/gpu-workers/gpu-paco-classifier/Dockerfile index 90f7151..87369c4 100644 --- a/celery/gpu-workers/gpu-paco-classifier/Dockerfile +++ b/celery/gpu-workers/gpu-paco-classifier/Dockerfile @@ -144,17 +144,17 @@ RUN rm -rf /var/lib/apt/lists/* #RUN pip uninstall opencv-python-headless && pip install opencv-python-headless==4.1.2.30 # Install GPU Rodan Jobs -COPY ../../scripts/install_gpu_jobs/install_paco_classifier /opt/ +COPY scripts/install_gpu_jobs/install_paco_classifier /opt/ # Install Rodan # Runs on both Rodan service, and Rodan-Celery -COPY ../../scripts/entrypoint /opt/ -COPY ../../scripts/start-celery /run/ -COPY ../../scripts/wait-for-app /run/ +COPY scripts/entrypoint /opt/ +COPY scripts/start-celery /run/ +COPY scripts/wait-for-app /run/ # Copying rodan core from build context into container # Rodan folder MUST be uppercase, otherwise many unittests fail. -COPY ../../backend/django /code/Rodan +COPY backend/django /code/Rodan # necessary for scikit-image > 0.17, or else it will try to make a cache directory # in a place where the www-data user does not have permissions to do so diff --git a/celery/gpu-workers/gpu-sae-binarization/Dockerfile b/celery/gpu-workers/gpu-sae-binarization/Dockerfile index 47280ac..9a3356b 100644 --- a/celery/gpu-workers/gpu-sae-binarization/Dockerfile +++ b/celery/gpu-workers/gpu-sae-binarization/Dockerfile @@ -144,17 +144,17 @@ RUN rm -rf /var/lib/apt/lists/* #RUN pip uninstall opencv-python-headless && pip install opencv-python-headless==4.1.2.30 # Install GPU Rodan Jobs -COPY ../../scripts/install_gpu_jobs/install_sae_binarization /opt/ +COPY scripts/install_gpu_jobs/install_sae_binarization /opt/ # Install Rodan # Runs on both Rodan service, and Rodan-Celery -COPY ../../scripts/entrypoint /opt/ -COPY ../../scripts/start-celery /run/ -COPY ../../scripts/wait-for-app /run/ +COPY scripts/entrypoint /opt/ +COPY scripts/start-celery /run/ +COPY scripts/wait-for-app /run/ # Copying rodan core from build context into container # Rodan folder MUST be uppercase, otherwise many unittests fail. -COPY ../../backend/django /code/Rodan +COPY backend/django /code/Rodan # necessary for scikit-image > 0.17, or else it will try to make a cache directory # in a place where the www-data user does not have permissions to do so diff --git a/celery/gpu-workers/gpu-text-alignment/Dockerfile b/celery/gpu-workers/gpu-text-alignment/Dockerfile index ee77600..5dda71d 100644 --- a/celery/gpu-workers/gpu-text-alignment/Dockerfile +++ b/celery/gpu-workers/gpu-text-alignment/Dockerfile @@ -144,17 +144,17 @@ RUN rm -rf /var/lib/apt/lists/* #RUN pip uninstall opencv-python-headless && pip install opencv-python-headless==4.1.2.30 # Install GPU Rodan Jobs -COPY ../../scripts/install_gpu_jobs/install_text_alignment /opt/ +COPY scripts/install_gpu_jobs/install_text_alignment /opt/ # Install Rodan # Runs on both Rodan service, and Rodan-Celery -COPY ../../scripts/entrypoint /opt/ -COPY ../../scripts/start-celery /run/ -COPY ../../scripts/wait-for-app /run/ +COPY scripts/entrypoint /opt/ +COPY scripts/start-celery /run/ +COPY scripts/wait-for-app /run/ # Copying rodan core from build context into container # Rodan folder MUST be uppercase, otherwise many unittests fail. -COPY ../../backend/django /code/Rodan +COPY backend/django /code/Rodan # necessary for scikit-image > 0.17, or else it will try to make a cache directory # in a place where the www-data user does not have permissions to do so From 0d5c22be619eda7a45ed61f7f809fde0343a40dd Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Tue, 5 Aug 2025 18:49:34 -0400 Subject: [PATCH 09/26] add entrypoint file --- scripts/entrypoint | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 scripts/entrypoint diff --git a/scripts/entrypoint b/scripts/entrypoint new file mode 100644 index 0000000..91d02d2 --- /dev/null +++ b/scripts/entrypoint @@ -0,0 +1,10 @@ +#!/bin/sh + +# Because it doesn't seem to want to run in the dockerfile. +# WHY NOT?! I've rebuilt the layers prior multiple times!!!! +# /opt/install_gpu_rodan_jobs || echo "GPU jobs not install in the entrypoint" + +/run/wait-for-app postgres:5432 +/run/wait-for-app redis:6379 + +exec "$@" From 259e6c6241fe3d345e01b3cfa1b7dcfd0d1934ff Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Mon, 11 Aug 2025 10:52:11 -0400 Subject: [PATCH 10/26] fix relative rodan path --- scripts/install_gpu_jobs/install_background_removal | 2 +- scripts/install_gpu_jobs/install_paco_classifier | 2 +- scripts/install_gpu_jobs/install_sae_binarization | 2 +- scripts/install_gpu_jobs/install_text_alignment | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/install_gpu_jobs/install_background_removal b/scripts/install_gpu_jobs/install_background_removal index 1cd4899..c266743 100644 --- a/scripts/install_gpu_jobs/install_background_removal +++ b/scripts/install_gpu_jobs/install_background_removal @@ -27,7 +27,7 @@ which pip3 && $PIP install --no-cache-dir . -cd /code/Rodan/rodan +cd /code/Rodan/code sed -i 's/#gpu //g' /code/Rodan/rodan/settings.py # Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. diff --git a/scripts/install_gpu_jobs/install_paco_classifier b/scripts/install_gpu_jobs/install_paco_classifier index cab30f8..d99fd3b 100644 --- a/scripts/install_gpu_jobs/install_paco_classifier +++ b/scripts/install_gpu_jobs/install_paco_classifier @@ -25,7 +25,7 @@ mv Paco_classifier .Paco_classifier cd .Paco_classifier which pip3 && $PIP install --no-cache-dir . -cd /code/Rodan/rodan +cd /code/Rodan/code sed -i 's/#gpu //g' /code/Rodan/rodan/settings.py # Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. diff --git a/scripts/install_gpu_jobs/install_sae_binarization b/scripts/install_gpu_jobs/install_sae_binarization index b392cb2..f4dbf36 100644 --- a/scripts/install_gpu_jobs/install_sae_binarization +++ b/scripts/install_gpu_jobs/install_sae_binarization @@ -28,7 +28,7 @@ cd .SAE_binarization which pip3 && $PIP install --no-cache-dir . -cd /code/Rodan/rodan +cd /code/Rodan/code sed -i 's/#gpu //g' /code/Rodan/rodan/settings.py # Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. diff --git a/scripts/install_gpu_jobs/install_text_alignment b/scripts/install_gpu_jobs/install_text_alignment index b991599..eb03781 100644 --- a/scripts/install_gpu_jobs/install_text_alignment +++ b/scripts/install_gpu_jobs/install_text_alignment @@ -21,7 +21,7 @@ EOF -cd /code/Rodan/rodan +cd /code/Rodan/code sed -i 's/#gpu //g' /code/Rodan/rodan/settings.py # Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. From 106fa1021a40d619b112f509204117e335d05a1a Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Mon, 11 Aug 2025 11:10:46 -0400 Subject: [PATCH 11/26] use no cache install --- celery/gpu-workers/gpu-background-removal/Dockerfile | 2 +- celery/gpu-workers/gpu-paco-classifier/Dockerfile | 2 +- celery/gpu-workers/gpu-sae-binarization/Dockerfile | 2 +- celery/gpu-workers/gpu-text-alignment/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/celery/gpu-workers/gpu-background-removal/Dockerfile b/celery/gpu-workers/gpu-background-removal/Dockerfile index eb4109d..79221c5 100644 --- a/celery/gpu-workers/gpu-background-removal/Dockerfile +++ b/celery/gpu-workers/gpu-background-removal/Dockerfile @@ -181,7 +181,7 @@ RUN set -x \ && chmod +x /run/wait-for-app # Install Rodan. -RUN pip3 install -r /code/Rodan/requirements.txt +RUN pip3 install --no-cache-dir -r /code/Rodan/requirements.txt RUN pip3 uninstall -y opencv-python opencv-python-headless RUN pip3 install opencv-python-headless==4.1.2.30 diff --git a/celery/gpu-workers/gpu-paco-classifier/Dockerfile b/celery/gpu-workers/gpu-paco-classifier/Dockerfile index 87369c4..d93d149 100644 --- a/celery/gpu-workers/gpu-paco-classifier/Dockerfile +++ b/celery/gpu-workers/gpu-paco-classifier/Dockerfile @@ -181,7 +181,7 @@ RUN set -x \ && chmod +x /run/wait-for-app # Install Rodan. -RUN pip3 install -r /code/Rodan/requirements.txt +RUN pip3 install --no-cache-dir -r /code/Rodan/requirements.txt RUN pip3 uninstall -y opencv-python opencv-python-headless RUN pip3 install opencv-python-headless==4.1.2.30 diff --git a/celery/gpu-workers/gpu-sae-binarization/Dockerfile b/celery/gpu-workers/gpu-sae-binarization/Dockerfile index 9a3356b..3ebb8ff 100644 --- a/celery/gpu-workers/gpu-sae-binarization/Dockerfile +++ b/celery/gpu-workers/gpu-sae-binarization/Dockerfile @@ -181,7 +181,7 @@ RUN set -x \ && chmod +x /run/wait-for-app # Install Rodan. -RUN pip3 install -r /code/Rodan/requirements.txt +RUN pip3 install --no-cache-dir -r /code/Rodan/requirements.txt RUN pip3 uninstall -y opencv-python opencv-python-headless RUN pip3 install opencv-python-headless==4.1.2.30 diff --git a/celery/gpu-workers/gpu-text-alignment/Dockerfile b/celery/gpu-workers/gpu-text-alignment/Dockerfile index 5dda71d..5f408f1 100644 --- a/celery/gpu-workers/gpu-text-alignment/Dockerfile +++ b/celery/gpu-workers/gpu-text-alignment/Dockerfile @@ -181,7 +181,7 @@ RUN set -x \ && chmod +x /run/wait-for-app # Install Rodan. -RUN pip3 install -r /code/Rodan/requirements.txt +RUN pip3 install --no-cache-dir -r /code/Rodan/requirements.txt RUN pip3 uninstall -y opencv-python opencv-python-headless RUN pip3 install opencv-python-headless==4.1.2.30 From 24d0f789cc0f7e013ccba5ece7add14ca68f2b84 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Mon, 11 Aug 2025 11:12:48 -0400 Subject: [PATCH 12/26] leave only one gpu job for testing --- docker-compose.yml | 48 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0827098..963ffde 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -148,32 +148,32 @@ services: environment: CELERY_JOB_QUEUE: GPU - celery-gpu-paco-classifier: - <<: *celery-gpu-worker-base - profiles: ["celery-gpu-workers"] - build: - context: . - dockerfile: ./celery/gpu-workers/gpu-paco-classifier/Dockerfile - environment: - CELERY_JOB_QUEUE: GPU + # celery-gpu-paco-classifier: + # <<: *celery-gpu-worker-base + # profiles: ["celery-gpu-workers"] + # build: + # context: . + # dockerfile: ./celery/gpu-workers/gpu-paco-classifier/Dockerfile + # environment: + # CELERY_JOB_QUEUE: GPU - celery-gpu-sae-binarization: - <<: *celery-gpu-worker-base - profiles: ["celery-gpu-workers"] - build: - context: . - dockerfile: ./celery/gpu-workers/gpu-sae-binarization/Dockerfile - environment: - CELERY_JOB_QUEUE: GPU + # celery-gpu-sae-binarization: + # <<: *celery-gpu-worker-base + # profiles: ["celery-gpu-workers"] + # build: + # context: . + # dockerfile: ./celery/gpu-workers/gpu-sae-binarization/Dockerfile + # environment: + # CELERY_JOB_QUEUE: GPU - celery-gpu-text-alignment: - <<: *celery-gpu-worker-base - profiles: ["celery-gpu-workers"] - build: - context: . - dockerfile: ./celery/gpu-workers/gpu-text-alignment/Dockerfile - environment: - CELERY_JOB_QUEUE: GPU + # celery-gpu-text-alignment: + # <<: *celery-gpu-worker-base + # profiles: ["celery-gpu-workers"] + # build: + # context: . + # dockerfile: ./celery/gpu-workers/gpu-text-alignment/Dockerfile + # environment: + # CELERY_JOB_QUEUE: GPU # See more: # - https://docs.docker.com/compose/how-tos/gpu-support/ # - https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html From 2e2f1f4d7cd9c256b3439b3af2c7265f5349f928 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Mon, 11 Aug 2025 11:47:12 -0400 Subject: [PATCH 13/26] fix path in bg removal set up --- scripts/install_gpu_jobs/install_background_removal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_gpu_jobs/install_background_removal b/scripts/install_gpu_jobs/install_background_removal index c266743..86b5a07 100644 --- a/scripts/install_gpu_jobs/install_background_removal +++ b/scripts/install_gpu_jobs/install_background_removal @@ -28,7 +28,7 @@ which pip3 && $PIP install --no-cache-dir . cd /code/Rodan/code -sed -i 's/#gpu //g' /code/Rodan/rodan/settings.py +sed -i 's/#gpu //g' /code/Rodan/code/settings.py # Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. pip install opencv-python==4.6.0.66 From 2303574b824ea41a20a6c6704db044acda89e9df Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Mon, 11 Aug 2025 12:31:32 -0400 Subject: [PATCH 14/26] remove entry point --- celery/gpu-workers/gpu-background-removal/Dockerfile | 4 +--- celery/gpu-workers/gpu-paco-classifier/Dockerfile | 4 +--- celery/gpu-workers/gpu-sae-binarization/Dockerfile | 2 -- celery/gpu-workers/gpu-text-alignment/Dockerfile | 2 -- scripts/entrypoint | 10 ---------- 5 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 scripts/entrypoint diff --git a/celery/gpu-workers/gpu-background-removal/Dockerfile b/celery/gpu-workers/gpu-background-removal/Dockerfile index 79221c5..b2803f5 100644 --- a/celery/gpu-workers/gpu-background-removal/Dockerfile +++ b/celery/gpu-workers/gpu-background-removal/Dockerfile @@ -184,6 +184,4 @@ RUN set -x \ RUN pip3 install --no-cache-dir -r /code/Rodan/requirements.txt RUN pip3 uninstall -y opencv-python opencv-python-headless -RUN pip3 install opencv-python-headless==4.1.2.30 - -ENTRYPOINT ["/opt/entrypoint"] +RUN pip3 install opencv-python-headless==4.1.2.30 \ No newline at end of file diff --git a/celery/gpu-workers/gpu-paco-classifier/Dockerfile b/celery/gpu-workers/gpu-paco-classifier/Dockerfile index d93d149..a08b9d6 100644 --- a/celery/gpu-workers/gpu-paco-classifier/Dockerfile +++ b/celery/gpu-workers/gpu-paco-classifier/Dockerfile @@ -184,6 +184,4 @@ RUN set -x \ RUN pip3 install --no-cache-dir -r /code/Rodan/requirements.txt RUN pip3 uninstall -y opencv-python opencv-python-headless -RUN pip3 install opencv-python-headless==4.1.2.30 - -ENTRYPOINT ["/opt/entrypoint"] +RUN pip3 install opencv-python-headless==4.1.2.30 \ No newline at end of file diff --git a/celery/gpu-workers/gpu-sae-binarization/Dockerfile b/celery/gpu-workers/gpu-sae-binarization/Dockerfile index 3ebb8ff..1b7dd8c 100644 --- a/celery/gpu-workers/gpu-sae-binarization/Dockerfile +++ b/celery/gpu-workers/gpu-sae-binarization/Dockerfile @@ -185,5 +185,3 @@ RUN pip3 install --no-cache-dir -r /code/Rodan/requirements.txt RUN pip3 uninstall -y opencv-python opencv-python-headless RUN pip3 install opencv-python-headless==4.1.2.30 - -ENTRYPOINT ["/opt/entrypoint"] diff --git a/celery/gpu-workers/gpu-text-alignment/Dockerfile b/celery/gpu-workers/gpu-text-alignment/Dockerfile index 5f408f1..a25aef0 100644 --- a/celery/gpu-workers/gpu-text-alignment/Dockerfile +++ b/celery/gpu-workers/gpu-text-alignment/Dockerfile @@ -185,5 +185,3 @@ RUN pip3 install --no-cache-dir -r /code/Rodan/requirements.txt RUN pip3 uninstall -y opencv-python opencv-python-headless RUN pip3 install opencv-python-headless==4.1.2.30 - -ENTRYPOINT ["/opt/entrypoint"] diff --git a/scripts/entrypoint b/scripts/entrypoint deleted file mode 100644 index 91d02d2..0000000 --- a/scripts/entrypoint +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -# Because it doesn't seem to want to run in the dockerfile. -# WHY NOT?! I've rebuilt the layers prior multiple times!!!! -# /opt/install_gpu_rodan_jobs || echo "GPU jobs not install in the entrypoint" - -/run/wait-for-app postgres:5432 -/run/wait-for-app redis:6379 - -exec "$@" From 95abda07bde340c2a154b1cd32141e9d6f82a5b3 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Mon, 11 Aug 2025 12:33:19 -0400 Subject: [PATCH 15/26] fix path in install gpu jobs --- scripts/install_gpu_rodan_jobs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install_gpu_rodan_jobs b/scripts/install_gpu_rodan_jobs index 2bffbd5..e4f3657 100644 --- a/scripts/install_gpu_rodan_jobs +++ b/scripts/install_gpu_rodan_jobs @@ -40,8 +40,8 @@ mv Paco_classifier .Paco_classifier cd .Paco_classifier which pip3 && $PIP install --no-cache-dir . -cd /code/Rodan/rodan -sed -i 's/#gpu //g' /code/Rodan/rodan/settings.py +cd /code/Rodan/code +sed -i 's/#gpu //g' /code/Rodan/code/settings.py # Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. pip install opencv-python==4.6.0.66 From 5263fbe3f11658d8be09b92c7042dde7da8075b5 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Mon, 11 Aug 2025 12:36:22 -0400 Subject: [PATCH 16/26] Delete .DS_Store --- scripts/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 scripts/.DS_Store diff --git a/scripts/.DS_Store b/scripts/.DS_Store deleted file mode 100644 index b50faaa2ca8e17e98a55e0a29440888f57cf0cc3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}N6?5T4X(w-jLy3VI88E!f3>yezf8fGc`Xsk>~^)pb+4TYD&lz3U74B0i5Z zNh(SeFCtQA%H*3&W)kwHWHtaG+QZlbZ~?$UC9Kr3`9f%%bVe%HLn!ndA*A^0!VUCi zqS^5u8KARsA%P5XDB$Dy4e=cBKtTj~G|Kx?sv?YiiTY`h7p>N-s4P?$mugPUX*gHG ztr`VISPb)aIJltRsZwb)wZrHv>5aPX%8|;7Fv)rYosh&m47of{vRIAUYM8~j&JE0f zQ+MiJcYQqG+TCeNZ~I`7PTPzLgs{_mLQZCeT$_*96=E#715*$`@|3?9sSbA`4&rqCLM%c8RxMp3;RM5dUfjj~QSF=86GP?F5}RZpqfxnaxqHm8jRKBotQ~ md{03`mtu^iQoM$$1^to?MBid*5IrdTA)smCh8g%%2HpS|c2gDr From 185a8d3b5afbc9547c28ffb3ba5444821d265011 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Mon, 11 Aug 2025 12:37:42 -0400 Subject: [PATCH 17/26] comment out job install in gpu script for now --- scripts/install_gpu_rodan_jobs | 64 +++++++++++++++++----------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/scripts/install_gpu_rodan_jobs b/scripts/install_gpu_rodan_jobs index e4f3657..6dfdada 100644 --- a/scripts/install_gpu_rodan_jobs +++ b/scripts/install_gpu_rodan_jobs @@ -7,41 +7,41 @@ PIP=$(which pip3) || PIP=$(which pip) cd /code/Rodan/code/jobs # Install Text Alignment -$PIP install -r ./text_alignment/requirements.txt +# $PIP install -r ./text_alignment/requirements.txt -cat << EOF | python3 -new_backend = """{ - "image_dim_ordering": "tf", - "epsilon": 1e-07, - "floatx": "float32", - "backend": "tensorflow" -}""" -open("keras.json", "w").write(new_backend) -EOF +# cat << EOF | python3 +# new_backend = """{ +# "image_dim_ordering": "tf", +# "epsilon": 1e-07, +# "floatx": "float32", +# "backend": "tensorflow" +# }""" +# open("keras.json", "w").write(new_backend) +# EOF -# Install Background Removal -cd / -git clone -b v1.1.1 https://github.com/DDMAL/background_removal.git -mv background_removal .background_removal -cd .background_removal -which pip3 && $PIP install --no-cache-dir . +# # Install Background Removal +# cd / +# git clone -b v1.1.1 https://github.com/DDMAL/background_removal.git +# mv background_removal .background_removal +# cd .background_removal +# which pip3 && $PIP install --no-cache-dir . -# Install SAE_binarization -cd / -git clone -b v1.1.1 https://github.com/DDMAL/SAE_binarization.git -mv SAE_binarization .SAE_binarization -cd .SAE_binarization -which pip3 && $PIP install --no-cache-dir . +# # Install SAE_binarization +# cd / +# git clone -b v1.1.1 https://github.com/DDMAL/SAE_binarization.git +# mv SAE_binarization .SAE_binarization +# cd .SAE_binarization +# which pip3 && $PIP install --no-cache-dir . -# Install Paco classifier -cd / -git clone -b v2.0.4 --depth 1 https://github.com/DDMAL/Paco_classifier.git -mv Paco_classifier .Paco_classifier -cd .Paco_classifier -which pip3 && $PIP install --no-cache-dir . +# # Install Paco classifier +# cd / +# git clone -b v2.0.4 --depth 1 https://github.com/DDMAL/Paco_classifier.git +# mv Paco_classifier .Paco_classifier +# cd .Paco_classifier +# which pip3 && $PIP install --no-cache-dir . -cd /code/Rodan/code -sed -i 's/#gpu //g' /code/Rodan/code/settings.py +# cd /code/Rodan/code +# sed -i 's/#gpu //g' /code/Rodan/code/settings.py -# Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. -pip install opencv-python==4.6.0.66 +# # Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. +# pip install opencv-python==4.6.0.66 From c50d9992af2fb70fbdd25c42a8bf5b651a367882 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Mon, 11 Aug 2025 12:41:10 -0400 Subject: [PATCH 18/26] remove entrypoint... --- celery/gpu-workers/gpu-background-removal/Dockerfile | 6 +++--- celery/gpu-workers/gpu-paco-classifier/Dockerfile | 6 +++--- celery/gpu-workers/gpu-sae-binarization/Dockerfile | 6 +++--- celery/gpu-workers/gpu-text-alignment/Dockerfile | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/celery/gpu-workers/gpu-background-removal/Dockerfile b/celery/gpu-workers/gpu-background-removal/Dockerfile index b2803f5..fe5a3c9 100644 --- a/celery/gpu-workers/gpu-background-removal/Dockerfile +++ b/celery/gpu-workers/gpu-background-removal/Dockerfile @@ -148,7 +148,7 @@ COPY scripts/install_gpu_jobs/install_background_removal /opt/ # Install Rodan # Runs on both Rodan service, and Rodan-Celery -COPY scripts/entrypoint /opt/ +# COPY scripts/entrypoint /opt/ COPY scripts/start-celery /run/ COPY scripts/wait-for-app /run/ @@ -171,8 +171,8 @@ RUN set -x \ && sed -i "s/lxml/#lxml/g" /code/Rodan/requirements.txt \ # && sed -i "s/pybagit==1.5.0/-e git:\/\/github.com\/deepio\/pybagit.git@a27c9e0fc3bdf99dab8bd327f3ce9ea884abd6b4#egg=pybagit/g" /code/Rodan/requirements.txt \ # Add Entrypoints - && sed -i 's/\r//' /opt/entrypoint \ - && chmod +x /opt/entrypoint \ + # && sed -i 's/\r//' /opt/entrypoint \ + # && chmod +x /opt/entrypoint \ # Add Celery script && chmod +x /run/start-celery \ # Change the concurency for gpu jobs because Calvo is very expensive diff --git a/celery/gpu-workers/gpu-paco-classifier/Dockerfile b/celery/gpu-workers/gpu-paco-classifier/Dockerfile index a08b9d6..f97f169 100644 --- a/celery/gpu-workers/gpu-paco-classifier/Dockerfile +++ b/celery/gpu-workers/gpu-paco-classifier/Dockerfile @@ -148,7 +148,7 @@ COPY scripts/install_gpu_jobs/install_paco_classifier /opt/ # Install Rodan # Runs on both Rodan service, and Rodan-Celery -COPY scripts/entrypoint /opt/ +# COPY scripts/entrypoint /opt/ COPY scripts/start-celery /run/ COPY scripts/wait-for-app /run/ @@ -171,8 +171,8 @@ RUN set -x \ && sed -i "s/lxml/#lxml/g" /code/Rodan/requirements.txt \ # && sed -i "s/pybagit==1.5.0/-e git:\/\/github.com\/deepio\/pybagit.git@a27c9e0fc3bdf99dab8bd327f3ce9ea884abd6b4#egg=pybagit/g" /code/Rodan/requirements.txt \ # Add Entrypoints - && sed -i 's/\r//' /opt/entrypoint \ - && chmod +x /opt/entrypoint \ + # && sed -i 's/\r//' /opt/entrypoint \ + # && chmod +x /opt/entrypoint \ # Add Celery script && chmod +x /run/start-celery \ # Change the concurency for gpu jobs because Calvo is very expensive diff --git a/celery/gpu-workers/gpu-sae-binarization/Dockerfile b/celery/gpu-workers/gpu-sae-binarization/Dockerfile index 1b7dd8c..28dd96c 100644 --- a/celery/gpu-workers/gpu-sae-binarization/Dockerfile +++ b/celery/gpu-workers/gpu-sae-binarization/Dockerfile @@ -148,7 +148,7 @@ COPY scripts/install_gpu_jobs/install_sae_binarization /opt/ # Install Rodan # Runs on both Rodan service, and Rodan-Celery -COPY scripts/entrypoint /opt/ +# COPY scripts/entrypoint /opt/ COPY scripts/start-celery /run/ COPY scripts/wait-for-app /run/ @@ -171,8 +171,8 @@ RUN set -x \ && sed -i "s/lxml/#lxml/g" /code/Rodan/requirements.txt \ # && sed -i "s/pybagit==1.5.0/-e git:\/\/github.com\/deepio\/pybagit.git@a27c9e0fc3bdf99dab8bd327f3ce9ea884abd6b4#egg=pybagit/g" /code/Rodan/requirements.txt \ # Add Entrypoints - && sed -i 's/\r//' /opt/entrypoint \ - && chmod +x /opt/entrypoint \ + # && sed -i 's/\r//' /opt/entrypoint \ + # && chmod +x /opt/entrypoint \ # Add Celery script && chmod +x /run/start-celery \ # Change the concurency for gpu jobs because Calvo is very expensive diff --git a/celery/gpu-workers/gpu-text-alignment/Dockerfile b/celery/gpu-workers/gpu-text-alignment/Dockerfile index a25aef0..1a97d57 100644 --- a/celery/gpu-workers/gpu-text-alignment/Dockerfile +++ b/celery/gpu-workers/gpu-text-alignment/Dockerfile @@ -148,7 +148,7 @@ COPY scripts/install_gpu_jobs/install_text_alignment /opt/ # Install Rodan # Runs on both Rodan service, and Rodan-Celery -COPY scripts/entrypoint /opt/ +# COPY scripts/entrypoint /opt/ COPY scripts/start-celery /run/ COPY scripts/wait-for-app /run/ @@ -171,8 +171,8 @@ RUN set -x \ && sed -i "s/lxml/#lxml/g" /code/Rodan/requirements.txt \ # && sed -i "s/pybagit==1.5.0/-e git:\/\/github.com\/deepio\/pybagit.git@a27c9e0fc3bdf99dab8bd327f3ce9ea884abd6b4#egg=pybagit/g" /code/Rodan/requirements.txt \ # Add Entrypoints - && sed -i 's/\r//' /opt/entrypoint \ - && chmod +x /opt/entrypoint \ + # && sed -i 's/\r//' /opt/entrypoint \ + # && chmod +x /opt/entrypoint \ # Add Celery script && chmod +x /run/start-celery \ # Change the concurency for gpu jobs because Calvo is very expensive From 801231ec846edf33e0e2962650a426ed6849c762 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Mon, 11 Aug 2025 12:52:58 -0400 Subject: [PATCH 19/26] comment out everything in install gpu jobs for now --- scripts/install_gpu_rodan_jobs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_gpu_rodan_jobs b/scripts/install_gpu_rodan_jobs index 6dfdada..275ffe3 100644 --- a/scripts/install_gpu_rodan_jobs +++ b/scripts/install_gpu_rodan_jobs @@ -5,7 +5,7 @@ set -o xtrace # Print commands and their arguments as they are executed. PIP=$(which pip3) || PIP=$(which pip) -cd /code/Rodan/code/jobs +# cd /code/Rodan/code/jobs # Install Text Alignment # $PIP install -r ./text_alignment/requirements.txt From 7e734711b57d1622f8d24f719de4bc9b1f71925b Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Mon, 11 Aug 2025 13:14:51 -0400 Subject: [PATCH 20/26] add back other jobs since bg removal works now --- docker-compose.yml | 48 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 963ffde..0827098 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -148,32 +148,32 @@ services: environment: CELERY_JOB_QUEUE: GPU - # celery-gpu-paco-classifier: - # <<: *celery-gpu-worker-base - # profiles: ["celery-gpu-workers"] - # build: - # context: . - # dockerfile: ./celery/gpu-workers/gpu-paco-classifier/Dockerfile - # environment: - # CELERY_JOB_QUEUE: GPU + celery-gpu-paco-classifier: + <<: *celery-gpu-worker-base + profiles: ["celery-gpu-workers"] + build: + context: . + dockerfile: ./celery/gpu-workers/gpu-paco-classifier/Dockerfile + environment: + CELERY_JOB_QUEUE: GPU - # celery-gpu-sae-binarization: - # <<: *celery-gpu-worker-base - # profiles: ["celery-gpu-workers"] - # build: - # context: . - # dockerfile: ./celery/gpu-workers/gpu-sae-binarization/Dockerfile - # environment: - # CELERY_JOB_QUEUE: GPU + celery-gpu-sae-binarization: + <<: *celery-gpu-worker-base + profiles: ["celery-gpu-workers"] + build: + context: . + dockerfile: ./celery/gpu-workers/gpu-sae-binarization/Dockerfile + environment: + CELERY_JOB_QUEUE: GPU - # celery-gpu-text-alignment: - # <<: *celery-gpu-worker-base - # profiles: ["celery-gpu-workers"] - # build: - # context: . - # dockerfile: ./celery/gpu-workers/gpu-text-alignment/Dockerfile - # environment: - # CELERY_JOB_QUEUE: GPU + celery-gpu-text-alignment: + <<: *celery-gpu-worker-base + profiles: ["celery-gpu-workers"] + build: + context: . + dockerfile: ./celery/gpu-workers/gpu-text-alignment/Dockerfile + environment: + CELERY_JOB_QUEUE: GPU # See more: # - https://docs.docker.com/compose/how-tos/gpu-support/ # - https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html From fe26ea773da390aebfe0bde847b4536da1a67f21 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:06:18 -0400 Subject: [PATCH 21/26] leave only paco for testing --- docker-compose.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0827098..f9c2a20 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -157,23 +157,23 @@ services: environment: CELERY_JOB_QUEUE: GPU - celery-gpu-sae-binarization: - <<: *celery-gpu-worker-base - profiles: ["celery-gpu-workers"] - build: - context: . - dockerfile: ./celery/gpu-workers/gpu-sae-binarization/Dockerfile - environment: - CELERY_JOB_QUEUE: GPU + # celery-gpu-sae-binarization: + # <<: *celery-gpu-worker-base + # profiles: ["celery-gpu-workers"] + # build: + # context: . + # dockerfile: ./celery/gpu-workers/gpu-sae-binarization/Dockerfile + # environment: + # CELERY_JOB_QUEUE: GPU - celery-gpu-text-alignment: - <<: *celery-gpu-worker-base - profiles: ["celery-gpu-workers"] - build: - context: . - dockerfile: ./celery/gpu-workers/gpu-text-alignment/Dockerfile - environment: - CELERY_JOB_QUEUE: GPU + # celery-gpu-text-alignment: + # <<: *celery-gpu-worker-base + # profiles: ["celery-gpu-workers"] + # build: + # context: . + # dockerfile: ./celery/gpu-workers/gpu-text-alignment/Dockerfile + # environment: + # CELERY_JOB_QUEUE: GPU # See more: # - https://docs.docker.com/compose/how-tos/gpu-support/ # - https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html From 74cf437c691e536e059cbbb476d56c3249aebfa1 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:20:13 -0400 Subject: [PATCH 22/26] fix path --- scripts/install_gpu_jobs/install_paco_classifier | 2 +- scripts/install_gpu_jobs/install_sae_binarization | 2 +- scripts/install_gpu_jobs/install_text_alignment | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/install_gpu_jobs/install_paco_classifier b/scripts/install_gpu_jobs/install_paco_classifier index d99fd3b..135e5a1 100644 --- a/scripts/install_gpu_jobs/install_paco_classifier +++ b/scripts/install_gpu_jobs/install_paco_classifier @@ -26,7 +26,7 @@ cd .Paco_classifier which pip3 && $PIP install --no-cache-dir . cd /code/Rodan/code -sed -i 's/#gpu //g' /code/Rodan/rodan/settings.py +sed -i 's/#gpu //g' /code/Rodan/code/settings.py # Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. pip install opencv-python==4.6.0.66 diff --git a/scripts/install_gpu_jobs/install_sae_binarization b/scripts/install_gpu_jobs/install_sae_binarization index f4dbf36..c9527aa 100644 --- a/scripts/install_gpu_jobs/install_sae_binarization +++ b/scripts/install_gpu_jobs/install_sae_binarization @@ -29,7 +29,7 @@ which pip3 && $PIP install --no-cache-dir . cd /code/Rodan/code -sed -i 's/#gpu //g' /code/Rodan/rodan/settings.py +sed -i 's/#gpu //g' /code/Rodan/code/settings.py # Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. pip install opencv-python==4.6.0.66 diff --git a/scripts/install_gpu_jobs/install_text_alignment b/scripts/install_gpu_jobs/install_text_alignment index eb03781..6331a32 100644 --- a/scripts/install_gpu_jobs/install_text_alignment +++ b/scripts/install_gpu_jobs/install_text_alignment @@ -22,7 +22,7 @@ EOF cd /code/Rodan/code -sed -i 's/#gpu //g' /code/Rodan/rodan/settings.py +sed -i 's/#gpu //g' /code/Rodan/code/settings.py # Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. pip install opencv-python==4.6.0.66 From 7dfb1f9d4ab19b8cc385d705ddcfb8b4d4af972f Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:34:48 -0400 Subject: [PATCH 23/26] add back sae --- docker-compose.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f9c2a20..8a45f1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -157,14 +157,14 @@ services: environment: CELERY_JOB_QUEUE: GPU - # celery-gpu-sae-binarization: - # <<: *celery-gpu-worker-base - # profiles: ["celery-gpu-workers"] - # build: - # context: . - # dockerfile: ./celery/gpu-workers/gpu-sae-binarization/Dockerfile - # environment: - # CELERY_JOB_QUEUE: GPU + celery-gpu-sae-binarization: + <<: *celery-gpu-worker-base + profiles: ["celery-gpu-workers"] + build: + context: . + dockerfile: ./celery/gpu-workers/gpu-sae-binarization/Dockerfile + environment: + CELERY_JOB_QUEUE: GPU # celery-gpu-text-alignment: # <<: *celery-gpu-worker-base From 086bb797af81ec179054cc768925a3d8a657dc5f Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:51:32 -0400 Subject: [PATCH 24/26] try to clean up before build in ci to solve "no enough space left"" --- .github/workflows/CI-celery-build.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/CI-celery-build.yml b/.github/workflows/CI-celery-build.yml index aecf86f..46ce1c3 100644 --- a/.github/workflows/CI-celery-build.yml +++ b/.github/workflows/CI-celery-build.yml @@ -18,6 +18,7 @@ concurrency: cancel-in-progress: true jobs: + # ------- Celery Controller ------- celery-controller-build: runs-on: ubuntu-latest @@ -45,6 +46,18 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Free disk space before Docker build + run: | + echo "=== Disk usage before cleanup ===" + df -h + sudo rm -rf /usr/local/lib/android || true + sudo rm -rf /opt/ghc || true + sudo rm -rf /usr/share/dotnet || true + docker system prune -af --volumes || true + sudo apt-get clean + sudo rm -rf /var/lib/apt/lists/* + echo "=== Disk usage after cleanup ===" + df -h - uses: docker/setup-compose-action@v1 with: version: latest From b8e142cc54d53fd7b4a3c941bb2ae7dd9f318847 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Tue, 12 Aug 2025 12:07:20 -0400 Subject: [PATCH 25/26] add back text alignment... --- docker-compose.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8a45f1a..855e6f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -166,17 +166,17 @@ services: environment: CELERY_JOB_QUEUE: GPU - # celery-gpu-text-alignment: - # <<: *celery-gpu-worker-base - # profiles: ["celery-gpu-workers"] - # build: - # context: . - # dockerfile: ./celery/gpu-workers/gpu-text-alignment/Dockerfile - # environment: - # CELERY_JOB_QUEUE: GPU - # See more: - # - https://docs.docker.com/compose/how-tos/gpu-support/ - # - https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html + celery-gpu-text-alignment: + <<: *celery-gpu-worker-base + profiles: ["celery-gpu-workers"] + build: + context: . + dockerfile: ./celery/gpu-workers/gpu-text-alignment/Dockerfile + environment: + CELERY_JOB_QUEUE: GPU + See more: + - https://docs.docker.com/compose/how-tos/gpu-support/ + - https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html # Database: PostgreSQL, Redit (messaging) database-postgres: From d772dfe02cbe11dcdbf566fda7970c2961a03d10 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Tue, 12 Aug 2025 12:10:12 -0400 Subject: [PATCH 26/26] comment out notes --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 855e6f5..0827098 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -174,9 +174,9 @@ services: dockerfile: ./celery/gpu-workers/gpu-text-alignment/Dockerfile environment: CELERY_JOB_QUEUE: GPU - See more: - - https://docs.docker.com/compose/how-tos/gpu-support/ - - https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html + # See more: + # - https://docs.docker.com/compose/how-tos/gpu-support/ + # - https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html # Database: PostgreSQL, Redit (messaging) database-postgres: