From bddd5b4f79363f7495eb471c72ab45fe6e14b87a Mon Sep 17 00:00:00 2001 From: garciagenrique Date: Tue, 20 May 2025 09:27:08 +0200 Subject: [PATCH 1/4] feat: integrate eso-edps pipeline into VRE --- .github/workflows/env-build.yaml | 2 +- vre-singleuser-eso-edps/Dockerfile | 39 + vre-singleuser-eso-edps/FROM | 0 vre-singleuser-eso-edps/README.md | 12 + .../application.properties | 160 ++ vre-singleuser-eso-edps/edps-gui.py | 104 + vre-singleuser-eso-edps/eso-logo.jpg | 1989 +++++++++++++++++ vre-singleuser-eso-edps/logging.yaml | 26 + vre-singleuser-eso-edps/pdf_handler.py | 16 + vre-singleuser-eso-edps/requirements.txt | 4 + 10 files changed, 2351 insertions(+), 1 deletion(-) create mode 100644 vre-singleuser-eso-edps/Dockerfile create mode 100644 vre-singleuser-eso-edps/FROM create mode 100644 vre-singleuser-eso-edps/README.md create mode 100644 vre-singleuser-eso-edps/application.properties create mode 100644 vre-singleuser-eso-edps/edps-gui.py create mode 100644 vre-singleuser-eso-edps/eso-logo.jpg create mode 100644 vre-singleuser-eso-edps/logging.yaml create mode 100644 vre-singleuser-eso-edps/pdf_handler.py create mode 100644 vre-singleuser-eso-edps/requirements.txt diff --git a/.github/workflows/env-build.yaml b/.github/workflows/env-build.yaml index 4fb04324..2d7d35ef 100644 --- a/.github/workflows/env-build.yaml +++ b/.github/workflows/env-build.yaml @@ -3,7 +3,7 @@ name: Docker automatic build and publish on: push: branches: - - main + - vre-eso_edps env: REGISTRY: ghcr.io diff --git a/vre-singleuser-eso-edps/Dockerfile b/vre-singleuser-eso-edps/Dockerfile new file mode 100644 index 00000000..a6c43ad8 --- /dev/null +++ b/vre-singleuser-eso-edps/Dockerfile @@ -0,0 +1,39 @@ +FROM ghcr.io/vre-hub/vre-singleuser-py311:sha-281055c + +RUN apt-get update && \ + apt-get install -y build-essential procps curl file git && \ + apt-get clean + +ENV XDG_RUNTIME_DIR=/var/run/adari +RUN mkdir $XDG_RUNTIME_DIR && chmod 777 $XDG_RUNTIME_DIR + +# Create user +RUN useradd -m -d /home/linuxbrew -s /bin/bash linuxbrew +USER linuxbrew +ENV PATH="/home/linuxbrew/.local/bin:$PATH" + +# Install Homebrew +RUN curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh | bash +ENV PATH="/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:${PATH}" + +WORKDIR /home/linuxbrew +RUN mkdir .edps EDPS_data + +COPY --chown=linuxbrew ./requirements.txt requirements.txt +COPY --chown=linuxbrew ./edps-gui.py edps-gui.py +COPY --chown=linuxbrew ./pdf_handler.py pdf_handler.py +COPY --chown=linuxbrew ./eso-logo.jpg eso-logo.jpg +COPY --chown=linuxbrew ./application.properties .edps/application.properties +COPY --chown=linuxbrew ./logging.yaml .edps/logging.yaml + +RUN brew install python@3.11 +RUN brew tap eso/pipelines +RUN brew install esopipe-fors + +RUN python3.11 -m venv venv && . venv/bin/activate && \ + pip install --no-cache-dir --upgrade -r requirements.txt + +ENV VIRTUAL_ENV=/home/linuxbrew/venv +ENV PATH=$VIRTUAL_ENV/bin:$PATH + +CMD ["panel", "serve", "edps-gui.py", "--plugins", "pdf_handler", "--address", "0.0.0.0", "--port", "7860", "--allow-websocket-origin", "*"] \ No newline at end of file diff --git a/vre-singleuser-eso-edps/FROM b/vre-singleuser-eso-edps/FROM new file mode 100644 index 00000000..e69de29b diff --git a/vre-singleuser-eso-edps/README.md b/vre-singleuser-eso-edps/README.md new file mode 100644 index 00000000..68784d77 --- /dev/null +++ b/vre-singleuser-eso-edps/README.md @@ -0,0 +1,12 @@ +--- +title: Edps +emoji: 🔥 +colorFrom: red +colorTo: gray +sdk: docker +pinned: false +license: bsd-3-clause +short_description: ESO Data Processing System +--- + +Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference diff --git a/vre-singleuser-eso-edps/application.properties b/vre-singleuser-eso-edps/application.properties new file mode 100644 index 00000000..28488735 --- /dev/null +++ b/vre-singleuser-eso-edps/application.properties @@ -0,0 +1,160 @@ +[server] +# Host or IP address to which EDPS server binds, e.g. localhost or 0.0.0.0 +host=0.0.0.0 + +# EDPS port number, e.g. 5000 +port=5000 + +[application] +# Comma-separated list of directories where workflows are installed. +# If not specified, EDPS will search for workflows in the pipeline installation tree. +# The naming convention for workflows is: /_wkf.py, e.g. espresso/espresso_wkf.py +workflow_dir= + +[executor] +# esorex is the command to execute pipeline recipes and it is installed with the pipeline. +# Please make sure that the path provided here can be located using the "which" command. +esorex_path=esorex + +# Path where pipeline plugins are installed. +# This configuration is used for ESO internal operations and can be left empty. +pipeline_path= + +# genreport is the command to execute quality control plots and it is installed with the Adari package. +genreport_path=genreport + +# EDPS data directory where recipe products, logs and quality control plots are saved. +# The files are organised in a directory structure under the base directory, defined as: +# /// +# Example: ESPRESSO/bias/fbf31155-a731-47f5-abf2-6445adce6c4b/master_bias.fits +# Please make sure that this directory has enough disk space available for storing the pipeline products, +# and consider enabling automatic data cleaning in the [cleanup] section. +base_dir=EDPS_data + +# If true, a dummy command is executed instead of esorex +dummy=False + +# If true, EDPS will attempt to execute a data reduction step even if the previous step has failed. +continue_on_error=False + +# Number of concurrent data reductions processes. +# Running concurrent data reductions will increase performance if sufficient resources are available, +# but can also lead to pipeline crashes if not enough memory is available to execute parallel reductions. +processes=1 + +# Number of CPUs (cores) available for data reduction. EDPS will not exceed the number of cores when scheduling +# data reduction tasks. +cores=1 + +# Pipeline recipes are parallelized using OpenMP. EDPS uses this parameter to set the number of threads when +# running a recipe, up to the available cores: OMP_NUM_THREADS=min(default_omp_threads, cores) +default_omp_threads=1 + +# Execution ordering. All orderings follow topological order so parent tasks are always placed before their children. +# Options: dfs, bfs, type, dynamic +# dfs - depth-first, give preference to reaching final reduction target quicker +# bfs - breadth-first, give preference to following reduction cascade level by level +# type - same as bfs, but make sure to process same type of data together (eg. first all BIASes) +# dynamic - immediately run whichever job is ready (has all needed inputs), no stalling but order is unpredictable +ordering=dfs + +# If provided, the recipe products will be renamed according to the following scheme: +# ..YYYY-MM-DDThh:mm.ss.mss.fits (Example: QC1.ESPRESSO.2023-02-09T17:30:14.326.fits), +# where timestamp is taken from the moment of renaming the file. +# Note that the renaming occurs in the base directory, not in the package (output) directory. +output_prefix= + +# In case EDPS was stopped while some jobs were waiting to be executed, should we execute them after restart. +resume_on_startup=False + +# EDPS will automatically re-execute a job if it's failed but needed as association, but only within this time window. +reexecution_window_minutes=60 + +[generator] +# Path to yaml file defining locations of static calibrations for each of the workflows. +# This configuration is used for ESO internal operations and can be left empty. +# EDPS will automatically load static calibrations delivered with the pipeline. +calibrations_config_file= + +# Path to yaml file defining locations of recipe and workflow parameters for each of the workflows. +# This configuration is used for ESO internal operations and can be left empty. +# EDPS will automatically load recipe and workflow parameters delivered with the pipeline. +parameters_config_file= + +# In case multiple matching associated inputs (e.g. calibrations) are available, which ones should be used. +# Options: raw, master, raw_per_quality_level, master_per_quality_level +# raw - use reduced raw data results even if master calibrations closer in time are available +# master - use master calibrations even if results of reduced raw data closer in time are available +# raw_per_quality_level - use calibrations closest in time but prefer reduced raw data results +# master_per_quality_level - use calibrations closest in time but prefer master calibrations +association_preference=raw_per_quality_level + +# URL to ESO-provided list of calibration breakpoints. +breakpoints_url= + +# Comma-separated list of workflows which should be combined together into one. +# This allows to submit data from different instruments to a single workflow "edps.workflow.meta_wkf" +meta_workflow= + +[repository] +# Clear the EDPS bookkeeping database on startup. +# This will cause all tasks to be re-executed even if they have been executed before on the same data. +truncate=False + +# Should we use local database for bookkeeping (currently always True). +local=True + +# Path where the bookkeeping database should be stored. +path=db.json + +# Type of bookkeeping database to use. +# Options: tiny, memory, caching +# tiny - directly use TinyDB json-file-based database +# memory - use fast in-memory non-persistent database +# caching - use in-memory cache on top of persistent TinyDB database for higher performance +type=caching + +# How many changes are needed to trigger TinyDB flushing data to disk. +flush_size=10 + +# How often automatically data should be flushed, regardless of changes. +flush_timeout=60 + +# Minimum amount of available disk space (in MB) required to flush data to disk. +min_disk_space_mb=100 + +[cleanup] +# Should automatic cleanup of reduced data be enabled. +enabled=False + +# How much time needs to pass since data got reduced to consider them for removal. +cleanup_older_than_seconds=1209600 + +# How often should we check if there are data to be removed. +cleanup_check_period_seconds=3600 + +[packager] +# Location where selected products should be placed. +package_base_dir= + +# Method to place files in the package directory. Options: link, symlink, copy. +# link - create hardlinks +# symlink - create symbolic links +# copy - copy the files +mode=symlink + +# Directory and filename pattern to use when placing files in the package directory. +# The pattern can contain any string, header keywords enclosed in $ (e.g. $pro.catg$), +# and the following predefined special variables: +# $NIGHT - year-month-day of when the data was taken +# $FILENAME - original name of the file +# $EXT - original extension of the file name +# $TASK - name of EDPS task which produced the file +# $TIMESTAMP - timestamp when data were submitted for reduction +# $DATASET - dataset name, derived from the first raw input file +# Example: $DATASET/$TIMESTAMP/$object$_$pro.catg$.$EXT +pattern=$DATASET/$TIMESTAMP/$object$_$pro.catg$.$EXT + +# Comma-separated list of product categories to place in the package directory. +# Empty means all products matching reduction target. +categories= \ No newline at end of file diff --git a/vre-singleuser-eso-edps/edps-gui.py b/vre-singleuser-eso-edps/edps-gui.py new file mode 100644 index 00000000..a38aee76 --- /dev/null +++ b/vre-singleuser-eso-edps/edps-gui.py @@ -0,0 +1,104 @@ +from pathlib import Path + +import panel as pn + +from edpsgui.gui.classifier import Classifier +from edpsgui.gui.dataset_creator import DatasetCreator +from edpsgui.gui.edps_ctl import EDPSControl +from edpsgui.gui.input_selector import InputSelector +from edpsgui.gui.job_viewer import JobViewer +from edpsgui.gui.reduction_history import ReductionHistory +from edpsgui.gui.reduction_queue import ReductionQueue +from edpsgui.gui.target_selector import TargetSelector +from edpsgui.gui.workflow import Workflow +from edpsgui.gui.workflow_selector import WorkflowSelector + +pn.extension('tabulator', 'tree', 'ipywidgets', 'terminal', 'codeeditor', 'modal', + css_files=["https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/all.min.css"], + disconnect_notification='Connection lost, try reloading the page!', + ready_notification='Application fully loaded.', + notifications=True) + +pn.state.notifications.position = 'top-center' + +input_selector = InputSelector() +edps_ctl = EDPSControl() +workflow_selector = WorkflowSelector(edps_status=edps_ctl.param.edps_status) +workflow_graph = Workflow(edps_status=edps_ctl.param.edps_status, + workflow=workflow_selector.param.workflow) +target_selector = TargetSelector(edps_status=edps_ctl.param.edps_status, + workflow=workflow_selector.param.workflow) +dataset_creator = DatasetCreator(edps_status=edps_ctl.param.edps_status, + workflow=workflow_selector.param.workflow, + inputs=input_selector.param.inputs, + targets=target_selector.param.targets) +classifier = Classifier(edps_status=edps_ctl.param.edps_status, + workflow=workflow_selector.param.workflow, + inputs=input_selector.param.inputs) +reduction_queue = ReductionQueue(edps_status=edps_ctl.param.edps_status, + workflow=workflow_selector.param.workflow) +reduction_history = ReductionHistory(edps_status=edps_ctl.param.edps_status, + workflow=workflow_selector.param.workflow) + +data_explorer = pn.Column( + f'### 1. Select input data', + input_selector, + pn.layout.Divider(), + f'### 2. Classify and inspect input data (optional)', + classifier, + pn.layout.Divider(), + f'### 3. Select reduction target', + target_selector, + pn.layout.Divider(), + f'### 4. Configure association (optional)', + dataset_creator, +) + +edps_aa_article = 'https://www.aanda.org/articles/aa/full_html/2024/01/aa47651-23/aa47651-23.html' +about_edps = pn.pane.HTML(f""" +If you make use of EDPS or its workflows for your research, we request to acknowledge +Freudling, Zampieri, Coccato et al 2024, A&A 681, A93. +""") + +reduction_tabs = pn.Tabs( + ('Processing', reduction_queue), + ('Processed', reduction_history), + dynamic=True, +) + + +def update_tabs(event): + if event == 2: + reduction_queue.update_table() + elif event == 3: + reduction_history.update_table() + + +tabs = pn.Tabs( + ('Workflow', workflow_graph), + ('Raw Data', data_explorer), + ('Processing Queue', reduction_queue), + ('Processed Data', reduction_history), + ('About EDPS', about_edps), + dynamic=True, +) + +pn.bind(update_tabs, tabs.param.active, watch=True) + +layout = pn.Column( + workflow_selector, + pn.layout.Divider(), + tabs, + sizing_mode='scale_width' +) + +logo_path = Path(__file__).resolve().parent / 'eso-logo.jpg' +template = pn.template.FastListTemplate(title='EDPS Dashboard', sidebar_width=250, logo=str(logo_path)) +template.main.append(layout) +template.header.append(edps_ctl) + +job_id = pn.state.location.query_params.get('job_id') +if job_id: + JobViewer(job_id).servable() +else: + template.servable() diff --git a/vre-singleuser-eso-edps/eso-logo.jpg b/vre-singleuser-eso-edps/eso-logo.jpg new file mode 100644 index 00000000..114bc878 --- /dev/null +++ b/vre-singleuser-eso-edps/eso-logo.jpg @@ -0,0 +1,1989 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + environments/eso_edps/eso-logo.jpg at 7c2374de629ce6bba7d77c0138839efa604b361f · vre-hub/environments · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ Skip to content + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + +
+ +
+ + + + + + + + +
+ + + + + +
+ + + + + + + + + +
+
+
+ + + + + + + + + + + + +
+ +
+ +
+ +
+ + + + / + + environments + + + Public +
+ + +
+ +
+ + +
+
+ +
+
+ + + + +
+ + + + + + +
+ + + + + + + + + + + + + + + + + + +
+ + + + + +
+ + + +
+
+ +
+ +
+

Footer

+ + + + +
+
+ + + + + © 2025 GitHub, Inc. + +
+ + +
+
+ + + + + + + + + + + + + + + + + + + +
+ +
+
+ + + diff --git a/vre-singleuser-eso-edps/logging.yaml b/vre-singleuser-eso-edps/logging.yaml new file mode 100644 index 00000000..2a8b402e --- /dev/null +++ b/vre-singleuser-eso-edps/logging.yaml @@ -0,0 +1,26 @@ +version: 1 +disable_existing_loggers: false + +formatters: + standard: + format: "%(asctime)s.%(msecs)03dZ %(levelname)5s %(process)d --- [%(threadName)s] %(name)s : %(message)s" + datefmt: "%Y-%m-%dT%H:%M:%S" + +handlers: + console: + class: logging.StreamHandler + level: DEBUG + formatter: standard + stream: ext://sys.stdout + + file: + class: logging.handlers.RotatingFileHandler + formatter: standard + filename: edps.log + maxBytes: 10485760 # 10MB + backupCount: 20 + encoding: utf8 + +root: + level: DEBUG + handlers: [ console,file ] \ No newline at end of file diff --git a/vre-singleuser-eso-edps/pdf_handler.py b/vre-singleuser-eso-edps/pdf_handler.py new file mode 100644 index 00000000..5090a920 --- /dev/null +++ b/vre-singleuser-eso-edps/pdf_handler.py @@ -0,0 +1,16 @@ +import os + +from tornado.web import RequestHandler + + +class PdfHandler(RequestHandler): + PDF_DIR = os.environ.get('EDPSGUI_PDF_DIR') or '.' + + def get(self): + filename = os.path.join(self.PDF_DIR, self.get_argument('file')) + self.set_header('Content-Type', 'application/pdf') + with open(filename, 'rb') as f: + self.write(f.read()) + + +ROUTES = [('/pdf', PdfHandler, {})] \ No newline at end of file diff --git a/vre-singleuser-eso-edps/requirements.txt b/vre-singleuser-eso-edps/requirements.txt new file mode 100644 index 00000000..ab11b961 --- /dev/null +++ b/vre-singleuser-eso-edps/requirements.txt @@ -0,0 +1,4 @@ +https://www.eso.org/~szampier/edps/adari_core-1.0.0.tar.gz +https://www.eso.org/~szampier/edps/edps-latest.tar.gz +https://www.eso.org/~szampier/edps/edpsgui-0.0.1.tar.gz +https://www.eso.org/~szampier/edps/replots-0.0.1.tar.gz \ No newline at end of file From f24e790c42624e3bef4d2c4ee1d61ee629bf8846 Mon Sep 17 00:00:00 2001 From: garciagenrique Date: Tue, 20 May 2025 10:06:29 +0200 Subject: [PATCH 2/4] test: change installation pckgs in dockerfile --- vre-singleuser-eso-edps/Dockerfile | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/vre-singleuser-eso-edps/Dockerfile b/vre-singleuser-eso-edps/Dockerfile index a6c43ad8..6fff4d50 100644 --- a/vre-singleuser-eso-edps/Dockerfile +++ b/vre-singleuser-eso-edps/Dockerfile @@ -1,20 +1,28 @@ FROM ghcr.io/vre-hub/vre-singleuser-py311:sha-281055c - -RUN apt-get update && \ - apt-get install -y build-essential procps curl file git && \ - apt-get clean +USER root + +RUN apt-get update \ + && apt-get install -y build-essential \ + procps \ + curl \ + file \ + git \ + ruby \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* ENV XDG_RUNTIME_DIR=/var/run/adari RUN mkdir $XDG_RUNTIME_DIR && chmod 777 $XDG_RUNTIME_DIR # Create user -RUN useradd -m -d /home/linuxbrew -s /bin/bash linuxbrew +#RUN useradd -m -d /home/linuxbrew -s /bin/bash linuxbrew +RUN useradd -m -s /bin/bash linuxbrew USER linuxbrew -ENV PATH="/home/linuxbrew/.local/bin:$PATH" +ENV HOME=/home/linuxbrew # Install Homebrew RUN curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh | bash -ENV PATH="/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:${PATH}" +ENV PATH="${HOME}/.linuxbrew/bin:${HOME}/.linuxbrew/sbin:${PATH}" WORKDIR /home/linuxbrew RUN mkdir .edps EDPS_data From 5adf78faa1a873184967aa90df0d730c162f099c Mon Sep 17 00:00:00 2001 From: garciagenrique Date: Tue, 20 May 2025 11:17:34 +0200 Subject: [PATCH 3/4] fix: start from scipy-notebook - change of user mess all the startups --- vre-singleuser-eso-edps/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vre-singleuser-eso-edps/Dockerfile b/vre-singleuser-eso-edps/Dockerfile index 6fff4d50..da0336c4 100644 --- a/vre-singleuser-eso-edps/Dockerfile +++ b/vre-singleuser-eso-edps/Dockerfile @@ -1,4 +1,5 @@ -FROM ghcr.io/vre-hub/vre-singleuser-py311:sha-281055c +#FROM ghcr.io/vre-hub/vre-singleuser-py311:sha-281055c +FROM quay.io/jupyter/scipy-notebook:python-3.11.8 USER root RUN apt-get update \ From d599a21298182b64752dce1ec2fd4a76e0dfddc9 Mon Sep 17 00:00:00 2001 From: garciagenrique Date: Tue, 20 May 2025 12:18:00 +0200 Subject: [PATCH 4/4] fix: remove CMD docker file --- vre-singleuser-eso-edps/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vre-singleuser-eso-edps/Dockerfile b/vre-singleuser-eso-edps/Dockerfile index da0336c4..4ece1f7e 100644 --- a/vre-singleuser-eso-edps/Dockerfile +++ b/vre-singleuser-eso-edps/Dockerfile @@ -45,4 +45,4 @@ RUN python3.11 -m venv venv && . venv/bin/activate && \ ENV VIRTUAL_ENV=/home/linuxbrew/venv ENV PATH=$VIRTUAL_ENV/bin:$PATH -CMD ["panel", "serve", "edps-gui.py", "--plugins", "pdf_handler", "--address", "0.0.0.0", "--port", "7860", "--allow-websocket-origin", "*"] \ No newline at end of file +#CMD ["panel", "serve", "edps-gui.py", "--plugins", "pdf_handler", "--address", "0.0.0.0", "--port", "7860", "--allow-websocket-origin", "*"] \ No newline at end of file