Integrate conda

.github/PULL_REQUEST_TEMPLATE.md

@@ -17,9 +17,10 @@ Put an `x` in the boxes that apply. You can also fill these out after creating t
 
 - [ ] Changes have been tested to ensure that fix is effective or that a feature works.
 - [ ] Changes pass the unit tests
+- [ ] Changes pass the wet run tests
 - [ ] Code has been run through the `poe quality` task
 - [ ] I have included necessary documentation or comments (as necessary)
 - [ ] Any dependent changes have been merged and published
 
 ## Notes
-All PRs will undergo the unit testing before being reviewed. You may be requested to explain or make additional changes before the PR is accepted.
+All PRs will undergo unit and wet run testing before being reviewed. You may be requested to explain or make additional changes before the PR is accepted.

.github/workflows/push_conda.yml

@@ -0,0 +1,78 @@
+---
+# .github/workflows/build-conda.yml
+name: Build and Upload Conda Package
+on:
+  push:
+    branches: [integrate-conda]
+  workflow_dispatch:
+jobs:
+  build-and-upload:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Cache conda build cache
+        uses: actions/cache@v3
+        with:
+          path: /home/runner/miniconda3/conda-bld
+          key: ${{ runner.os }}-conda-bld-${{ hashFiles('recipe/meta.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-conda-bld-
+      - name: Setup Miniforge and mamba
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-variant: Miniforge3
+          miniforge-version: latest
+          mamba-version: '*'
+          use-mamba: true
+          conda-solver: libmamba
+          auto-activate-base: true
+      - name: Install dependencies
+        run: |
+          mamba install -y -c conda-forge conda-build anaconda-client boa
+      - name: Get latest commit SHA from integrate-conda branch
+        id: get_sha
+        run: |
+          echo "Fetching latest commit on integrate-conda branch..."
+          COMMIT_SHA=$(git ls-remote origin refs/heads/integrate-conda | cut -f1)
+          echo "COMMIT_SHA=$COMMIT_SHA" >> $GITHUB_ENV
+      - name: Update meta.yaml with latest ref and sha256
+        run: |
+          echo "Updating meta.yaml with commit: $COMMIT_SHA"
+
+          # 1) Update the Jinja-set 'ref' (this is fine)
+          sed -i "s|^{% set ref = \".*\" %}|{% set ref = \"$COMMIT_SHA\" %}|" ./recipe/meta.yaml
+
+          # 2) Download the tarball and compute the checksum
+          curl -L -o source.tar.gz \
+            https://github.com/khanlab/labelmerge/archive/${COMMIT_SHA}.tar.gz
+          SHA256_SUM=$(sha256sum source.tar.gz | awk '{print $1}')
+          echo "New sha256: $SHA256_SUM"
+
+          # 3) Replace the literal 'sha256:' line under source:
+          sed -i "s|^\(\s*sha256:\s*\).*|\1$SHA256_SUM|" ./recipe/meta.yaml
+      - name: Build & capture path
+        id: build
+        run: |
+          cd recipe
+          OUT=$(conda build . -c conda-forge -c bioconda --output 2>/dev/null)
+          echo "PACKAGE_PATH=$OUT" >> $GITHUB_ENV
+          conda mambabuild . -c conda-forge -c bioconda
+      - name: Upload package to Khanlab channel
+        shell: bash -l {0}
+        env:
+          ANACONDA_HOSTNAME: "gha-${{ github.run_id }}"
+        run: |-
+          conda activate base
+
+          # delete any leftover token or netrc entry
+          rm -f ~/.anaconda/token ~/.netrc || true
+
+          anaconda login \
+            --username ${{ secrets.ANACONDA_USERNAME }} \
+            --password ${{ secrets.ANACONDA_PASSWORD }} \
+            --hostname "$ANACONDA_HOSTNAME"
+
+          anaconda upload "$PACKAGE_PATH" \
+            --label main \
+            --force \

.github/workflows/deploy.yml → .github/workflows/push_docker.yml

@@ -10,7 +10,3 @@ jobs:
     secrets:
       DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
       DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-  deploy-pypi:
-    uses: khanlab/actions/.github/workflows/workflow-release_task-deployPypi.yml@v0.3.4
-    secrets:
-      PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/quality.yml

@@ -2,9 +2,9 @@
 name: Lint and format workflow
 on:
   push:
-    branches: ["*", "!push-action/*"]
+    branches: ['*', '!push-action/*']
   pull_request:
-    branches: ["*", "!push-action/*"]
+    branches: ['*', '!push-action/*']
 jobs:
   quality:
     name: Lint code
@@ -14,7 +14,7 @@ jobs:
       - name: Setup Python environment
         uses: khanlab/actions/.github/actions/action-setup_task-installPyProject@v0.3.4
         with:
-          python-version: "3.10"
+          python-version: '3.11'
       - name: yamlfix
         run: poetry run poe yamlfix-check
       - name: ruff

.github/workflows/wetrun_test.yml

@@ -0,0 +1,49 @@
+---
+name: test workflow with wet run
+on:
+  pull_request:
+jobs:
+  wetrun_test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Miniforge and mamba
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-variant: Miniforge3
+          miniforge-version: latest
+          mamba-version: '*'
+          use-mamba: true
+          conda-solver: libmamba
+          auto-activate-base: true
+      - name: Create environment with snakebids
+        run: |
+          mamba create -n snakebids-env snakebids scikit-learn pandas -c bioconda -c conda-forge -y
+      - name: Cache Snakemake Conda environments
+        uses: actions/cache@v4
+        with:
+          path: test_data/labelmerge_cache_dir/conda
+          key: conda-env-${{ hashFiles('labelmerge/workflow/envs/*.yaml') }}
+          restore-keys: |
+            conda-env-
+      - name: Set LABELMERGE_CACHE_DIR
+        run: |
+          echo "LABELMERGE_CACHE_DIR=`pwd`/test_data/labelmerge_cache_dir" >> $GITHUB_ENV
+      - name: Run wet-run test for cortical and subcortical files
+        shell: bash -l {0}
+        run: |-
+          conda activate snakebids-env
+          ./labelmerge/run.py test_data/bids_wetrun_testing/tpl-MNI152NLin2009cAsym test_out participant \
+            --base-desc 100Parcels7Networks --overlay_bids_dir test_data/bids_wetrun_testing/tpl-MNI152NLin2009cAsym \
+            --overlay_desc tn --cores all --force-output --conda-frontend mamba | tee labelmerge_output.log
+<<<<<<< chain-multiple-labels
+      - name: Run wet-run test for merging 3 labels
+        shell: bash -l {0}
+        run: |
+          conda activate snakebids-env
+          ./labelmerge/run.py test_data/bids_wetrun_testing/tpl-MNI152NLin2009cAsym test_out participant \
+            --base-desc 100Parcels7Networks --overlay_bids_dir test_data/bids_wetrun_testing/tpl-MNI152NLin2009cAsym \
+            --overlay_desc tn --overlay2_bids_dir test_data/bids_wetrun_testing/tpl-MNI152NLin2009cAsym/ --overlay2_desc carpet --cores all --force-output --conda-frontend mamba | tee labelmerge_output.log
+=======
+>>>>>>> integrate-conda

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 ---
 default_language_version:
-  python: python3.10
+  python: python3.11
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.3.0

.readthedocs.yaml

@@ -1,27 +1,23 @@
+---
 # .readthedocs.yml
 # Read the Docs configuration file
 # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
 
 # Required
 version: 2
-
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: docs/conf.py
-
 # Build documentation with MkDocs
-#mkdocs:
+# mkdocs:
 #  configuration: mkdocs.yml
 
 # Optionally build your docs in additional formats such as PDF
-formats:
-  - pdf
-
+formats: [pdf]
 build:
-  os: 'ubuntu-20.04'
+  os: ubuntu-20.04
   tools:
-    python: '3.10'
-
+    python: '3.11'
 python:
   install:
     - requirements: docs/requirements.txt

Dockerfile

@@ -1,21 +1,43 @@
-# Stage: build
-FROM python:3.9-slim-bullseye AS build
-COPY . /opt/labelmerge/
-RUN cd /opt/labelmerge \
-    && pip install --prefer-binary --no-cache-dir poetry \  
-    && poetry build -f wheel
-
-# Stage: runtime
-# NOTE: g++ required to install wheel (snakebids)
-FROM python:3.9-slim-bullseye AS runtime
-COPY --from=build /opt/labelmerge/dist/*.whl /opt/labelmerge/
-RUN apt-get update -qq \
-    && apt-get install -y -q --no-install-recommends \
-    g++=4:10.2.1-1 \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
-    && WHEEL=`ls /opt/labelmerge | grep whl` \
-    && pip install /opt/labelmerge/$WHEEL \
-    && rm -r /opt/labelmerge \
-    && apt-get purge -y -q g++ \
-    && apt-get --purge -y -qq autoremove
-ENTRYPOINT ["labelmerge"]
+# 1) Start from a minimal base that already includes mamba
+FROM condaforge/mambaforge:latest
+
+# 2) Set working directory
+WORKDIR /src/
+
+# 3) Copy your entire repository into /src/
+COPY . /src/
+
+# 4) Disable user‐site packages
+ENV PYTHONNOUSERSITE=1
+
+
+RUN mamba create -n snakebids-env \
+    -c conda-forge \
+    -c bioconda \
+    snakebids -y \
+    && mamba clean --all --yes
+
+RUN echo "source /opt/conda/etc/profile.d/conda.sh && conda activate snakebids-env" >> ~/.bashrc
+
+RUN bash -lc "\
+    source /opt/conda/etc/profile.d/conda.sh && \
+    conda activate snakebids-env && \
+    ./labelmerge/run.py \
+    test_data/bids_wetrun_testing/tpl-MNI152NLin2009cAsym \
+    test_out participant \
+    --base-desc 100Parcels7Networks \
+    --overlay_bids_dir test_data/bids_wetrun_testing/tpl-MNI152NLin2009cAsym \
+    --overlay_desc tn \
+    --use-conda \
+    --conda-create-envs-only \
+    --cores all \
+    --conda-prefix /src/conda-envs \
+    && mamba clean --all --yes \
+    && rm -rf /opt/conda/pkgs /root/.cache \
+    "
+
+# 8) Point Snakemake to the correct profile
+ENV SNAKEMAKE_PROFILE=/src/labelmerge/workflow/profiles/docker-conda
+
+# 9) Default entrypoint
+ENTRYPOINT ["/src/entrypoint.sh"]

MANIFEST.in

@@ -0,0 +1,2 @@
+recursive-include labelmerge/config *
+recursive-include labelmerge/workflow *

README.md

@@ -1,7 +1,7 @@
 ## Labelmerge
 
 ![Version](https://img.shields.io/github/v/tag/khanlab/labelmerge?label=version)
-![Python3](https://img.shields.io/badge/python-3.8_|_3.9_|_3.10_|_3.11-blue.svg)
+![Python3](https://img.shields.io/badge/python-3.11-blue.svg)
 ![Docker Pulls](https://img.shields.io/docker/pulls/khanlab/labelmerge)
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7636410.svg)](https://doi.org/10.5281/zenodo.7636410)
 

entrypoint.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+source /opt/conda/etc/profile.d/conda.sh
+conda activate snakebids-env
+exec /src/labelmerge/run.py "$@" 

labelmerge-dev.yml

@@ -0,0 +1,14 @@
+---
+name: labelmerge-dev
+channels: [conda-forge, bioconda, defaults]
+dependencies:
+  - python=3.11
+  - snakebids=0.14.0
+  - pytest>=7.0.1,<8.0.0
+  - pytest-console-scripts>=1.2,<2.0
+  - poethepoet>=0.10.0,<0.11.0
+  - pre-commit>=2.17.0,<3.0.0
+  - snakefmt>=0.6.1,<0.7.0
+  - yamlfix>=1.1.0,<2.0.0
+  - mypy>=0.991,<1.0.0
+  - ruff>=0.1.14,<0.2.0

labelmerge/config/snakebids.yml

@@ -70,24 +70,37 @@ parse_args:
     help: Directory containing labelmaps to be overlaid on the base input labelmaps.
     nargs: '?'
     type: Path
+  --overlay2_bids_dir:
+    help: Directory containing labelmaps to be overlaid on the combined labelmaps.
+    nargs: '?'
+    type: Path
   --base_desc:
     help: Description entity for base labelmaps
     nargs: '?'
   --overlay_desc:
     help: Description entity for overlay labelmaps
     nargs: '?'
+  --overlay2_desc:
+    help: Description entity for second overlay labelmaps
+    nargs: '?'
   --base_exceptions:
     help: Space separated integer labels from the base labelmap to keep over overlay labels at the same voxels.
     nargs: '*'
   --overlay_exceptions:
     help: Space separated integer labels from the overlay image to be overwritten by base labels at the same voxels.
     nargs: '*'
+  --overlay2_exceptions:
+    help: Space separated integer labels from the overlay image 2 to be overwritten by the combined labels at the same voxels.
+    nargs: '*'
   --base_drops:
     help: Space separated integer labels from the base image to drop from the output.
     nargs: '*'
   --overlay_drops:
     help: Space separated integer labels from the overlay image to drop from the output.
     nargs: '*'
+  --overlay2_drops:
+    help: Space separated integer labels from the second overlay image to drop from the output.
+    nargs: '*'
 
 # Workflow specific config