From 6e61e0fc10b0e0f76d75ef686e5214dfb5a6f8db Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Mon, 20 Nov 2017 15:11:55 +0400 Subject: [PATCH 01/45] lets see if it works with cache --- recipes/gencore_rnaseq/1.0/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index 945b6a1..a14484a 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -1,6 +1,7 @@ {% set name = "gencore_rnaseq" %} {% set version = "1.0" %} + package: name: {{ name }} version: {{ version }} From 70bfbe18436849ae7dab122f8baea647f07cc75c Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Tue, 21 Nov 2017 10:10:32 +0400 Subject: [PATCH 02/45] commit --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 01967bc..5796c51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ cache: - "/home/travis/.conda/pkgs" jobs: include: - - stage: Linux+macOS + - stage: Linux if: "(type = push) AND ((branch IN (master, bulk)) OR (fork = true))" os: linux script: scripts/travis-run.sh From cbc2661ad84ad8379764ebf6aa51d9fbde4b33ba Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Tue, 5 Dec 2017 13:52:41 +0400 Subject: [PATCH 03/45] rnaseq --- configs/bioconda_utils-requirements.txt | 23 +++++++++++++++++++++++ recipes/gencore_rnaseq/1.0/meta.yaml | 1 + scripts/travis-run.sh | 3 +-- scripts/travis-setup.sh | 6 ++---- 4 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 configs/bioconda_utils-requirements.txt diff --git a/configs/bioconda_utils-requirements.txt b/configs/bioconda_utils-requirements.txt new file mode 100644 index 0000000..4feb63d --- /dev/null +++ b/configs/bioconda_utils-requirements.txt @@ -0,0 +1,23 @@ +anaconda-client=1.6.3 +argh=0.26.2 +beautifulsoup4=4.6.0 +conda=4.3.21 +conda-build=2.1.16 +galaxy-lib=17.9.7 +jinja2=2.9.6 +jsonschema=2.5.1 +networkx=1.11 +pyaml=15.8.2 +pydotplus=2.0.2 +python=3.5.2 +requests=2.12.4 +sphinx +docutils +sphinx_rtd_theme +involucro=1.1.2 +pandas=0.19.2 +numpy=1.12.1 +pygithub=1.29 +colorlog=2.10.* +six=1.10.0 +alabaster=0.7.9 diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index a14484a..99e282b 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -53,3 +53,4 @@ about: home: 'https://cgsb.abudhabi.nyu.edu/' license: BSD summary: 'Gencore RnaSeq Module' + diff --git a/scripts/travis-run.sh b/scripts/travis-run.sh index 11e818c..1e9fdd5 100755 --- a/scripts/travis-run.sh +++ b/scripts/travis-run.sh @@ -8,7 +8,6 @@ set +u [[ -z $RANGE_ARG ]] && RANGE_ARG="--git-range master HEAD" [[ -z $DISABLE_BIOCONDA_UTILS_BUILD_GIT_RANGE_CHECK ]] && DISABLE_BIOCONDA_UTILS_BUILD_GIT_RANGE_CHECK="false" [[ -z $SKIP_LINTING ]] && SKIP_LINTING=false -# [[ -z $CONTAINER_NAMESPACE ]] && CONTAINER_NAMESPACE="nyuad_cgsb" set -u if [[ $TRAVIS_BRANCH != "master" && $TRAVIS_BRANCH != "bulk" && $TRAVIS_PULL_REQUEST == "false" && $TRAVIS_REPO_SLUG == "$MY_TRAVIS_REPO_SLUG" ]] @@ -111,6 +110,6 @@ fi set -x pip install --upgrade git+https://github.com/jerowe/bioconda-utils.git@develop -bioconda-utils build recipes config.yml $UPLOAD_ARG $DOCKER_ARG $BIOCONDA_UTILS_BUILD_ARGS $RANGE_ARG || echo "Build Failed!" +bioconda-utils build recipes config.yml --channels nyuad-cgsb $UPLOAD_ARG $DOCKER_ARG $BIOCONDA_UTILS_BUILD_ARGS $RANGE_ARG || echo "Build Failed!" docker images set +x diff --git a/scripts/travis-setup.sh b/scripts/travis-setup.sh index 77b7d03..24a8919 100755 --- a/scripts/travis-setup.sh +++ b/scripts/travis-setup.sh @@ -29,7 +29,5 @@ sudo chown -R $USER /anaconda $SCRIPT_DIR/../simulate-travis.py --bootstrap /anaconda --overwrite /anaconda/bin/conda index /anaconda/conda-bld/linux-64 /anaconda/conda-bld/osx-64 /anaconda/bin/conda config --add channels file://anaconda/conda-bld -/anaconda/bin/conda install -y r-base r-essentials openjdk perl bioconductor-biobase - -/anaconda/bin/conda info -ls -lah /home/travis/.conda/pkgs +/anaconda/bin/conda install -y r-base r-essentials openjdk perl bioconductor-biobase nodejs +/anaconda/bin/npm install marked-man From 65c6e513d8d20b9b56cd3ecfda8f755dbc939bbc Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Tue, 5 Dec 2017 14:13:24 +0400 Subject: [PATCH 04/45] build rnaseq --- recipes/gencore_rnaseq/1.0/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index 99e282b..4097260 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -54,3 +54,4 @@ about: license: BSD summary: 'Gencore RnaSeq Module' + From acab09a2fb8c3ec00690152f94b1831c1fc5eb26 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Tue, 5 Dec 2017 14:21:20 +0400 Subject: [PATCH 05/45] commit --- scripts/travis-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/travis-run.sh b/scripts/travis-run.sh index 1e9fdd5..029fc1c 100755 --- a/scripts/travis-run.sh +++ b/scripts/travis-run.sh @@ -110,6 +110,6 @@ fi set -x pip install --upgrade git+https://github.com/jerowe/bioconda-utils.git@develop -bioconda-utils build recipes config.yml --channels nyuad-cgsb $UPLOAD_ARG $DOCKER_ARG $BIOCONDA_UTILS_BUILD_ARGS $RANGE_ARG || echo "Build Failed!" +bioconda-utils build recipes config.yml --channel nyuad-cgsb $UPLOAD_ARG $DOCKER_ARG $BIOCONDA_UTILS_BUILD_ARGS $RANGE_ARG || echo "Build Failed!" docker images set +x From 1e3503a54722e8f8c7112e480a955d9a9c482db8 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Tue, 5 Dec 2017 14:21:43 +0400 Subject: [PATCH 06/45] rnaseq --- recipes/gencore_rnaseq/1.0/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index 4097260..99e282b 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -54,4 +54,3 @@ about: license: BSD summary: 'Gencore RnaSeq Module' - From 58b1676760d1f3398403844a8c360ba30d5826ed Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Tue, 5 Dec 2017 14:44:17 +0400 Subject: [PATCH 07/45] commiting --- recipes/gencore_rnaseq/1.0/meta.yaml | 1 + scripts/travis-run.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index 99e282b..4097260 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -54,3 +54,4 @@ about: license: BSD summary: 'Gencore RnaSeq Module' + diff --git a/scripts/travis-run.sh b/scripts/travis-run.sh index 029fc1c..48804a8 100755 --- a/scripts/travis-run.sh +++ b/scripts/travis-run.sh @@ -110,6 +110,6 @@ fi set -x pip install --upgrade git+https://github.com/jerowe/bioconda-utils.git@develop -bioconda-utils build recipes config.yml --channel nyuad-cgsb $UPLOAD_ARG $DOCKER_ARG $BIOCONDA_UTILS_BUILD_ARGS $RANGE_ARG || echo "Build Failed!" +bioconda-utils build recipes config.yml $UPLOAD_ARG $DOCKER_ARG $BIOCONDA_UTILS_BUILD_ARGS $RANGE_ARG || echo "Build Failed!" docker images set +x From 838af343b544a7701a83f859c648a0bcfb22a2bf Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Thu, 7 Dec 2017 11:01:30 +0400 Subject: [PATCH 08/45] starting new branch stupid rnaseq --- .gitignore | 1 + recipes/gencore_rnaseq/1.0/meta.yaml | 32 +++++++++---------- .../gencore_rnaseq_alignment/1.0/meta.yaml | 30 +++++++++++++++++ 3 files changed, 46 insertions(+), 17 deletions(-) create mode 100644 .gitignore create mode 100644 recipes/gencore_rnaseq_alignment/1.0/meta.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b237adb --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +set_envs.sh diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index 4097260..db805f4 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -12,38 +12,38 @@ build: requirements: run: + ##Alignment - samtools ==1.3.1 - - bcftools ==1.3.1 - kallisto ==0.43.0 + - cufflinks-patch ==2.2.1 + - bowtie2 ==2.2.8 + - tophat ==2.1.0 + - star ==2.5.2a + - bcftools ==1.3.1 - blat ==35 - gatk ==3.6 - picard ==2.5.0 - - trimmomatic ==0.36 - - fastqc ==0.11.5 - prinseq ==0.20.4 - - star ==2.5.2a - vcftools ==0.1.14 - bedtools ==2.25.0 - vcflib ==1.0.0_rc1 - bamtools ==2.4.0 - seqtk ==1.2 - - bowtie2 ==2.2.8 - - tophat ==2.1.0 - - cufflinks-patch ==2.2.1 + ##Downstream processing - r-base - r-essentials - r-cairo - bioconductor-deseq2 - bioconductor-edger >=3.14.0 - bioconductor-cummerbund >=2.16.0 - - bioconductor-genomicranges >=1.26.1 - - bioconductor-gviz >=1.16.5 - - bioconductor-biovizbase >=1.18.0 - - bioconductor-s4vectors >=0.12.0 - - bioconductor-ballgown >=2.2.0 - - htseq ==0.6.1.post1 - - hisat2 ==2.0.4 - - stringtie ==1.3.0 + # - bioconductor-genomicranges >=1.26.1 + # - bioconductor-gviz >=1.16.5 + # - bioconductor-biovizbase >=1.18.0 + # - bioconductor-s4vectors >=0.12.0 + # - bioconductor-ballgown >=2.2.0 + # - htseq ==0.6.1.post1 + # - hisat2 ==2.0.4 + # - stringtie ==1.3.0 test: commands: @@ -53,5 +53,3 @@ about: home: 'https://cgsb.abudhabi.nyu.edu/' license: BSD summary: 'Gencore RnaSeq Module' - - diff --git a/recipes/gencore_rnaseq_alignment/1.0/meta.yaml b/recipes/gencore_rnaseq_alignment/1.0/meta.yaml new file mode 100644 index 0000000..cc83403 --- /dev/null +++ b/recipes/gencore_rnaseq_alignment/1.0/meta.yaml @@ -0,0 +1,30 @@ +{% set name = "gencore_rnaseq_alignment" %} +{% set version = "1.0" %} + + +package: + name: {{ name }} + version: {{ version }} + +build: + number: 0 + skip: True # [osx] + +requirements: + run: + ##Alignment + - samtools ==1.3.1 + - kallisto ==0.43.0 + - cufflinks-patch ==2.2.1 + - bowtie2 ==2.2.8 + - tophat ==2.1.0 + - star ==2.5.2a + +test: + commands: + - bowtie2 --help 2>&1 | grep 'bowtie' + +about: + home: 'https://cgsb.abudhabi.nyu.edu/' + license: BSD + summary: 'Gencore RnaSeq Module' From 8fe3aff240a2141c19bda221536a890d23b3e631 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Fri, 8 Dec 2017 09:20:35 +0400 Subject: [PATCH 09/45] commit --- .gitignore | 1 + recipes/gencore_rnaseq/1.0/meta.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b237adb..535b6dc 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ set_envs.sh +involucro diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index db805f4..71c1cd7 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -47,7 +47,7 @@ requirements: test: commands: - - bowtie2 --help 2>&1 | grep 'bowtie' + - bowtie2 --help 2>&1 | grep "bowtie" about: home: 'https://cgsb.abudhabi.nyu.edu/' From 1a1713e598d1c6b21f021d17264bef31cbfcc33a Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Fri, 8 Dec 2017 09:43:39 +0400 Subject: [PATCH 10/45] trying ot see if this will speed up build --- scripts/travis-run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/travis-run.sh b/scripts/travis-run.sh index 48804a8..873bc14 100755 --- a/scripts/travis-run.sh +++ b/scripts/travis-run.sh @@ -73,7 +73,8 @@ export PATH=/anaconda/bin:$PATH # for local testing. if [[ $TRAVIS_OS_NAME == "linux" && $TRAVIS == "true" ]] then - DOCKER_ARG="--docker --mulled-test" + #DOCKER_ARG="--docker --mulled-test" + DOCKER_ARG=" --mulled-test" fi # When building master or bulk, upload packages to anaconda and quay.io. From f218aca273b06ae5a8b934bcf431f8381acdd8b3 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Fri, 8 Dec 2017 09:59:25 +0400 Subject: [PATCH 11/45] try this --- scripts/travis-run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/travis-run.sh b/scripts/travis-run.sh index 873bc14..30afd8c 100755 --- a/scripts/travis-run.sh +++ b/scripts/travis-run.sh @@ -73,8 +73,8 @@ export PATH=/anaconda/bin:$PATH # for local testing. if [[ $TRAVIS_OS_NAME == "linux" && $TRAVIS == "true" ]] then - #DOCKER_ARG="--docker --mulled-test" - DOCKER_ARG=" --mulled-test" + DOCKER_ARG="--docker --mulled-test" + #DOCKER_ARG=" --mulled-test" fi # When building master or bulk, upload packages to anaconda and quay.io. From c22263490c45e2751c952cf51aab453542a4073e Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Fri, 8 Dec 2017 10:05:54 +0400 Subject: [PATCH 12/45] building cache dir --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5796c51..0a72eef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ services: install: scripts/travis-setup.sh cache: directories: - - "/home/travis/.conda/pkgs" + - "/anaconda/conda-bld/pkg-cache" jobs: include: - stage: Linux @@ -27,7 +27,7 @@ env: - GITHUB_USER='nyuad-cgsb' - MY_TRAVIS_REPO_SLUG='biosails/biostacks' - CONTAINER_NAMESPACE='nyuad_cgsb' - - CONDA_PKGS_DIRS='/home/travis/.conda/pkgs' + - CONDA_PKGS_DIRS='/anaconda/conda-bld/pkg-cache' - secure: JHMiExNBJUTCd/cSXb7cY6DK4MmxIPO3IBfQAD6fSNOEXJe2KY9b8qAjxPwzKzD2FzQOZ1dln1rJiF/3HeVUEu3QN96PfhuqSeIaICP+m8q5ZHgQIyvHwDc09ne/yip1l6BIDtXCuNw/tYUZh3tgpw+bTve3cGXFiqYopvqMlZ2NcW/thQbpWCsdTnJq1g+Sq6DO3oRZm3WGdfS2sh3mv6TErXhQJi8nqWoNyOl2S6JGHRNOHp5vA11D24cAyxY+f5ezhp6Z3QxWL2mnnEwnazEC1PgcE6nA443R2fQRnF5q+ze42g+lSE3FTelWgwY2Jq5hIZRcov2Wj9H+i5zFDmnf503YZ2YLQ52WXnkPtuYCCOi5Sl7/qALLnjyLYNGoBCTZaONeToNSBs/058teuDZCDSLoLFCLlKnpPf1/FnkpEE8cegedHjvCeo2ffehCCrXiAoelUeAOscq9f7GSGa0UrybQckAhIQpR/J1C+FrOaOoFUXhtpxFFwxT6l36Vnt8i0dW0BKve6WU+DeG84JC5NnoULVK40Rg/VgGaB0qq1rX7+TWPw2kGWGWuAUDy+d6jAa+q41VzXjErmgeh3ftCtqS4TLS8tGGuhXkFj3wn9Z7fHIC0r82OqAQNyzj07hNk0MM/btGlU//bquyTFq4gwT2+jS3t1UqENFvAOA0= - secure: BX03MYZvVxOOLCXezmYGytgnpAHuaNh8kwVq2bYShUAAPNECZVPtNiXF+TQSuTfnXMM+9fm9M6dXyvCvbc5S8I5BcWEjqJj0fWvxmayejElySxEnNdH6FCuV0vWIwTzDyTUjiZcio/NaUgKlbhKfCqo85pxwOErHxuJlUILDmzwFQa84UzpFaAK2fEn5yja2GJbWfOmHiormHUz+2itchNPVQKXjhw9uZqnDflrVjhNzmF4+zUCLwiM9V5NRKobVeovRNTNg6sQ8ur5aL97tkR8FQz2Y92uYLfUtly8tzJX3/ytQ6F6kSSlat3rJcLjw5x1+J/kE1jjXvQ0Wj1Uy7qCowTMZ6rsOY3WpUiEas0YsIPYfn382ZnhqKW6OspPIFw+3tYVOv2GZtJPaDjfXCYvC5UC2qIpc/Po25nPA8Z6JgTSw9q0Y1gsn6EA/Auf8RD8SUXgwrxqWe/rT7rrpbOAhlDvM+fsCf666yl4sTe+AeJ36PepJ1H5pMZ82oz+K5Gh6wjj74t4hta/u/bbkd6HK0DIFZ+uHoHLq3sEQc9hq/4ljvy6CCq1S6w3sLd4/PNtHhbYlKywLckID255lkkla+mZegMzVE1VjsKE+tQhbvWkitlhJwbIowIcTmXFBxBZJzyp/8qcVwOhUy8ghSACQ6MWXov0QKW0BXnvqNmQ= - secure: jwpTmy+EHGsvVBo4e2auBKrY17niCy7lTfmiNPSpWwVIBALKH8TehRKk3IBnGV0SeWCPgd5JUV6UMBIwJQFM/4vMF9BfJ1o9Z0Rsrw+w6OgV7A4kIDK4aDxohHbCouGEk47LDfGt/tBcD97jfWaHxB7Uj2wgwakVEk9hraBFhajf0IcmZ8FkAqZXDhfc0ePtZZlHhRcRSjsOV2nZOVsiLRPcdLEnR/7+5YgswUNwxnUKDqimt3d8tDYWwnOzbuh9wsiQZt90v8b/ELIsARjx5wDsm9ksIaDi7Wg3BdUZSVX2JZf8lIgH5u1TiLPM1kdCsEila0Qdi+O2oGm7lYv0Y3XiXOjqpJKmWdpaFFlU7cN7CagSBxUzgziEYlHGKoDWjwnl6e20JQunkkVBJBl04PedDOUR2bAy0BumhNfXH0OmZokPJ8VZ226C2NjIWRRWKLyBxYp+k7OSouGdNpr30h22w/rwMY+4BnsHpI80M+LFbtWEhOOXbQd6/0witz/gNpPuG72I96fZnnz0KTRFaUdp6fsC4w5ITMYgmuaFzazhCyU7isP8k3ghdJTNGeY6S9HnRyb3k1SsRZsCP0qB12NEb9dVVc+/Ip0gtju4SurH11bj6fhbaKl0LZogGQHqCnnoBNrHvGmutIHZDPFJC4JysbAtfRsC77p1c7jucGA= From e6a539b78b8fc04ef4af39737c4b00788cb64a9b Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Fri, 8 Dec 2017 10:23:40 +0400 Subject: [PATCH 13/45] trying ot add in package cache --- .travis.yml | 4 ++-- scripts/travis-setup.sh | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0a72eef..4f11641 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ services: install: scripts/travis-setup.sh cache: directories: - - "/anaconda/conda-bld/pkg-cache" + - "/opt/anaconda-pkg-cache" jobs: include: - stage: Linux @@ -27,7 +27,7 @@ env: - GITHUB_USER='nyuad-cgsb' - MY_TRAVIS_REPO_SLUG='biosails/biostacks' - CONTAINER_NAMESPACE='nyuad_cgsb' - - CONDA_PKGS_DIRS='/anaconda/conda-bld/pkg-cache' + - CONDA_PKGS_DIRS='/opt/anaconda-pkg-cache' - secure: JHMiExNBJUTCd/cSXb7cY6DK4MmxIPO3IBfQAD6fSNOEXJe2KY9b8qAjxPwzKzD2FzQOZ1dln1rJiF/3HeVUEu3QN96PfhuqSeIaICP+m8q5ZHgQIyvHwDc09ne/yip1l6BIDtXCuNw/tYUZh3tgpw+bTve3cGXFiqYopvqMlZ2NcW/thQbpWCsdTnJq1g+Sq6DO3oRZm3WGdfS2sh3mv6TErXhQJi8nqWoNyOl2S6JGHRNOHp5vA11D24cAyxY+f5ezhp6Z3QxWL2mnnEwnazEC1PgcE6nA443R2fQRnF5q+ze42g+lSE3FTelWgwY2Jq5hIZRcov2Wj9H+i5zFDmnf503YZ2YLQ52WXnkPtuYCCOi5Sl7/qALLnjyLYNGoBCTZaONeToNSBs/058teuDZCDSLoLFCLlKnpPf1/FnkpEE8cegedHjvCeo2ffehCCrXiAoelUeAOscq9f7GSGa0UrybQckAhIQpR/J1C+FrOaOoFUXhtpxFFwxT6l36Vnt8i0dW0BKve6WU+DeG84JC5NnoULVK40Rg/VgGaB0qq1rX7+TWPw2kGWGWuAUDy+d6jAa+q41VzXjErmgeh3ftCtqS4TLS8tGGuhXkFj3wn9Z7fHIC0r82OqAQNyzj07hNk0MM/btGlU//bquyTFq4gwT2+jS3t1UqENFvAOA0= - secure: BX03MYZvVxOOLCXezmYGytgnpAHuaNh8kwVq2bYShUAAPNECZVPtNiXF+TQSuTfnXMM+9fm9M6dXyvCvbc5S8I5BcWEjqJj0fWvxmayejElySxEnNdH6FCuV0vWIwTzDyTUjiZcio/NaUgKlbhKfCqo85pxwOErHxuJlUILDmzwFQa84UzpFaAK2fEn5yja2GJbWfOmHiormHUz+2itchNPVQKXjhw9uZqnDflrVjhNzmF4+zUCLwiM9V5NRKobVeovRNTNg6sQ8ur5aL97tkR8FQz2Y92uYLfUtly8tzJX3/ytQ6F6kSSlat3rJcLjw5x1+J/kE1jjXvQ0Wj1Uy7qCowTMZ6rsOY3WpUiEas0YsIPYfn382ZnhqKW6OspPIFw+3tYVOv2GZtJPaDjfXCYvC5UC2qIpc/Po25nPA8Z6JgTSw9q0Y1gsn6EA/Auf8RD8SUXgwrxqWe/rT7rrpbOAhlDvM+fsCf666yl4sTe+AeJ36PepJ1H5pMZ82oz+K5Gh6wjj74t4hta/u/bbkd6HK0DIFZ+uHoHLq3sEQc9hq/4ljvy6CCq1S6w3sLd4/PNtHhbYlKywLckID255lkkla+mZegMzVE1VjsKE+tQhbvWkitlhJwbIowIcTmXFBxBZJzyp/8qcVwOhUy8ghSACQ6MWXov0QKW0BXnvqNmQ= - secure: jwpTmy+EHGsvVBo4e2auBKrY17niCy7lTfmiNPSpWwVIBALKH8TehRKk3IBnGV0SeWCPgd5JUV6UMBIwJQFM/4vMF9BfJ1o9Z0Rsrw+w6OgV7A4kIDK4aDxohHbCouGEk47LDfGt/tBcD97jfWaHxB7Uj2wgwakVEk9hraBFhajf0IcmZ8FkAqZXDhfc0ePtZZlHhRcRSjsOV2nZOVsiLRPcdLEnR/7+5YgswUNwxnUKDqimt3d8tDYWwnOzbuh9wsiQZt90v8b/ELIsARjx5wDsm9ksIaDi7Wg3BdUZSVX2JZf8lIgH5u1TiLPM1kdCsEila0Qdi+O2oGm7lYv0Y3XiXOjqpJKmWdpaFFlU7cN7CagSBxUzgziEYlHGKoDWjwnl6e20JQunkkVBJBl04PedDOUR2bAy0BumhNfXH0OmZokPJ8VZ226C2NjIWRRWKLyBxYp+k7OSouGdNpr30h22w/rwMY+4BnsHpI80M+LFbtWEhOOXbQd6/0witz/gNpPuG72I96fZnnz0KTRFaUdp6fsC4w5ITMYgmuaFzazhCyU7isP8k3ghdJTNGeY6S9HnRyb3k1SsRZsCP0qB12NEb9dVVc+/Ip0gtju4SurH11bj6fhbaKl0LZogGQHqCnnoBNrHvGmutIHZDPFJC4JysbAtfRsC77p1c7jucGA= diff --git a/scripts/travis-setup.sh b/scripts/travis-setup.sh index 24a8919..16b71f7 100755 --- a/scripts/travis-setup.sh +++ b/scripts/travis-setup.sh @@ -25,9 +25,11 @@ SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) pip install pyyaml sudo mkdir /anaconda +sudo mkdir -p /opt/anaconda-pkg-cache +sudo chown -R $USER /opt/anaconda-pkg-cache sudo chown -R $USER /anaconda $SCRIPT_DIR/../simulate-travis.py --bootstrap /anaconda --overwrite /anaconda/bin/conda index /anaconda/conda-bld/linux-64 /anaconda/conda-bld/osx-64 /anaconda/bin/conda config --add channels file://anaconda/conda-bld /anaconda/bin/conda install -y r-base r-essentials openjdk perl bioconductor-biobase nodejs -/anaconda/bin/npm install marked-man +/anaconda/bin/npm install marked-man From 8b49298eaa513148683199985c669c26213ea021 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Sat, 9 Dec 2017 11:38:35 +0400 Subject: [PATCH 14/45] try this --- recipes/gencore_rnaseq/1.0/meta.yaml | 1 - recipes/gencore_test/1.0/meta.yaml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index f102083..da71699 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -1,7 +1,6 @@ {% set name = "gencore_rnaseq" %} {% set version = "1.0" %} - package: name: {{ name }} version: {{ version }} diff --git a/recipes/gencore_test/1.0/meta.yaml b/recipes/gencore_test/1.0/meta.yaml index 530a4e1..e9eebab 100644 --- a/recipes/gencore_test/1.0/meta.yaml +++ b/recipes/gencore_test/1.0/meta.yaml @@ -13,6 +13,7 @@ build: requirements: run: - perl + - cufflinks-patch test: commands: From ac698e7a0bb3b12c16c4c26bcd81b2f159850434 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Sat, 9 Dec 2017 11:39:29 +0400 Subject: [PATCH 15/45] commit --- recipes/gencore_rnaseq/1.0/meta.yaml | 1 + .../gencore_rnaseq_alignment/1.0/meta.yaml | 30 ------------------- 2 files changed, 1 insertion(+), 30 deletions(-) delete mode 100644 recipes/gencore_rnaseq_alignment/1.0/meta.yaml diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index da71699..f102083 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -1,6 +1,7 @@ {% set name = "gencore_rnaseq" %} {% set version = "1.0" %} + package: name: {{ name }} version: {{ version }} diff --git a/recipes/gencore_rnaseq_alignment/1.0/meta.yaml b/recipes/gencore_rnaseq_alignment/1.0/meta.yaml deleted file mode 100644 index cc83403..0000000 --- a/recipes/gencore_rnaseq_alignment/1.0/meta.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{% set name = "gencore_rnaseq_alignment" %} -{% set version = "1.0" %} - - -package: - name: {{ name }} - version: {{ version }} - -build: - number: 0 - skip: True # [osx] - -requirements: - run: - ##Alignment - - samtools ==1.3.1 - - kallisto ==0.43.0 - - cufflinks-patch ==2.2.1 - - bowtie2 ==2.2.8 - - tophat ==2.1.0 - - star ==2.5.2a - -test: - commands: - - bowtie2 --help 2>&1 | grep 'bowtie' - -about: - home: 'https://cgsb.abudhabi.nyu.edu/' - license: BSD - summary: 'Gencore RnaSeq Module' From ad5725b59bd902d0ce45d099fd4d30e3c5ab4b36 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Sun, 10 Dec 2017 06:00:20 +0400 Subject: [PATCH 16/45] BUILD --- recipes/gencore_rnaseq/1.0/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index f102083..da71699 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -1,7 +1,6 @@ {% set name = "gencore_rnaseq" %} {% set version = "1.0" %} - package: name: {{ name }} version: {{ version }} From 644dc4605e3d2bbd84315041be7b331751b94b9f Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Sun, 10 Dec 2017 06:10:20 +0400 Subject: [PATCH 17/45] commiting --- recipes/gencore_rnaseq/1.0/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index da71699..f102083 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -1,6 +1,7 @@ {% set name = "gencore_rnaseq" %} {% set version = "1.0" %} + package: name: {{ name }} version: {{ version }} From e5627a3b14480ca8f4056c806cbe7e741efc9839 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Sun, 10 Dec 2017 09:50:23 +0400 Subject: [PATCH 18/45] lets try this --- recipes/gencore_rnaseq/1.0/meta.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index f102083..7082cdf 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -1,13 +1,12 @@ {% set name = "gencore_rnaseq" %} {% set version = "1.0" %} - package: name: {{ name }} version: {{ version }} build: - number: 0 + number: 1 skip: True # [osx] requirements: From 2cc941d75ec13dd79631385200ee9c538b15b375 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Mon, 11 Dec 2017 09:43:23 +0400 Subject: [PATCH 19/45] reopen --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4f11641..c640be1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,3 +35,4 @@ env: - SUBDAG=0 matrix: fast_finish: true + From 0434f79689223ce2e5ba85f27b68e6ee66648d3f Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Mon, 11 Dec 2017 10:00:26 +0400 Subject: [PATCH 20/45] incremental build --- recipes/gencore_rnaseq/1.0/meta.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index 7082cdf..711d7dc 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -29,20 +29,20 @@ requirements: - bamtools ==2.4.0 - seqtk ==1.2 ##Downstream processing - - r-base - - r-essentials - - r-cairo - - bioconductor-deseq2 - - bioconductor-edger >=3.14.0 - - bioconductor-cummerbund >=2.16.0 - - bioconductor-genomicranges >=1.26.1 - - bioconductor-gviz >=1.16.5 - - bioconductor-biovizbase >=1.18.0 - - bioconductor-s4vectors >=0.12.0 - - bioconductor-ballgown >=2.2.0 - - htseq ==0.6.1.post1 - - hisat2 ==2.0.4 - - stringtie ==1.3.0 + # - r-base + # - r-essentials + # - r-cairo + # - bioconductor-deseq2 + # - bioconductor-edger >=3.14.0 + # - bioconductor-cummerbund >=2.16.0 + # - bioconductor-genomicranges >=1.26.1 + # - bioconductor-gviz >=1.16.5 + # - bioconductor-biovizbase >=1.18.0 + # - bioconductor-s4vectors >=0.12.0 + # - bioconductor-ballgown >=2.2.0 + # - htseq ==0.6.1.post1 + # - hisat2 ==2.0.4 + # - stringtie ==1.3.0 test: commands: From fef85cec2e39c853534374dfecc4756df7fcb031 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Mon, 11 Dec 2017 10:22:16 +0400 Subject: [PATCH 21/45] commit --- .travis.yml | 3 +-- recipes/gencore_rnaseq/1.0/meta.yaml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index c640be1..b144728 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ env: global: - SUBDAGS=1 - BIOCONDA_UTILS_TAG=master - - BIOCONDA_UTILS_BUILD_ARGS="--loglevel=info" + - BIOCONDA_UTILS_BUILD_ARGS="--loglevel=debug" - BIOCONDA_UTILS_LINT_ARGS= - MINICONDA_VER="4.3.21" - ANACONDA_USER='nyuad-cgsb' @@ -35,4 +35,3 @@ env: - SUBDAG=0 matrix: fast_finish: true - diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index 711d7dc..83f5444 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -29,7 +29,7 @@ requirements: - bamtools ==2.4.0 - seqtk ==1.2 ##Downstream processing - # - r-base + - r-base # - r-essentials # - r-cairo # - bioconductor-deseq2 From c8266b603221a06b6ba5b6edad965185634d804c Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Mon, 11 Dec 2017 11:22:47 +0400 Subject: [PATCH 22/45] try it without the docker --- scripts/travis-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/travis-run.sh b/scripts/travis-run.sh index 30afd8c..cb297e4 100755 --- a/scripts/travis-run.sh +++ b/scripts/travis-run.sh @@ -73,7 +73,7 @@ export PATH=/anaconda/bin:$PATH # for local testing. if [[ $TRAVIS_OS_NAME == "linux" && $TRAVIS == "true" ]] then - DOCKER_ARG="--docker --mulled-test" + DOCKER_ARG="--mulled-test" #DOCKER_ARG=" --mulled-test" fi From 044b769db720a0fcc78a098fe723aac9bcf74212 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 09:20:30 +0400 Subject: [PATCH 23/45] BUILD --- .travis.yml | 5 +++-- Dockerfile | 14 ++++++++++++++ recipes/gencore_rnaseq/1.0/meta.yaml | 26 +++++++++++++------------- recipes/gencore_test/1.0/meta.yaml | 1 - scripts/build_docs.py | 11 +++++++++++ scripts/travis-setup.sh | 5 +++-- 6 files changed, 44 insertions(+), 18 deletions(-) create mode 100644 Dockerfile create mode 100644 scripts/build_docs.py diff --git a/.travis.yml b/.travis.yml index b144728..ef40ef9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ services: install: scripts/travis-setup.sh cache: directories: - - "/opt/anaconda-pkg-cache" + - "/opt/anaconda3/pkgs" jobs: include: - stage: Linux @@ -27,7 +27,8 @@ env: - GITHUB_USER='nyuad-cgsb' - MY_TRAVIS_REPO_SLUG='biosails/biostacks' - CONTAINER_NAMESPACE='nyuad_cgsb' - - CONDA_PKGS_DIRS='/opt/anaconda-pkg-cache' + - CONDA_PKGS_DIRS='/opt/anaconda3/pkgs' + - CONDA_BLD_PATH='/opt/anaconda3/conda-bld' - secure: JHMiExNBJUTCd/cSXb7cY6DK4MmxIPO3IBfQAD6fSNOEXJe2KY9b8qAjxPwzKzD2FzQOZ1dln1rJiF/3HeVUEu3QN96PfhuqSeIaICP+m8q5ZHgQIyvHwDc09ne/yip1l6BIDtXCuNw/tYUZh3tgpw+bTve3cGXFiqYopvqMlZ2NcW/thQbpWCsdTnJq1g+Sq6DO3oRZm3WGdfS2sh3mv6TErXhQJi8nqWoNyOl2S6JGHRNOHp5vA11D24cAyxY+f5ezhp6Z3QxWL2mnnEwnazEC1PgcE6nA443R2fQRnF5q+ze42g+lSE3FTelWgwY2Jq5hIZRcov2Wj9H+i5zFDmnf503YZ2YLQ52WXnkPtuYCCOi5Sl7/qALLnjyLYNGoBCTZaONeToNSBs/058teuDZCDSLoLFCLlKnpPf1/FnkpEE8cegedHjvCeo2ffehCCrXiAoelUeAOscq9f7GSGa0UrybQckAhIQpR/J1C+FrOaOoFUXhtpxFFwxT6l36Vnt8i0dW0BKve6WU+DeG84JC5NnoULVK40Rg/VgGaB0qq1rX7+TWPw2kGWGWuAUDy+d6jAa+q41VzXjErmgeh3ftCtqS4TLS8tGGuhXkFj3wn9Z7fHIC0r82OqAQNyzj07hNk0MM/btGlU//bquyTFq4gwT2+jS3t1UqENFvAOA0= - secure: BX03MYZvVxOOLCXezmYGytgnpAHuaNh8kwVq2bYShUAAPNECZVPtNiXF+TQSuTfnXMM+9fm9M6dXyvCvbc5S8I5BcWEjqJj0fWvxmayejElySxEnNdH6FCuV0vWIwTzDyTUjiZcio/NaUgKlbhKfCqo85pxwOErHxuJlUILDmzwFQa84UzpFaAK2fEn5yja2GJbWfOmHiormHUz+2itchNPVQKXjhw9uZqnDflrVjhNzmF4+zUCLwiM9V5NRKobVeovRNTNg6sQ8ur5aL97tkR8FQz2Y92uYLfUtly8tzJX3/ytQ6F6kSSlat3rJcLjw5x1+J/kE1jjXvQ0Wj1Uy7qCowTMZ6rsOY3WpUiEas0YsIPYfn382ZnhqKW6OspPIFw+3tYVOv2GZtJPaDjfXCYvC5UC2qIpc/Po25nPA8Z6JgTSw9q0Y1gsn6EA/Auf8RD8SUXgwrxqWe/rT7rrpbOAhlDvM+fsCf666yl4sTe+AeJ36PepJ1H5pMZ82oz+K5Gh6wjj74t4hta/u/bbkd6HK0DIFZ+uHoHLq3sEQc9hq/4ljvy6CCq1S6w3sLd4/PNtHhbYlKywLckID255lkkla+mZegMzVE1VjsKE+tQhbvWkitlhJwbIowIcTmXFBxBZJzyp/8qcVwOhUy8ghSACQ6MWXov0QKW0BXnvqNmQ= - secure: jwpTmy+EHGsvVBo4e2auBKrY17niCy7lTfmiNPSpWwVIBALKH8TehRKk3IBnGV0SeWCPgd5JUV6UMBIwJQFM/4vMF9BfJ1o9Z0Rsrw+w6OgV7A4kIDK4aDxohHbCouGEk47LDfGt/tBcD97jfWaHxB7Uj2wgwakVEk9hraBFhajf0IcmZ8FkAqZXDhfc0ePtZZlHhRcRSjsOV2nZOVsiLRPcdLEnR/7+5YgswUNwxnUKDqimt3d8tDYWwnOzbuh9wsiQZt90v8b/ELIsARjx5wDsm9ksIaDi7Wg3BdUZSVX2JZf8lIgH5u1TiLPM1kdCsEila0Qdi+O2oGm7lYv0Y3XiXOjqpJKmWdpaFFlU7cN7CagSBxUzgziEYlHGKoDWjwnl6e20JQunkkVBJBl04PedDOUR2bAy0BumhNfXH0OmZokPJ8VZ226C2NjIWRRWKLyBxYp+k7OSouGdNpr30h22w/rwMY+4BnsHpI80M+LFbtWEhOOXbQd6/0witz/gNpPuG72I96fZnnz0KTRFaUdp6fsC4w5ITMYgmuaFzazhCyU7isP8k3ghdJTNGeY6S9HnRyb3k1SsRZsCP0qB12NEb9dVVc+/Ip0gtju4SurH11bj6fhbaKl0LZogGQHqCnnoBNrHvGmutIHZDPFJC4JysbAtfRsC77p1c7jucGA= diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..481239d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +from continuumio/miniconda3 + +COPY /home/jillian/.eb/software/Anaconda3/5.0.1/pkgs /anaconda-pkg-cache +COPY /home/jillian/.eb/software/Anaconda3/5.0.1/conda-bld /anaconda-conda-bld + +ENV CONDA_PKGS_DIRS /anaconda/pkg-cache +ENV CONDA_BLD_PATH /anaconda-conda-bld + +RUN conda add --channels file://anaconda-conda-bld +RUN conda index /anaconda-conda-bld/linux-64 + +RUN cat ~/.condarc + +RUN conda install -y gencore_test diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index 83f5444..7082cdf 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -30,19 +30,19 @@ requirements: - seqtk ==1.2 ##Downstream processing - r-base - # - r-essentials - # - r-cairo - # - bioconductor-deseq2 - # - bioconductor-edger >=3.14.0 - # - bioconductor-cummerbund >=2.16.0 - # - bioconductor-genomicranges >=1.26.1 - # - bioconductor-gviz >=1.16.5 - # - bioconductor-biovizbase >=1.18.0 - # - bioconductor-s4vectors >=0.12.0 - # - bioconductor-ballgown >=2.2.0 - # - htseq ==0.6.1.post1 - # - hisat2 ==2.0.4 - # - stringtie ==1.3.0 + - r-essentials + - r-cairo + - bioconductor-deseq2 + - bioconductor-edger >=3.14.0 + - bioconductor-cummerbund >=2.16.0 + - bioconductor-genomicranges >=1.26.1 + - bioconductor-gviz >=1.16.5 + - bioconductor-biovizbase >=1.18.0 + - bioconductor-s4vectors >=0.12.0 + - bioconductor-ballgown >=2.2.0 + - htseq ==0.6.1.post1 + - hisat2 ==2.0.4 + - stringtie ==1.3.0 test: commands: diff --git a/recipes/gencore_test/1.0/meta.yaml b/recipes/gencore_test/1.0/meta.yaml index e9eebab..530a4e1 100644 --- a/recipes/gencore_test/1.0/meta.yaml +++ b/recipes/gencore_test/1.0/meta.yaml @@ -13,7 +13,6 @@ build: requirements: run: - perl - - cufflinks-patch test: commands: diff --git a/scripts/build_docs.py b/scripts/build_docs.py new file mode 100644 index 0000000..0db8ef2 --- /dev/null +++ b/scripts/build_docs.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +from conda_build.config import Config +from conda_build.metadata import MetaData +from conda_build.render import render_recipe + +recipe_dir = 'recipes/gencore_rnaseq/1.0/meta.yaml' + +config = Config(croot=recipe_dir, anaconda_upload=False, verbose=True, +activate=False, debug=False, variant=None) +metadata = render_recipe(recipe_dir, config=config) diff --git a/scripts/travis-setup.sh b/scripts/travis-setup.sh index 16b71f7..c4b06a0 100755 --- a/scripts/travis-setup.sh +++ b/scripts/travis-setup.sh @@ -25,9 +25,10 @@ SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) pip install pyyaml sudo mkdir /anaconda -sudo mkdir -p /opt/anaconda-pkg-cache -sudo chown -R $USER /opt/anaconda-pkg-cache + +sudo chown -R $USER /opt/anaconda3 sudo chown -R $USER /anaconda + $SCRIPT_DIR/../simulate-travis.py --bootstrap /anaconda --overwrite /anaconda/bin/conda index /anaconda/conda-bld/linux-64 /anaconda/conda-bld/osx-64 /anaconda/bin/conda config --add channels file://anaconda/conda-bld From 90a6d5b4d456b0e6b202fb67be1fe224bf78bf48 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 09:42:53 +0400 Subject: [PATCH 24/45] trying with conda-build-all instead --- .travis.yml | 33 ++++++++++++-------- Dockerfile | 15 ++------- recipes/gencore_rnaseq/1.0/meta.yaml | 22 ++++++------- scripts/travis-run-conda-build-all.sh | 45 +++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 36 deletions(-) create mode 100644 scripts/travis-run-conda-build-all.sh diff --git a/.travis.yml b/.travis.yml index ef40ef9..186e566 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,24 +11,31 @@ jobs: - stage: Linux if: "(type = push) AND ((branch IN (master, bulk)) OR (fork = true))" os: linux - script: scripts/travis-run.sh + script: scripts/travis-run-conda-build-all.sh - stage: Linux if: type = pull_request os: linux - script: scripts/travis-run.sh + script: scripts/travis-run-conda-build-all.sh env: global: - - SUBDAGS=1 - - BIOCONDA_UTILS_TAG=master - - BIOCONDA_UTILS_BUILD_ARGS="--loglevel=debug" - - BIOCONDA_UTILS_LINT_ARGS= - - MINICONDA_VER="4.3.21" - - ANACONDA_USER='nyuad-cgsb' - - GITHUB_USER='nyuad-cgsb' - - MY_TRAVIS_REPO_SLUG='biosails/biostacks' - - CONTAINER_NAMESPACE='nyuad_cgsb' - - CONDA_PKGS_DIRS='/opt/anaconda3/pkgs' - - CONDA_BLD_PATH='/opt/anaconda3/conda-bld' + - MINICONDA_VER: '4.3.21' + - ANACONDA_USER: 'nyuad-cgsb' + - GITHUB_USER: 'nyuad-cgsb' + - MY_TRAVIS_REPO_SLUG: 'biosails/biostacks' + - CONTAINER_NAMESPACE: 'nyuad_cgsb' + - CONDA_PKGS_DIRS: '/opt/anaconda3/pkgs' + - CONDA_BLD_PATH: '/opt/anaconda3/conda-bld' + # - SUBDAGS=1 + # - BIOCONDA_UTILS_TAG=master + # - BIOCONDA_UTILS_BUILD_ARGS="--loglevel=debug" + # - BIOCONDA_UTILS_LINT_ARGS= + # - MINICONDA_VER="4.3.21" + # - ANACONDA_USER='nyuad-cgsb' + # - GITHUB_USER='nyuad-cgsb' + # - MY_TRAVIS_REPO_SLUG='biosails/biostacks' + # - CONTAINER_NAMESPACE='nyuad_cgsb' + # - CONDA_PKGS_DIRS='/opt/anaconda3/pkgs' + # - CONDA_BLD_PATH='/opt/anaconda3/conda-bld' - secure: JHMiExNBJUTCd/cSXb7cY6DK4MmxIPO3IBfQAD6fSNOEXJe2KY9b8qAjxPwzKzD2FzQOZ1dln1rJiF/3HeVUEu3QN96PfhuqSeIaICP+m8q5ZHgQIyvHwDc09ne/yip1l6BIDtXCuNw/tYUZh3tgpw+bTve3cGXFiqYopvqMlZ2NcW/thQbpWCsdTnJq1g+Sq6DO3oRZm3WGdfS2sh3mv6TErXhQJi8nqWoNyOl2S6JGHRNOHp5vA11D24cAyxY+f5ezhp6Z3QxWL2mnnEwnazEC1PgcE6nA443R2fQRnF5q+ze42g+lSE3FTelWgwY2Jq5hIZRcov2Wj9H+i5zFDmnf503YZ2YLQ52WXnkPtuYCCOi5Sl7/qALLnjyLYNGoBCTZaONeToNSBs/058teuDZCDSLoLFCLlKnpPf1/FnkpEE8cegedHjvCeo2ffehCCrXiAoelUeAOscq9f7GSGa0UrybQckAhIQpR/J1C+FrOaOoFUXhtpxFFwxT6l36Vnt8i0dW0BKve6WU+DeG84JC5NnoULVK40Rg/VgGaB0qq1rX7+TWPw2kGWGWuAUDy+d6jAa+q41VzXjErmgeh3ftCtqS4TLS8tGGuhXkFj3wn9Z7fHIC0r82OqAQNyzj07hNk0MM/btGlU//bquyTFq4gwT2+jS3t1UqENFvAOA0= - secure: BX03MYZvVxOOLCXezmYGytgnpAHuaNh8kwVq2bYShUAAPNECZVPtNiXF+TQSuTfnXMM+9fm9M6dXyvCvbc5S8I5BcWEjqJj0fWvxmayejElySxEnNdH6FCuV0vWIwTzDyTUjiZcio/NaUgKlbhKfCqo85pxwOErHxuJlUILDmzwFQa84UzpFaAK2fEn5yja2GJbWfOmHiormHUz+2itchNPVQKXjhw9uZqnDflrVjhNzmF4+zUCLwiM9V5NRKobVeovRNTNg6sQ8ur5aL97tkR8FQz2Y92uYLfUtly8tzJX3/ytQ6F6kSSlat3rJcLjw5x1+J/kE1jjXvQ0Wj1Uy7qCowTMZ6rsOY3WpUiEas0YsIPYfn382ZnhqKW6OspPIFw+3tYVOv2GZtJPaDjfXCYvC5UC2qIpc/Po25nPA8Z6JgTSw9q0Y1gsn6EA/Auf8RD8SUXgwrxqWe/rT7rrpbOAhlDvM+fsCf666yl4sTe+AeJ36PepJ1H5pMZ82oz+K5Gh6wjj74t4hta/u/bbkd6HK0DIFZ+uHoHLq3sEQc9hq/4ljvy6CCq1S6w3sLd4/PNtHhbYlKywLckID255lkkla+mZegMzVE1VjsKE+tQhbvWkitlhJwbIowIcTmXFBxBZJzyp/8qcVwOhUy8ghSACQ6MWXov0QKW0BXnvqNmQ= - secure: jwpTmy+EHGsvVBo4e2auBKrY17niCy7lTfmiNPSpWwVIBALKH8TehRKk3IBnGV0SeWCPgd5JUV6UMBIwJQFM/4vMF9BfJ1o9Z0Rsrw+w6OgV7A4kIDK4aDxohHbCouGEk47LDfGt/tBcD97jfWaHxB7Uj2wgwakVEk9hraBFhajf0IcmZ8FkAqZXDhfc0ePtZZlHhRcRSjsOV2nZOVsiLRPcdLEnR/7+5YgswUNwxnUKDqimt3d8tDYWwnOzbuh9wsiQZt90v8b/ELIsARjx5wDsm9ksIaDi7Wg3BdUZSVX2JZf8lIgH5u1TiLPM1kdCsEila0Qdi+O2oGm7lYv0Y3XiXOjqpJKmWdpaFFlU7cN7CagSBxUzgziEYlHGKoDWjwnl6e20JQunkkVBJBl04PedDOUR2bAy0BumhNfXH0OmZokPJ8VZ226C2NjIWRRWKLyBxYp+k7OSouGdNpr30h22w/rwMY+4BnsHpI80M+LFbtWEhOOXbQd6/0witz/gNpPuG72I96fZnnz0KTRFaUdp6fsC4w5ITMYgmuaFzazhCyU7isP8k3ghdJTNGeY6S9HnRyb3k1SsRZsCP0qB12NEb9dVVc+/Ip0gtju4SurH11bj6fhbaKl0LZogGQHqCnnoBNrHvGmutIHZDPFJC4JysbAtfRsC77p1c7jucGA= diff --git a/Dockerfile b/Dockerfile index 481239d..dc41f92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,5 @@ from continuumio/miniconda3 -COPY /home/jillian/.eb/software/Anaconda3/5.0.1/pkgs /anaconda-pkg-cache -COPY /home/jillian/.eb/software/Anaconda3/5.0.1/conda-bld /anaconda-conda-bld - -ENV CONDA_PKGS_DIRS /anaconda/pkg-cache -ENV CONDA_BLD_PATH /anaconda-conda-bld - -RUN conda add --channels file://anaconda-conda-bld -RUN conda index /anaconda-conda-bld/linux-64 - -RUN cat ~/.condarc - -RUN conda install -y gencore_test +ARG CONDA_PKG=local +ENV CONDA_PKG ${CONDA_PKG} +RUN echo ${CONDA_PKG} diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index 7082cdf..0323b20 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -32,17 +32,17 @@ requirements: - r-base - r-essentials - r-cairo - - bioconductor-deseq2 - - bioconductor-edger >=3.14.0 - - bioconductor-cummerbund >=2.16.0 - - bioconductor-genomicranges >=1.26.1 - - bioconductor-gviz >=1.16.5 - - bioconductor-biovizbase >=1.18.0 - - bioconductor-s4vectors >=0.12.0 - - bioconductor-ballgown >=2.2.0 - - htseq ==0.6.1.post1 - - hisat2 ==2.0.4 - - stringtie ==1.3.0 + # - bioconductor-deseq2 + # - bioconductor-edger >=3.14.0 + # - bioconductor-cummerbund >=2.16.0 + # - bioconductor-genomicranges >=1.26.1 + # - bioconductor-gviz >=1.16.5 + # - bioconductor-biovizbase >=1.18.0 + # - bioconductor-s4vectors >=0.12.0 + # - bioconductor-ballgown >=2.2.0 + # - htseq ==0.6.1.post1 + # - hisat2 ==2.0.4 + # - stringtie ==1.3.0 test: commands: diff --git a/scripts/travis-run-conda-build-all.sh b/scripts/travis-run-conda-build-all.sh new file mode 100644 index 0000000..0852c44 --- /dev/null +++ b/scripts/travis-run-conda-build-all.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -euo pipefail +# Set some defaults +set +u +[[ -z $DOCKER_ARG ]] && DOCKER_ARG="" +[[ -z $TRAVIS ]] && TRAVIS="false" +[[ -z $BIOCONDA_UTILS_LINT_ARGS ]] && BIOCONDA_UTILS_LINT_ARGS="" +[[ -z $RANGE_ARG ]] && RANGE_ARG="--git-range master HEAD" +[[ -z $DISABLE_BIOCONDA_UTILS_BUILD_GIT_RANGE_CHECK ]] && DISABLE_BIOCONDA_UTILS_BUILD_GIT_RANGE_CHECK="false" +[[ -z $SKIP_LINTING ]] && SKIP_LINTING=false +set -u + +if [[ $TRAVIS_BRANCH != "master" && $TRAVIS_BRANCH != "bulk" && $TRAVIS_PULL_REQUEST == "false" && $TRAVIS_REPO_SLUG == "$MY_TRAVIS_REPO_SLUG" ]] +then + echo "" + echo "Tests are skipped for pushes to the main bioconda-recipes repo." + echo "If you have opened a pull request, please see the full tests for that PR." + echo "See https://bioconda.github.io/build-system.html for details" + echo "" + exit 0 +fi + +export PATH=/anaconda/bin:$PATH + +# TODO Add another upload +if [[ ( $TRAVIS_BRANCH == "master" || $TRAVIS_BRANCH == "bulk" ) && "$TRAVIS_PULL_REQUEST" == "false" && $TRAVIS_REPO_SLUG == "$MY_TRAVIS_REPO_SLUG" ]] +then + UPLOAD_ARG="--upload-channels nyuad-cgsb " +fi + + +if [[ $DISABLE_BIOCONDA_UTILS_BUILD_GIT_RANGE_CHECK == "true" ]] +then + echo + echo "DISABLE_BIOCONDA_UTILS_BUILD_GIT_RANGE_CHECK is true." + echo "A comprehensive check will be performed to see what needs to be built." + RANGE_ARG="" +fi + +set -x +conda-build-all recipes --inspect-channels nyuad-cgsb $UPLOAD_ARG +ls -lahR $CONDA_BLD_PATH + +docker images +set +x From 07eff50702ae07f7b4fe7d7c22197aa690008518 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 09:46:18 +0400 Subject: [PATCH 25/45] trying build with conda-build-all instead --- scripts/travis-run-conda-build-all.sh | 1 + scripts/travis-setup.sh | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/travis-run-conda-build-all.sh b/scripts/travis-run-conda-build-all.sh index 0852c44..c1282a6 100644 --- a/scripts/travis-run-conda-build-all.sh +++ b/scripts/travis-run-conda-build-all.sh @@ -38,6 +38,7 @@ then fi set -x +conda install -y conda-build-all conda-build-all recipes --inspect-channels nyuad-cgsb $UPLOAD_ARG ls -lahR $CONDA_BLD_PATH diff --git a/scripts/travis-setup.sh b/scripts/travis-setup.sh index c4b06a0..28b1bfa 100755 --- a/scripts/travis-setup.sh +++ b/scripts/travis-setup.sh @@ -29,8 +29,16 @@ sudo mkdir /anaconda sudo chown -R $USER /opt/anaconda3 sudo chown -R $USER /anaconda -$SCRIPT_DIR/../simulate-travis.py --bootstrap /anaconda --overwrite +wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh +bash miniconda.sh -b -p /anaconda3 +export PATH="/anaconda3/bin:$PATH" +hash -r +conda config --set always_yes yes --set changeps1 no +conda config --add channels r +conda config --add channels conda-forge +conda config --add channels bioconda + /anaconda/bin/conda index /anaconda/conda-bld/linux-64 /anaconda/conda-bld/osx-64 /anaconda/bin/conda config --add channels file://anaconda/conda-bld /anaconda/bin/conda install -y r-base r-essentials openjdk perl bioconductor-biobase nodejs -/anaconda/bin/npm install marked-man +/anaconda/bin/npm install -g marked-man From 113d6ccf454fbb990a0f22f34db996d20ea687a0 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 09:57:47 +0400 Subject: [PATCH 26/45] lets see if this works --- Dockerfile | 16 ++++++++++++++++ scripts/travis-run-conda-build-all.sh | 3 +++ 2 files changed, 19 insertions(+) diff --git a/Dockerfile b/Dockerfile index dc41f92..759f5a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,4 +2,20 @@ from continuumio/miniconda3 ARG CONDA_PKG=local ENV CONDA_PKG ${CONDA_PKG} + RUN echo ${CONDA_PKG} +RUN which conda +RUN conda info + + +## We have to add the pkgs and the build-system +COPY pkgs/* /opt/conda/pkgs/ +COPY conda-bld /opt/conda/conda-bld + +#RUN conda config --set always_yes yes --set changeps1 no +#RUN conda config --add channels r +#RUN conda config --add channels conda-forge +#RUN conda config --add channels bioconda +#RUN conda config --add channels nyuad-cgsb +# +# conda config --add channels file://anaconda-conda-bld/linux-64 diff --git a/scripts/travis-run-conda-build-all.sh b/scripts/travis-run-conda-build-all.sh index c1282a6..69be0a5 100644 --- a/scripts/travis-run-conda-build-all.sh +++ b/scripts/travis-run-conda-build-all.sh @@ -42,5 +42,8 @@ conda install -y conda-build-all conda-build-all recipes --inspect-channels nyuad-cgsb $UPLOAD_ARG ls -lahR $CONDA_BLD_PATH +cp Dockerfile /opt/anaconda3 +cd /opt/anaconda3 + docker images set +x From ac79e2435ab5ac41cb59a613fa3e0f282bfc64c2 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 10:00:20 +0400 Subject: [PATCH 27/45] lets try the whole workflow --- Dockerfile | 6 +++--- scripts/travis-run-conda-build-all.sh | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 759f5a2..c13d24b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,11 +6,11 @@ ENV CONDA_PKG ${CONDA_PKG} RUN echo ${CONDA_PKG} RUN which conda RUN conda info - +RUN mkdir -p /opt/conda/conda-bld/linux-64 ## We have to add the pkgs and the build-system COPY pkgs/* /opt/conda/pkgs/ -COPY conda-bld /opt/conda/conda-bld +COPY conda-bld/linux-64/${CONDA_PKG} /opt/conda/conda-bld/linux-64 #RUN conda config --set always_yes yes --set changeps1 no #RUN conda config --add channels r @@ -18,4 +18,4 @@ COPY conda-bld /opt/conda/conda-bld #RUN conda config --add channels bioconda #RUN conda config --add channels nyuad-cgsb # -# conda config --add channels file://anaconda-conda-bld/linux-64 +#conda config --add channels file://opt/conda/conda-bld/linux-64 diff --git a/scripts/travis-run-conda-build-all.sh b/scripts/travis-run-conda-build-all.sh index 69be0a5..35aabf7 100644 --- a/scripts/travis-run-conda-build-all.sh +++ b/scripts/travis-run-conda-build-all.sh @@ -44,6 +44,7 @@ ls -lahR $CONDA_BLD_PATH cp Dockerfile /opt/anaconda3 cd /opt/anaconda3 +docker build --build-arg CONDA_PKG='gencore_rnaseq-1.0-r3.4.1_1.tar.bz2' -t 'gencore_rnaseq:1.0-r3.4.1_1' docker images set +x From a0cb6655721bebaa29e35f2d601219b73f4bb922 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 10:00:59 +0400 Subject: [PATCH 28/45] trying again --- scripts/travis-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/travis-setup.sh b/scripts/travis-setup.sh index 28b1bfa..dc1f645 100755 --- a/scripts/travis-setup.sh +++ b/scripts/travis-setup.sh @@ -30,8 +30,8 @@ sudo chown -R $USER /opt/anaconda3 sudo chown -R $USER /anaconda wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh -bash miniconda.sh -b -p /anaconda3 -export PATH="/anaconda3/bin:$PATH" +bash miniconda.sh -b -p /anaconda +export PATH="/anaconda/bin:$PATH" hash -r conda config --set always_yes yes --set changeps1 no conda config --add channels r From d48195777cba66ac6ea4f670673b80b4d1620363 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 10:20:36 +0400 Subject: [PATCH 29/45] quiet wget --- scripts/travis-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/travis-setup.sh b/scripts/travis-setup.sh index dc1f645..0f46f0d 100755 --- a/scripts/travis-setup.sh +++ b/scripts/travis-setup.sh @@ -29,7 +29,7 @@ sudo mkdir /anaconda sudo chown -R $USER /opt/anaconda3 sudo chown -R $USER /anaconda -wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh +wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh --quiet -O miniconda.sh bash miniconda.sh -b -p /anaconda export PATH="/anaconda/bin:$PATH" hash -r From e70acd67408700e8a58f9cc4427a1f48827caedb Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 10:27:14 +0400 Subject: [PATCH 30/45] just install it to home --- scripts/travis-setup.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/travis-setup.sh b/scripts/travis-setup.sh index 0f46f0d..69db1dc 100755 --- a/scripts/travis-setup.sh +++ b/scripts/travis-setup.sh @@ -30,15 +30,15 @@ sudo chown -R $USER /opt/anaconda3 sudo chown -R $USER /anaconda wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh --quiet -O miniconda.sh -bash miniconda.sh -b -p /anaconda -export PATH="/anaconda/bin:$PATH" +bash miniconda.sh -b -p /home/travis/anaconda3 +export PATH="/home/travis/anaconda3/bin:$PATH" hash -r conda config --set always_yes yes --set changeps1 no conda config --add channels r conda config --add channels conda-forge conda config --add channels bioconda -/anaconda/bin/conda index /anaconda/conda-bld/linux-64 /anaconda/conda-bld/osx-64 -/anaconda/bin/conda config --add channels file://anaconda/conda-bld -/anaconda/bin/conda install -y r-base r-essentials openjdk perl bioconductor-biobase nodejs -/anaconda/bin/npm install -g marked-man +conda index /anaconda/conda-bld/linux-64 /anaconda/conda-bld/osx-64 +conda config --add channels file://anaconda/conda-bld +conda install -y r-base r-essentials openjdk perl bioconductor-biobase nodejs +npm install -g marked-man From 1fb00fb9a67836f45b432f6c102c6a3ab021075a Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 10:28:41 +0400 Subject: [PATCH 31/45] more cleanup --- scripts/travis-setup.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/travis-setup.sh b/scripts/travis-setup.sh index 69db1dc..8940238 100755 --- a/scripts/travis-setup.sh +++ b/scripts/travis-setup.sh @@ -24,10 +24,8 @@ done SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) pip install pyyaml -sudo mkdir /anaconda sudo chown -R $USER /opt/anaconda3 -sudo chown -R $USER /anaconda wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh --quiet -O miniconda.sh bash miniconda.sh -b -p /home/travis/anaconda3 @@ -37,8 +35,9 @@ conda config --set always_yes yes --set changeps1 no conda config --add channels r conda config --add channels conda-forge conda config --add channels bioconda +conda config --add channels nyuad-cgsb -conda index /anaconda/conda-bld/linux-64 /anaconda/conda-bld/osx-64 -conda config --add channels file://anaconda/conda-bld +conda index /home/travis/anaconda3/conda-bld/linux-64 /home/travis/anaconda3/conda-bld/osx-64 +conda config --add channels file://home/travis/anaconda3/conda-bld conda install -y r-base r-essentials openjdk perl bioconductor-biobase nodejs npm install -g marked-man From 83e737c7b180c617029e16e2c316cb9a80494e03 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 10:53:07 +0400 Subject: [PATCH 32/45] no index? --- scripts/travis-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/travis-setup.sh b/scripts/travis-setup.sh index 8940238..4ebec4f 100755 --- a/scripts/travis-setup.sh +++ b/scripts/travis-setup.sh @@ -37,7 +37,7 @@ conda config --add channels conda-forge conda config --add channels bioconda conda config --add channels nyuad-cgsb -conda index /home/travis/anaconda3/conda-bld/linux-64 /home/travis/anaconda3/conda-bld/osx-64 +# conda index /home/travis/anaconda3/conda-bld/linux-64 conda config --add channels file://home/travis/anaconda3/conda-bld conda install -y r-base r-essentials openjdk perl bioconductor-biobase nodejs npm install -g marked-man From ed5dd061848e70ad05347a8e089909b6845c5668 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 11:05:47 +0400 Subject: [PATCH 33/45] commiting --- scripts/travis-run-conda-build-all.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/travis-run-conda-build-all.sh b/scripts/travis-run-conda-build-all.sh index 35aabf7..3591c22 100644 --- a/scripts/travis-run-conda-build-all.sh +++ b/scripts/travis-run-conda-build-all.sh @@ -20,7 +20,6 @@ then exit 0 fi -export PATH=/anaconda/bin:$PATH # TODO Add another upload if [[ ( $TRAVIS_BRANCH == "master" || $TRAVIS_BRANCH == "bulk" ) && "$TRAVIS_PULL_REQUEST" == "false" && $TRAVIS_REPO_SLUG == "$MY_TRAVIS_REPO_SLUG" ]] @@ -38,6 +37,8 @@ then fi set -x + +export PATH=/home/travis/anaconda3/bin:$PATH conda install -y conda-build-all conda-build-all recipes --inspect-channels nyuad-cgsb $UPLOAD_ARG ls -lahR $CONDA_BLD_PATH From 8bc9143f4ace98b520286f9c0ce511ea2a9c0adc Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 11:13:48 +0400 Subject: [PATCH 34/45] it wasn't executable darn --- scripts/travis-run-conda-build-all.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/travis-run-conda-build-all.sh diff --git a/scripts/travis-run-conda-build-all.sh b/scripts/travis-run-conda-build-all.sh old mode 100644 new mode 100755 From ba20db2867d73cd02ab5be8e344a8c3a9270a4da Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 11:45:57 +0400 Subject: [PATCH 35/45] bah --- scripts/travis-run-conda-build-all.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/travis-run-conda-build-all.sh b/scripts/travis-run-conda-build-all.sh index 3591c22..fa6afe8 100755 --- a/scripts/travis-run-conda-build-all.sh +++ b/scripts/travis-run-conda-build-all.sh @@ -8,6 +8,7 @@ set +u [[ -z $RANGE_ARG ]] && RANGE_ARG="--git-range master HEAD" [[ -z $DISABLE_BIOCONDA_UTILS_BUILD_GIT_RANGE_CHECK ]] && DISABLE_BIOCONDA_UTILS_BUILD_GIT_RANGE_CHECK="false" [[ -z $SKIP_LINTING ]] && SKIP_LINTING=false +[[ -z $UPLOAD_ARG ]] && UPLOAD_ARG="" set -u if [[ $TRAVIS_BRANCH != "master" && $TRAVIS_BRANCH != "bulk" && $TRAVIS_PULL_REQUEST == "false" && $TRAVIS_REPO_SLUG == "$MY_TRAVIS_REPO_SLUG" ]] From 4c2c03a72c8143861e6ff5a3e7cd799d952852e3 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 12:02:52 +0400 Subject: [PATCH 36/45] adding in some conda build args --- .travis.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.travis.yml b/.travis.yml index 186e566..abcba99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,18 @@ env: - CONTAINER_NAMESPACE: 'nyuad_cgsb' - CONDA_PKGS_DIRS: '/opt/anaconda3/pkgs' - CONDA_BLD_PATH: '/opt/anaconda3/conda-bld' + - CONDA_R: '3.4.1' + - CONDA_PERL: '5.22.0' + - CONDA_HTSLIB: "1.6" + - CONDA_BOOST: "1.64" + - CONDA_NPY: "112" + - CONDA_NCURSES: "5.9" + - CONDA_GSL: "1.16" + - CONDA_GMP: "5.1" + - CONDA_HDF5: "1.8.17" + - CONDA_ZLIB: "1.2.8" + - CONDA_BZIP2: "1.0" + - CONDA_XZ: "5.2" # - SUBDAGS=1 # - BIOCONDA_UTILS_TAG=master # - BIOCONDA_UTILS_BUILD_ARGS="--loglevel=debug" From 9b134fe14368a3b7a99e086fedecea4b7fd34dcc Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 12:03:32 +0400 Subject: [PATCH 37/45] removing build --- scripts/travis-run-conda-build-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/travis-run-conda-build-all.sh b/scripts/travis-run-conda-build-all.sh index fa6afe8..fbfa604 100755 --- a/scripts/travis-run-conda-build-all.sh +++ b/scripts/travis-run-conda-build-all.sh @@ -46,7 +46,7 @@ ls -lahR $CONDA_BLD_PATH cp Dockerfile /opt/anaconda3 cd /opt/anaconda3 -docker build --build-arg CONDA_PKG='gencore_rnaseq-1.0-r3.4.1_1.tar.bz2' -t 'gencore_rnaseq:1.0-r3.4.1_1' +docker build --build-arg CONDA_PKG='gencore_rnaseq-1.0-r3.4.1_1.tar.bz2' docker images set +x From 01e5bad506a8393e7898489f8910bec4b63b9449 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 13:26:54 +0400 Subject: [PATCH 38/45] try try again --- scripts/travis-run-conda-build-all.sh | 4 +++- scripts/travis-setup.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/travis-run-conda-build-all.sh b/scripts/travis-run-conda-build-all.sh index fbfa604..7c32cbf 100755 --- a/scripts/travis-run-conda-build-all.sh +++ b/scripts/travis-run-conda-build-all.sh @@ -44,9 +44,11 @@ conda install -y conda-build-all conda-build-all recipes --inspect-channels nyuad-cgsb $UPLOAD_ARG ls -lahR $CONDA_BLD_PATH +echo "We should be building things now...." + cp Dockerfile /opt/anaconda3 cd /opt/anaconda3 -docker build --build-arg CONDA_PKG='gencore_rnaseq-1.0-r3.4.1_1.tar.bz2' +travis_wait 30 docker build --build-arg CONDA_PKG='gencore_rnaseq-1.0-r3.4.1_1.tar.bz2' docker images set +x diff --git a/scripts/travis-setup.sh b/scripts/travis-setup.sh index 4ebec4f..5be81e0 100755 --- a/scripts/travis-setup.sh +++ b/scripts/travis-setup.sh @@ -38,6 +38,6 @@ conda config --add channels bioconda conda config --add channels nyuad-cgsb # conda index /home/travis/anaconda3/conda-bld/linux-64 -conda config --add channels file://home/travis/anaconda3/conda-bld +# conda config --add channels file://home/travis/anaconda3/conda-bld conda install -y r-base r-essentials openjdk perl bioconductor-biobase nodejs npm install -g marked-man From a7493b27de9c93fab24e7fe07ebe358f5fe31caf Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 13:53:26 +0400 Subject: [PATCH 39/45] baaaah travis times out --- recipes/gencore_rnaseq/1.0/meta.yaml | 34 +++++++++++++-------------- scripts/travis-run-conda-build-all.sh | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index 0323b20..06ce4e8 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -14,24 +14,24 @@ requirements: ##Alignment - samtools ==1.3.1 - kallisto ==0.43.0 - - cufflinks-patch ==2.2.1 - - bowtie2 ==2.2.8 - - tophat ==2.1.0 - - star ==2.5.2a - - bcftools ==1.3.1 - - blat ==35 - - gatk ==3.6 - - picard ==2.5.0 - - prinseq ==0.20.4 - - vcftools ==0.1.14 - - bedtools ==2.25.0 - - vcflib ==1.0.0_rc1 - - bamtools ==2.4.0 - - seqtk ==1.2 + # - cufflinks-patch ==2.2.1 + # - bowtie2 ==2.2.8 + # - tophat ==2.1.0 + # - star ==2.5.2a + # - bcftools ==1.3.1 + # - blat ==35 + # - gatk ==3.6 + # - picard ==2.5.0 + # - prinseq ==0.20.4 + # - vcftools ==0.1.14 + # - bedtools ==2.25.0 + # - vcflib ==1.0.0_rc1 + # - bamtools ==2.4.0 + # - seqtk ==1.2 ##Downstream processing - - r-base - - r-essentials - - r-cairo + # - r-base + # - r-essentials + # - r-cairo # - bioconductor-deseq2 # - bioconductor-edger >=3.14.0 # - bioconductor-cummerbund >=2.16.0 diff --git a/scripts/travis-run-conda-build-all.sh b/scripts/travis-run-conda-build-all.sh index 7c32cbf..ec8a7a7 100755 --- a/scripts/travis-run-conda-build-all.sh +++ b/scripts/travis-run-conda-build-all.sh @@ -41,7 +41,7 @@ set -x export PATH=/home/travis/anaconda3/bin:$PATH conda install -y conda-build-all -conda-build-all recipes --inspect-channels nyuad-cgsb $UPLOAD_ARG +travis_wait 30 conda-build-all recipes --inspect-channels nyuad-cgsb $UPLOAD_ARG ls -lahR $CONDA_BLD_PATH echo "We should be building things now...." From 57552ee950f1eb3e430b4667435e2fff7e7b1b0c Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 14:08:48 +0400 Subject: [PATCH 40/45] WHERE IST HE TRAVIS WAIT --- .travis.yml | 4 ++-- scripts/travis-run-conda-build-all.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index abcba99..85b13bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,11 +11,11 @@ jobs: - stage: Linux if: "(type = push) AND ((branch IN (master, bulk)) OR (fork = true))" os: linux - script: scripts/travis-run-conda-build-all.sh + script: travis_wait 30 scripts/travis-run-conda-build-all.sh - stage: Linux if: type = pull_request os: linux - script: scripts/travis-run-conda-build-all.sh + script: travis_wait 30 scripts/travis-run-conda-build-all.sh env: global: - MINICONDA_VER: '4.3.21' diff --git a/scripts/travis-run-conda-build-all.sh b/scripts/travis-run-conda-build-all.sh index ec8a7a7..48c1cf8 100755 --- a/scripts/travis-run-conda-build-all.sh +++ b/scripts/travis-run-conda-build-all.sh @@ -41,14 +41,14 @@ set -x export PATH=/home/travis/anaconda3/bin:$PATH conda install -y conda-build-all -travis_wait 30 conda-build-all recipes --inspect-channels nyuad-cgsb $UPLOAD_ARG +conda-build-all recipes --inspect-channels nyuad-cgsb $UPLOAD_ARG ls -lahR $CONDA_BLD_PATH echo "We should be building things now...." cp Dockerfile /opt/anaconda3 cd /opt/anaconda3 -travis_wait 30 docker build --build-arg CONDA_PKG='gencore_rnaseq-1.0-r3.4.1_1.tar.bz2' +docker build --build-arg CONDA_PKG='gencore_rnaseq-1.0-r3.4.1_1.tar.bz2' docker images set +x From 71962d147ca962c56c1588c8b6d029d88571bea7 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 14:21:17 +0400 Subject: [PATCH 41/45] stupid docker build --- scripts/travis-run-conda-build-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/travis-run-conda-build-all.sh b/scripts/travis-run-conda-build-all.sh index 48c1cf8..baad27a 100755 --- a/scripts/travis-run-conda-build-all.sh +++ b/scripts/travis-run-conda-build-all.sh @@ -48,7 +48,7 @@ echo "We should be building things now...." cp Dockerfile /opt/anaconda3 cd /opt/anaconda3 -docker build --build-arg CONDA_PKG='gencore_rnaseq-1.0-r3.4.1_1.tar.bz2' +docker build --build-arg CONDA_PKG='gencore_rnaseq-1.0-r3.4.1_1.tar.bz2' . docker images set +x From afca61fbf5f77961bd77e8d81e7379d10dc48214 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 14:21:47 +0400 Subject: [PATCH 42/45] lets try this --- scripts/travis-run-conda-build-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/travis-run-conda-build-all.sh b/scripts/travis-run-conda-build-all.sh index baad27a..b077c0b 100755 --- a/scripts/travis-run-conda-build-all.sh +++ b/scripts/travis-run-conda-build-all.sh @@ -48,7 +48,7 @@ echo "We should be building things now...." cp Dockerfile /opt/anaconda3 cd /opt/anaconda3 -docker build --build-arg CONDA_PKG='gencore_rnaseq-1.0-r3.4.1_1.tar.bz2' . +docker build --build-arg CONDA_PKG='gencore_rnaseq-1.0-r3.4.1_1.tar.bz2' -t 'gencore_rnaseq:1.0-r3.4.1_1' . docker images set +x From 02d9cba078e43032e7f063cc2d367d1aee020ccb Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 14:22:43 +0400 Subject: [PATCH 43/45] need R for build string --- recipes/gencore_rnaseq/1.0/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/gencore_rnaseq/1.0/meta.yaml b/recipes/gencore_rnaseq/1.0/meta.yaml index 06ce4e8..743c9db 100644 --- a/recipes/gencore_rnaseq/1.0/meta.yaml +++ b/recipes/gencore_rnaseq/1.0/meta.yaml @@ -29,7 +29,7 @@ requirements: # - bamtools ==2.4.0 # - seqtk ==1.2 ##Downstream processing - # - r-base + - r-base # - r-essentials # - r-cairo # - bioconductor-deseq2 From 02950555d203d83f186eb0be189f8279be12fe7e Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 14:41:20 +0400 Subject: [PATCH 44/45] try the whole dockerfile workflow --- Dockerfile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index c13d24b..0830168 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,10 +12,11 @@ RUN mkdir -p /opt/conda/conda-bld/linux-64 COPY pkgs/* /opt/conda/pkgs/ COPY conda-bld/linux-64/${CONDA_PKG} /opt/conda/conda-bld/linux-64 -#RUN conda config --set always_yes yes --set changeps1 no -#RUN conda config --add channels r -#RUN conda config --add channels conda-forge -#RUN conda config --add channels bioconda -#RUN conda config --add channels nyuad-cgsb -# -#conda config --add channels file://opt/conda/conda-bld/linux-64 +RUN conda config --set always_yes yes --set changeps1 no +RUN conda config --add channels r +RUN conda config --add channels conda-forge +RUN conda config --add channels bioconda +RUN conda config --add channels nyuad-cgsb + +RUN conda config --add channels file://opt/conda/conda-bld/linux-64 +RUN conda install -y --use-local gencore_rnaseq From b4a4f1b2fef438e590c1f1f946f4f21c961b7b39 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Wed, 13 Dec 2017 15:09:26 +0400 Subject: [PATCH 45/45] index the channel --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0830168..880e846 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,5 +18,7 @@ RUN conda config --add channels conda-forge RUN conda config --add channels bioconda RUN conda config --add channels nyuad-cgsb +RUN ls -lah /opt/conda/conda-bld/linux-64 RUN conda config --add channels file://opt/conda/conda-bld/linux-64 -RUN conda install -y --use-local gencore_rnaseq +RUN conda index /opt/conda/conda-bld/linux-64 +RUN conda install -y --use-local gencore_rnaseq