Merge pull request #25 from xavierabellan/master

Replace miniconda by miniforge to avoid licensing issues,
Fix previously undetected leakage of ~/.conda/environments.txt

Author: Nortamo

README.md

@@ -136,7 +136,7 @@ on the sqfs image itself. This behavior can be disabled by setting `CW_NO_FIX_PE
 **BASH MAGIC**
 
 ```
-/usr/bin/singularity --silent exec -B $DIR/../$SQFS_IMAGE:$INSTALLATION_PATH:image-src=/ $DIR/../$CONTAINER_IMAGE  bash -c "eval \"\$(/CSC_CONTAINER/miniconda/bin/conda shell.bash hook )\"  && conda activate env1 &>/dev/null &&  exec -a $_O_SOURCE $DIR/python $( test $# -eq 0 || printf " %q" "$@" )"
+/usr/bin/singularity --silent exec -B $DIR/../$SQFS_IMAGE:$INSTALLATION_PATH:image-src=/ $DIR/../$CONTAINER_IMAGE  bash -c "eval \"\$(/CSC_CONTAINER/miniforge/bin/conda shell.bash hook )\"  && conda activate env1 &>/dev/null &&  exec -a $_O_SOURCE $DIR/python $( test $# -eq 0 || printf " %q" "$@" )"
 ```
 
 - `exec -a` is there to enable creating virtual environments as otherwise the venv symlink -> tykky wrapper -> executable in container chain is broken and python does not consider itself to be a venv. 

create_inst.sh

@@ -40,11 +40,10 @@ SINGULARITY_BIND="$SINGULARITY_BIND,/tmp"
 # By default we want to disable the user condarc as that
 # might interfere with the installation
 if [[ ! ${CW_ENABLE_CONDARC+defined} ]]; then
-    echo "pkgs_dirs: 
-        - $CW_INSTALLATION_PATH/miniconda/pkgs
-    " > $PWD/_inst_dir/condarc_override
-    SINGULARITY_BIND="$SINGULARITY_BIND,$PWD/_inst_dir/condarc_override:$(readlink -f $HOME/.condarc)"
+    export CONDA_PKGS_DIRS=$CW_INSTALLATION_PATH/miniforge/pkgs
 fi
+mkdir -p $CW_BUILD_TMPDIR/.conda_override
+SINGULARITY_BIND="$SINGULARITY_BIND,$CW_BUILD_TMPDIR/.conda_override:$(readlink -f $HOME/.conda)"
 export SINGULARITY_BIND
 echo "export install_root=$CW_INSTALLATION_PATH" >> _extra_envs.sh
 echo "export install_root=$CW_INSTALLATION_PATH" >> _vars.sh

templates/conda.sh

@@ -3,22 +3,25 @@ set -e
 
 
 cd  $CW_BUILD_TMPDIR
-echo "export env_root=$CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/" >> _extra_envs.sh
-echo "export env_root=$CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/" >> _vars.sh
-export env_root=$CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/
+echo "export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/" >> _extra_envs.sh
+echo "export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/" >> _vars.sh
+export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/
 
 cd $CW_INSTALLATION_PATH
 
+[ "$CW_CONDA_VERSION" = "latest" ] && CW_CONDA_VERSION=$(curl -s https://api.github.com/repos/conda-forge/miniforge/releases/latest | grep "tag_name" | cut -d: -f2 | tr -d \" | tr -d , | tr -d " ")
 
-print_info "Using miniconda version Miniconda3-$CW_CONDA_VERSION-$CW_CONDA_ARCH" 1
-print_info "Downloading miniconda " 2
-curl https://repo.anaconda.com/miniconda/Miniconda3-$CW_CONDA_VERSION-$CW_CONDA_ARCH.sh --output Miniconda_inst.sh &>/dev/null 
-print_info "Installing miniconda " 1 
-bash Miniconda_inst.sh -b -p $CW_INSTALLATION_PATH/miniconda  > $CW_BUILD_TMPDIR/_inst_miniconda.log &   
+print_info "Using miniforge version Miniforge3-$CW_CONDA_VERSION-$CW_CONDA_ARCH" 1
+print_info "Downloading miniforge " 2
+curl -sL https://github.com/conda-forge/miniforge/releases/download/$CW_CONDA_VERSION/Miniforge3-$CW_CONDA_VERSION-$CW_CONDA_ARCH.sh --output Miniforge_inst.sh &>/dev/null
+print_info "Installing miniforge " 1
+bash Miniforge_inst.sh -b -p $CW_INSTALLATION_PATH/miniforge > $CW_BUILD_TMPDIR/_inst_miniforge.log &
 inst_pid=$!
-follow_log $inst_pid $CW_BUILD_TMPDIR/_inst_miniconda.log 20
-rm Miniconda_inst.sh
-eval "$($CW_INSTALLATION_PATH/miniconda/bin/conda shell.bash hook)"
+
+follow_log $inst_pid $CW_BUILD_TMPDIR/_inst_miniforge.log 20
+rm Miniforge_inst.sh
+eval "$($CW_INSTALLATION_PATH/miniforge/bin/conda shell.bash hook)"
+
 cd $CW_WORKDIR
 source $CW_INSTALLATION_PATH/_pre_install.sh
 if [[ ! -z "$(echo "$CW_ENV_FILE" | grep ".*\.yaml\|.*\.yml")" ]];then 
@@ -32,7 +35,6 @@ print_info "Creating env, full log in $CW_BUILD_TMPDIR/build.log" 1
 
 if [[ ${CW_MAMBA} == "yes" ]] ;then
     print_info "Using mamba to install packages" 1 
-    conda install -y mamba -n base -c conda-forge
     mamba $_EC create --name $CW_ENV_NAME $_FF $( basename $CW_ENV_FILE ) &>> $CW_BUILD_TMPDIR/build.log &
 else
     conda $_EC create --name $CW_ENV_NAME $_FF $( basename $CW_ENV_FILE ) &>> $CW_BUILD_TMPDIR/build.log &
@@ -48,16 +50,16 @@ fi
 cd $CW_WORKDIR
 print_info "Running user supplied commands" 1
 source $CW_INSTALLATION_PATH/_post_install.sh
-if [[ -d $CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/bin/ ]];then
-    echo 'echo "' > $CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/bin/list-packages
-    conda list >> $CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/bin/list-packages 
-    echo '"' >> $CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/bin/list-packages 
-    chmod +x $CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/bin/list-packages 
+if [[ -d $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/ ]];then
+    echo 'echo "' > $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages
+    conda list >> $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages
+    echo '"' >> $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages
+    chmod +x $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages
 else
     print_warn "Created env is empty"
 fi
 
 
 # Set here as they are dynamic
 # Could also set them in construct.py...
-echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/bin/\" )" >>  $CW_BUILD_TMPDIR/_vars.sh
+echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/\" )" >>  $CW_BUILD_TMPDIR/_vars.sh

templates/conda_modify.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 cd  $CW_BUILD_TMPDIR
-echo "export env_root=$CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/" >> _extra_envs.sh
-echo "export env_root=$CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/" >> _vars.sh
+echo "export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/" >> _extra_envs.sh
+echo "export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/" >> _vars.sh
 cd $CW_INSTALLATION_PATH
-export env_root=$CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/
-eval "$($CW_INSTALLATION_PATH/miniconda/bin/conda shell.bash hook)"
+export env_root=$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/
+eval "$($CW_INSTALLATION_PATH/miniforge/bin/conda shell.bash hook)"
 cd $CW_WORKDIR
 source $CW_INSTALLATION_PATH/_pre_install.sh
 conda activate $CW_ENV_NAME
@@ -17,9 +17,9 @@ if [[ ${CW_REQUIREMENTS_FILE+defined}  ]];then
 fi
 cd $CW_WORKDIR
 source $CW_INSTALLATION_PATH/_post_install.sh
-echo 'echo "' > $CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/bin/list-packages
-conda list >> $CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/bin/list-packages 
-echo '"' >> $CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/bin/list-packages 
-chmod +x $CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/bin/list-packages 
+echo 'echo "' > $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages
+conda list >> $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages
+echo '"' >> $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages
+chmod +x $CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/list-packages
 
-echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/miniconda/envs/$CW_ENV_NAME/bin/\" )" >>  $CW_BUILD_TMPDIR/_vars.sh
+echo "CW_WRAPPER_PATHS+=( \"$CW_INSTALLATION_PATH/miniforge/envs/$CW_ENV_NAME/bin/\" )" >>  $CW_BUILD_TMPDIR/_vars.sh

tests/tests.sh

@@ -65,7 +65,7 @@ t_run "conda-containerize new conda_base.yml --prefix A_DIR_NO_EXE | grep ERROR"
 
 mkdir CONDA_INSTALL_DIR
 
-t_run "conda-containerize new conda_broken.yaml --prefix CONDA_INSTALL_DIR | tee conda_inst.out | grep 'ResolvePackageNotFound'" "Conda errors are propagated to the user"
+t_run "conda-containerize new conda_broken.yaml --prefix CONDA_INSTALL_DIR | tee conda_inst.out | grep 'ResolvePackageNotFound\|PackagesNotFoundError'" "Conda errors are propagated to the user"
 t_run "grep ERROR conda_inst.out" "Failed run contains error" 
 t_run "grep INFO conda_inst.out"  "Info is present"
 t_run "test -z \"\$(grep ' DEBUG ' conda_inst.out )\" "  "Default no debug message"