Skip to content

Commit 62f4675

Browse files
NortamoNortamo
authored
Merge pull request #20 from CSCfi/no_hardcoded
Don't hardcode path to singularity
2 parents 2255b4d + 4006ed7 commit 62f4675

File tree

9 files changed

+80
-12
lines changed

9 files changed

+80
-12
lines changed

‎configs/local.yaml‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,18 @@ force:
3333
conda_arch: 'Linux-x86_64'
3434
max_num_cpus: 4
3535
# No logic currently to refer to other variables here
36-
# So either implement that, throw them to $TMPDIR
37-
#
36+
# So either implement that, throw them to $TMPDIR
3837
workdir: $PWD
38+
# What executable name or path to use for singularity
39+
# For fully dynamic behavior
40+
# You can just set singularity or apptainer here.
41+
# Keep in mind that this could break installations
42+
# if e.g the user installs singularity via conda
43+
# when user namespaces are not enabled
44+
# Environment variables can be used here
45+
# but they will be expanded during build,
46+
# not during runtime
47+
singularity_executable_path: "singularity"
3948

4049
# Value will override any user set value
4150

‎configs/lumi.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,16 @@ force:
4141
# So either implement that, throw them to $TMPDIR
4242
#
4343
workdir: $PWD
44+
# What executable name or path to use for singularity
45+
# For fully dynamic behavior
46+
# You can just set singularity or apptainer here.
47+
# Keep in mind that this could break installations
48+
# if e.g the user installs singularity via conda
49+
# when user namespaces are not enabled
50+
# Environment variables can be used here
51+
# but they will be expanded during build,
52+
# not during runtime
53+
singularity_executable_path: "/usr/bin/singularity"
4454

4555
# Value will override any user set value
4656

‎configs/mahti.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,16 @@ force:
3939
# So either implement that, throw them to $TMPDIR
4040
#
4141
workdir: $PWD
42+
# What executable name or path to use for singularity
43+
# For fully dynamic behavior
44+
# You can just set singularity or apptainer here.
45+
# Keep in mind that this could break installations
46+
# if e.g the user installs singularity via conda
47+
# when user namespaces are not enabled
48+
# Environment variables can be used here
49+
# but they will be expanded during build,
50+
# not during runtime
51+
singularity_executable_path: "/usr/bin/singularity"
4252

4353
# Value will override any user set value
4454

‎configs/puhti.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,16 @@ force:
4040
# So either implement that, throw them to $TMPDIR
4141
#
4242
workdir: $PWD
43+
# What executable name or path to use for singularity
44+
# For fully dynamic behavior
45+
# You can just set singularity or apptainer here.
46+
# Keep in mind that this could break installations
47+
# if e.g the user installs singularity via conda
48+
# when user namespaces are not enabled
49+
# Environment variables can be used here
50+
# but they will be expanded during build,
51+
# not during runtime
52+
singularity_executable_path: "/usr/bin/singularity"
4353

4454
# Value will override any user set value
4555

‎create_inst.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,18 +54,18 @@ export install_root=$CW_INSTALLATION_PATH
5454

5555

5656
if [[ "$CW_UPDATE_INSTALLATION" == "yes" ]];then
57-
_CONTAINER_EXEC="singularity --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_SOURCE_MOUNT_POINT:image-src=/ _deploy/$CW_CONTAINER_IMAGE"
57+
_CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_SOURCE_MOUNT_POINT:image-src=/ _deploy/$CW_CONTAINER_IMAGE"
5858
export SINGULARITY_BIND="$SINGULARITY_BIND,$PWD/_inst_dir:$CW_INSTALLATION_PATH,$_inst_path/_bin:$_inst_path/bin"
5959
print_info "Copying installation to writable area, might take a while" 1
6060
print_info "$(readlink -f $CW_INSTALLATION_PREFIX)" 1
6161
$_CONTAINER_EXEC cp -a $CW_SOURCE_MOUNT_POINT/. $CW_INSTALLATION_PATH || { print_err "Failed to copy some files, most likely incorrect file permissions inside the squashfs image" && false ; }
6262
elif [[ "$CW_MODE" == "wrapdisk" ]];then
6363
export SINGULARITY_BIND="$SINGULARITY_BIND,$PWD/_inst_dir:$CW_INSTALLATION_PATH,$CW_WRAP_SRC:$CW_SOURCE_MOUNT_POINT"
64-
_CONTAINER_EXEC="singularity --silent exec _deploy/$CW_CONTAINER_IMAGE"
64+
_CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec _deploy/$CW_CONTAINER_IMAGE"
6565

6666
else
6767
export SINGULARITY_BIND="$SINGULARITY_BIND,$PWD/_inst_dir:$CW_INSTALLATION_PATH"
68-
_CONTAINER_EXEC="singularity --silent exec _deploy/$CW_CONTAINER_IMAGE"
68+
_CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec _deploy/$CW_CONTAINER_IMAGE"
6969
fi
7070
cp ./_sing_inst_script.sh _pre_install.sh _post_install.sh _inst_dir
7171
print_info "Running installation script" 1

‎frontends/containerize‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,19 @@ else
9898
export _inst_path=$(realpath -s $PWD/$CW_INSTALLATION_PREFIX )
9999
fi
100100

101+
if ! command -v $CW_SINGULARITY_EXECUTABLE_PATH &>/dev/null; then
102+
{ print_err "The command $CW_SINGULARITY_EXECUTABLE_PATH does not exists
103+
Verify that the value for 'singularity_executable_path' in $CW_GLOBAL_YAML is correct"; false ;}
104+
fi
105+
106+
print_info "Validating the set singularity command $CW_SINGULARITY_EXECUTABLE_PATH" 2
107+
num_res=$($CW_SINGULARITY_EXECUTABLE_PATH --help | grep "build\|exec\|shell" 2>/dev/null | wc -l)
108+
if [[ "$num_res" -lt 3 ]];then
109+
{ print_err "$CW_SINGULARITY_EXECUTABLE_PATH does not seem to be a valid apptainer/singularity executable"; false ;}
110+
fi
111+
112+
113+
101114
$M_SCRIPT_DIR/../pre.sh || { print_err "Failed getting container image"; false ;}
102115
# Nothing to do if we are just wrapping a container
103116
if [[ ! "$CW_MODE" == "wrapcont" ]];then

‎generate_wrappers.sh‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,14 @@ echo "#!/bin/bash" > _deploy/common.sh
1515

1616

1717
if [[ "$CW_MODE" == "wrapcont" ]];then
18-
_CONTAINER_EXEC="/usr/bin/singularity --silent exec _deploy/$CW_CONTAINER_IMAGE"
19-
_RUN_CMD="/usr/bin/singularity --silent exec \$DIR/../\$CONTAINER_IMAGE"
20-
_SHELL_CMD="/usr/bin/singularity --silent shell \$DIR/../\$CONTAINER_IMAGE"
18+
_CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec _deploy/$CW_CONTAINER_IMAGE"
19+
_RUN_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec \$DIR/../\$CONTAINER_IMAGE"
20+
_SHELL_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent shell \$DIR/../\$CONTAINER_IMAGE"
2121
else
22-
_CONTAINER_EXEC="/usr/bin/singularity --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_INSTALLATION_PATH:image-src=/ _deploy/$CW_CONTAINER_IMAGE"
22+
_CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_INSTALLATION_PATH:image-src=/ _deploy/$CW_CONTAINER_IMAGE"
2323
echo "SQFS_IMAGE=$CW_SQFS_IMAGE" >> _deploy/common.sh
24-
_RUN_CMD="/usr/bin/singularity --silent exec \$DIR/../\$CONTAINER_IMAGE"
25-
_SHELL_CMD="/usr/bin/singularity --silent shell \$DIR/../\$CONTAINER_IMAGE"
24+
_RUN_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec \$DIR/../\$CONTAINER_IMAGE"
25+
_SHELL_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent shell \$DIR/../\$CONTAINER_IMAGE"
2626
fi
2727

2828
# Need to unset the path, otherwise we might be stuck in a nasty loop

‎pre.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,11 @@ mkdir -p $CW_BUILD_TMPDIR/_deploy/
1414

1515
if [[ ! -e $CW_CONTAINER_SRC ]]; then
1616
print_info "Fetching container $CW_CONTAINER_SRC" 1
17-
singularity --silent pull $CW_BUILD_TMPDIR/_deploy/$CW_CONTAINER_IMAGE $CW_CONTAINER_SRC || \
17+
$CW_SINGULARITY_EXECUTABLE_PATH --silent pull $CW_BUILD_TMPDIR/_deploy/$CW_CONTAINER_IMAGE $CW_CONTAINER_SRC || \
1818
{ print_err "Failed fetching container"; exit 1 ;}
19+
if [[ ! -e $CW_BUILD_TMPDIR/_deploy/$CW_CONTAINER_IMAGE ]];then
20+
{ print_err "$CW_SINGULARITY_EXECUTABLE_PATH pull returned success, but no container was fetched"; exit 1 ;}
21+
fi
1922
else
2023
if [[ ${CW_SHARE_CONTAINER+defined} && ${CW_SHARE_CONTAINER} == "yes" ]];then
2124
print_info "Using container $CW_CONTAINER_SRC" 1

‎tests/test-03.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,19 @@ rule ghelp:
3232
"""
3333
' > Snakefile
3434

35+
cat ../../default_config/config.yaml | sed "s@singularity_executable_path.*@singularity_executable_path: 'ThisIsNotACommand'@g" > my_config.yaml
36+
export CW_GLOBAL_YAML=$( readlink -f my_config.yaml)
37+
t_run "conda-containerize new --mamba env2.yml --prefix S | grep 'ThisIsNotACommand does not exists'" "Exit if configured singularity command does not exist"
38+
cat ../../default_config/config.yaml | sed "s@singularity_executable_path.*@singularity_executable_path: 'my_sing_command'@g" > my_config.yaml
39+
echo "#!/bin/bash" > my_sing_command
40+
echo "exit 0" >> my_sing_command
41+
chmod +x my_sing_command
42+
export PATH=$PATH:$PWD
43+
t_run "conda-containerize new --mamba env2.yml --prefix S | grep 'does not seem to be a valid apptainer/singularity executable'" "Exit if configured singularity command seems broken"
44+
# Run the rest of the test with singularity found from path.
45+
cat ../../default_config/config.yaml | sed "s@singularity_executable_path.*@singularity_executable_path: 'singularity'@g" > my_config.yaml
46+
47+
3548
t_run "conda-containerize new --mamba env2.yml --prefix S" "mamba works"
3649
t_run "conda-containerize new --mamba env.yml --prefix Gdal" "gdal installed"
3750
export OPATH=$PATH

0 commit comments

Comments
 (0)