diff --git a/.gitignore b/.gitignore index 3e0406724..22609e17e 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,6 @@ build/ # setuptools-scm constructor/_version.py + +# Temporary workspaces +tmp/ diff --git a/constructor/header.sh b/constructor/header.sh index 8dfd30c32..e11d5cfd9 100644 --- a/constructor/header.sh +++ b/constructor/header.sh @@ -598,8 +598,7 @@ CONDA_EXTRA_SAFETY_CHECKS=no \ CONDA_CHANNELS="{{ channels }}" \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ CONDA_QUIET="$BATCH" \ -"$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" $shortcuts {{ no_rcs_arg }} || exit 1 -rm -f "$PREFIX/pkgs/env.txt" +"$CONDA_EXEC" install --offline --file "$PREFIX/conda-meta/initial-state.explicit.txt" -yp "$PREFIX" $shortcuts {{ no_rcs_arg }} || exit 1 {%- if has_conda %} mkdir -p "$PREFIX/envs" @@ -638,8 +637,7 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do CONDA_CHANNELS="$env_channels" \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ CONDA_QUIET="$BATCH" \ - "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1 - rm -f "${env_pkgs}env.txt" + "$CONDA_EXEC" install --offline --file "$PREFIX/envs/$env_name/conda-meta/initial-state.explicit.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1 done {%- endif %} diff --git a/constructor/nsis/main.nsi.tmpl b/constructor/nsis/main.nsi.tmpl index 6dec989ef..c14c6fd19 100644 --- a/constructor/nsis/main.nsi.tmpl +++ b/constructor/nsis/main.nsi.tmpl @@ -1375,13 +1375,12 @@ Section "Install" ${Print} "Setting up the {{ env.name }} environment..." SetDetailsPrint listonly - # List of packages to install - SetOutPath "{{ env.env_txt_dir }}" - File "{{ env.env_txt_abspath }}" # A conda-meta\history file is required for a valid conda prefix SetOutPath "{{ env.conda_meta }}" File "{{ env.history_abspath }}" + # List of packages to install, as a lockfile + File "{{ env.lockfile_txt_abspath }}" # Set channels System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_CHANNELS", "{{ channels }}").r0' @@ -1391,10 +1390,10 @@ Section "Install" # Run conda install ${If} $Ana_CreateShortcuts_State = ${BST_CHECKED} ${Print} "Installing packages for {{ env.name }}, creating shortcuts if necessary..." - push '"$INSTDIR\_conda.exe" install --offline -yp "{{ env.prefix }}" --file "{{ env.env_txt }}" {{ env.shortcuts }} {{ env.no_rcs_arg }}' + push '"$INSTDIR\_conda.exe" install --offline -yp "{{ env.prefix }}" --file "{{ env.lockfile_txt }}" {{ env.shortcuts }} {{ env.no_rcs_arg }}' ${Else} ${Print} "Installing packages for {{ env.name }}..." - push '"$INSTDIR\_conda.exe" install --offline -yp "{{ env.prefix }}" --file "{{ env.env_txt }}" --no-shortcuts {{ env.no_rcs_arg }}' + push '"$INSTDIR\_conda.exe" install --offline -yp "{{ env.prefix }}" --file "{{ env.lockfile_txt }}" --no-shortcuts {{ env.no_rcs_arg }}' ${EndIf} push 'Failed to link extracted packages to {{ env.prefix }}!' push 'WithLog' @@ -1402,10 +1401,6 @@ Section "Install" call AbortRetryNSExecWait SetDetailsPrint both - # Cleanup {{ env.name }} env.txt - SetOutPath "$INSTDIR" - Delete "{{ env.env_txt }}" - # Restore shipped conda-meta\history for remapped # channels and retain only the first transaction SetOutPath "{{ env.conda_meta }}" diff --git a/constructor/osx/run_installation.sh b/constructor/osx/run_installation.sh index 4225ebac2..ec69a9b1c 100644 --- a/constructor/osx/run_installation.sh +++ b/constructor/osx/run_installation.sh @@ -45,6 +45,11 @@ fi # Perform the conda install notify "Installing packages. This might take a few minutes." + +# 'install' below will modify the history file in a way we don't want; +# keep a copy to restore later +cp "$PREFIX/conda-meta/history" "$PREFIX/conda-meta/history.bak" + # shellcheck disable=SC2086 if ! \ CONDA_REGISTER_ENVS="{{ register_envs }}" \ @@ -53,14 +58,13 @@ CONDA_SAFETY_CHECKS=disabled \ CONDA_EXTRA_SAFETY_CHECKS=no \ CONDA_CHANNELS={{ channels }} \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ -"$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" $shortcuts {{ no_rcs_arg }}; then +"$CONDA_EXEC" install --offline --file "$PREFIX/conda-meta/initial-state.explicit.txt" -yp "$PREFIX" $shortcuts {{ no_rcs_arg }}; then echo "ERROR: could not complete the conda install" exit 1 fi -# Move the prepackaged history file into place -mv "$PREFIX/pkgs/conda-meta/history" "$PREFIX/conda-meta/history" -rm -f "$PREFIX/env.txt" +# Restore history file as provided by installer +mv "$PREFIX/conda-meta/history.bak" "$PREFIX/conda-meta/history" # Same, but for the extra environments @@ -73,8 +77,9 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do fi notify "Installing ${env_name} packages..." - mkdir -p "$PREFIX/envs/$env_name/conda-meta" - touch "$PREFIX/envs/$env_name/conda-meta/history" + # 'install' below will modify the history file in a way we don't want; + # keep a copy to restore later + cp "$PREFIX/envs/$env_name/conda-meta/history" "$PREFIX/envs/$env_name/conda-meta/history.bak" if [[ -f "${env_pkgs}channels.txt" ]]; then env_channels="$(cat "${env_pkgs}channels.txt")" @@ -99,10 +104,10 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do CONDA_EXTRA_SAFETY_CHECKS=no \ CONDA_CHANNELS="$env_channels" \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ - "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1 - # Move the prepackaged history file into place - mv "${env_pkgs}/conda-meta/history" "$PREFIX/envs/$env_name/conda-meta/history" - rm -f "${env_pkgs}env.txt" + "$CONDA_EXEC" install --offline --file "$PREFIX/envs/$env_name/conda-meta/initial-state.explicit.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1 + + # Restore history file as provided by installer + mv "$PREFIX/envs/$env_name/conda-meta/history.bak" "$PREFIX/envs/$env_name/conda-meta/history" done # Cleanup! diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index bc6d7a7ef..3785ac617 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -563,7 +563,7 @@ def create(info, verbose=False): fresh_dir(SCRIPTS_DIR) pkgs_dir = join(prefix, "pkgs") os.makedirs(pkgs_dir) - preconda.write_files(info, pkgs_dir) + preconda.write_files(info, prefix) preconda.copy_extra_files(info.get("extra_files", []), prefix) # These are the user-provided scripts, maybe patched to have a shebang # They will be called by a wrapping script added later, if present diff --git a/constructor/preconda.py b/constructor/preconda.py index 6509cf925..2beeb3dc6 100644 --- a/constructor/preconda.py +++ b/constructor/preconda.py @@ -48,7 +48,12 @@ except ImportError: import ruamel_json as json -files = ".constructor-build.info", "urls", "urls.txt", "env.txt" +files = ( + "pkgs/.constructor-build.info", + "pkgs/urls", + "pkgs/urls.txt", + "conda-meta/initial-state.explicit.txt", +) def write_index_cache(info, dst_dir, used_packages): @@ -135,8 +140,28 @@ def system_info(): return out -def write_files(info, dst_dir): - with open(join(dst_dir, ".constructor-build.info"), "w") as fo: +def write_files(info: dict, workspace: str): + """ + Prepare files on disk to be shipped as part of the pre-conda payload, mostly + configuration and metadata files: + + - `conda-meta/initial-state.explicit.txt`: Lockfile to provision the base environment. + - `conda-meta/history`: Prepared history file with the right requested specs in input file. + - `pkgs/urls` and `pkgs/urls.txt`: Direct URLs of packages used, with and without MD5 hashes. + - `pkgs/cache/*.json`: Trimmed repodata to mock offline channels in use. + - `pkgs/channels.txt`: Channels in use. + - `pkgs/shortcuts.txt`: Which packages should have their shortcuts created, if any. + + If extra envs are requested, this will also write: + + - Their corresponding `envs//conda-meta/` files. + - Their corresponding `pkgs/channels.txt` and `pkgs/shortcuts.txt` under + `pkgs/envs/`. + """ + os.makedirs(join(workspace, "conda-meta"), exist_ok=True) + pkgs_dir = join(workspace, "pkgs") + os.makedirs(pkgs_dir, exist_ok=True) + with open(join(pkgs_dir, ".constructor-build.info"), "w") as fo: json.dump(system_info(), fo) all_urls = info["_urls"].copy() @@ -146,7 +171,7 @@ def write_files(info, dst_dir): final_urls_md5s = tuple((get_final_url(info, url), md5) for url, md5 in info["_urls"]) all_final_urls_md5s = tuple((get_final_url(info, url), md5) for url, md5 in all_urls) - with open(join(dst_dir, "urls"), "w") as fo: + with open(join(pkgs_dir, "urls"), "w") as fo: for url, md5 in all_final_urls_md5s: maybe_different_url = ensure_transmuted_ext(info, url) if maybe_different_url != url: # transmuted, no md5 @@ -154,7 +179,7 @@ def write_files(info, dst_dir): else: fo.write(f"{url}#{md5}\n") - with open(join(dst_dir, "urls.txt"), "w") as fo: + with open(join(pkgs_dir, "urls.txt"), "w") as fo: for url, _ in all_final_urls_md5s: fo.write("%s\n" % url) @@ -163,33 +188,36 @@ def write_files(info, dst_dir): all_dists += env_info["_dists"] all_dists = list({dist: None for dist in all_dists}) # de-duplicate - write_index_cache(info, dst_dir, all_dists) + write_index_cache(info, pkgs_dir, all_dists) # base environment conda-meta - write_conda_meta(info, dst_dir, final_urls_md5s) + write_conda_meta(info, join(workspace, "conda-meta"), final_urls_md5s) - write_repodata_record(info, dst_dir) + write_repodata_record(info, pkgs_dir) # base environment file used with conda install --file # (list of specs/dists to install) - write_env_txt(info, dst_dir, final_urls_md5s) + write_initial_state_explicit_txt(info, join(workspace, "conda-meta"), final_urls_md5s) for fn in files: - os.chmod(join(dst_dir, fn), 0o664) + os.chmod(join(workspace, fn), 0o664) for env_name, env_info in info.get("_extra_envs_info", {}).items(): env_config = info["extra_envs"][env_name] - env_dst_dir = os.path.join(dst_dir, "envs", env_name) + env_pkgs = os.path.join(workspace, "pkgs", "envs", env_name) + env_conda_meta = os.path.join(workspace, "envs", env_name, "conda-meta") + os.makedirs(env_pkgs, exist_ok=True) + os.makedirs(env_conda_meta, exist_ok=True) # environment conda-meta env_urls_md5 = tuple((get_final_url(info, url), md5) for url, md5 in env_info["_urls"]) user_requested_specs = env_config.get("user_requested_specs", env_config.get("specs", ())) - write_conda_meta(info, env_dst_dir, env_urls_md5, user_requested_specs) + write_conda_meta(info, env_conda_meta, env_urls_md5, user_requested_specs) # environment installation list - write_env_txt(info, env_dst_dir, env_urls_md5) + write_initial_state_explicit_txt(info, env_conda_meta, env_urls_md5) # channels - write_channels_txt(info, env_dst_dir, env_config) + write_channels_txt(info, env_pkgs, env_config) # shortcuts - write_shortcuts_txt(info, env_dst_dir, env_config) + write_shortcuts_txt(info, env_pkgs, env_config) def write_conda_meta(info, dst_dir, final_urls_md5s, user_requested_specs=None): @@ -212,9 +240,7 @@ def write_conda_meta(info, dst_dir, final_urls_md5s, user_requested_specs=None): builder.append("# update specs: %s" % update_specs) builder.append("\n") - if not isdir(join(dst_dir, "conda-meta")): - os.makedirs(join(dst_dir, "conda-meta")) - with open(join(dst_dir, "conda-meta", "history"), "w") as fh: + with open(join(dst_dir, "history"), "w") as fh: fh.write("\n".join(builder)) @@ -245,7 +271,7 @@ def write_repodata_record(info, dst_dir): json.dump(rr_json, rf, indent=2, sort_keys=True) -def write_env_txt(info, dst_dir, urls): +def write_initial_state_explicit_txt(info, dst_dir, urls): """ urls is an iterable of tuples with url and md5 values """ @@ -257,7 +283,7 @@ def write_env_txt(info, dst_dir, urls): @EXPLICIT """ ).lstrip() - with open(join(dst_dir, "env.txt"), "w") as envf: + with open(join(dst_dir, "initial-state.explicit.txt"), "w") as envf: envf.write(header) for url, md5 in urls: maybe_different_url = ensure_transmuted_ext(info, url) diff --git a/constructor/shar.py b/constructor/shar.py index 727da01ca..f6361a0f9 100644 --- a/constructor/shar.py +++ b/constructor/shar.py @@ -131,15 +131,15 @@ def create(info, verbose=False): postconda_tarball = join(tmp_dir, "postconda.tar.bz2") pre_t = tarfile.open(preconda_tarball, "w:bz2") post_t = tarfile.open(postconda_tarball, "w:bz2") - for dist in preconda_files: - fn = filename_dist(dist) - pre_t.add(join(tmp_dir, fn), "pkgs/" + fn) + for rel_path in preconda_files: + pre_t.add(join(tmp_dir, rel_path), rel_path) for env_name in info.get("_extra_envs_info", ()): - pre_t.add(join(tmp_dir, "envs", env_name, "env.txt"), f"pkgs/envs/{env_name}/env.txt") - pre_t.add( - join(tmp_dir, "envs", env_name, "shortcuts.txt"), f"pkgs/envs/{env_name}/shortcuts.txt" - ) + for rel_path in ( + f"pkgs/envs/{env_name}/shortcuts.txt", + f"envs/{env_name}/conda-meta/initial-state.explicit.txt", + ): + pre_t.add(join(tmp_dir, rel_path), rel_path) for key in "pre_install", "post_install": if key in info: @@ -165,7 +165,7 @@ def create(info, verbose=False): elif filename_dist(dist).endswith(".tar.bz2"): _dist = filename_dist(dist)[:-8] record_file = join(_dist, "info", "repodata_record.json") - record_file_src = join(tmp_dir, record_file) + record_file_src = join(tmp_dir, "pkgs", record_file) record_file_dest = join("pkgs", record_file) pre_t.add(record_file_src, record_file_dest) pre_t.addfile(tarinfo=tarfile.TarInfo("conda-meta/history")) @@ -185,7 +185,7 @@ def create(info, verbose=False): pre_t.close() post_t.close() - tarball = join(tmp_dir, "tmp.tar") + tarball = join(tmp_dir, "pkgs", "tmp.tar") t = tarfile.open(tarball, "w") t.add(preconda_tarball, basename(preconda_tarball)) t.add(postconda_tarball, basename(postconda_tarball)) diff --git a/constructor/winexe.py b/constructor/winexe.py index 913b07b17..bf0f91a22 100644 --- a/constructor/winexe.py +++ b/constructor/winexe.py @@ -81,11 +81,10 @@ def setup_envs_commands(info, dir_path): { "name": "base", "prefix": r"$INSTDIR", - "env_txt": r"$INSTDIR\pkgs\env.txt", # env.txt as seen by the running installer - "env_txt_dir": r"$INSTDIR\pkgs", # env.txt location in the installer filesystem - "env_txt_abspath": join( - dir_path, "env.txt" - ), # env.txt path while building the installer + # initial-state.explicit.txt as seen by the running installer + "lockfile_txt": r"$INSTDIR\conda-meta\initial-state.explicit.txt", + # initial-state.explicit.txt path while building the installer + "lockfile_txt_abspath": join(dir_path, "conda-meta", "initial-state.explicit.txt"), "conda_meta": r"$INSTDIR\conda-meta", "history_abspath": join(dir_path, "conda-meta", "history"), "final_channels": get_final_channels(info), @@ -108,9 +107,12 @@ def setup_envs_commands(info, dir_path): { "name": env_name, "prefix": join("$INSTDIR", "envs", env_name), - "env_txt": join("$INSTDIR", "pkgs", "envs", env_name, "env.txt"), - "env_txt_dir": join("$INSTDIR", "pkgs", "envs", env_name), - "env_txt_abspath": join(dir_path, "envs", env_name, "env.txt"), + "lockfile_txt": join( + "$INSTDIR", "envs", env_name, "conda-meta", "initial-state.explicit.txt" + ), + "lockfile_txt_abspath": join( + dir_path, "envs", env_name, "conda-meta", "initial-state.explicit.txt" + ), "conda_meta": join("$INSTDIR", "envs", env_name, "conda-meta"), "history_abspath": join(dir_path, "envs", env_name, "conda-meta", "history"), "final_channels": get_final_channels(channel_info), @@ -169,20 +171,20 @@ def make_nsi( "outfile": info["_outpath"], "vipv": make_VIProductVersion(info["version"]), "constructor_version": info["CONSTRUCTOR_VERSION"], + # @-prefixed paths point to {dir_path} "iconfile": "@icon.ico", "headerimage": "@header.bmp", "welcomeimage": "@welcome.bmp", "licensefile": abspath(info.get("license_file", join(NSIS_DIR, "placeholder_license.txt"))), "conda_history": "@" + join("conda-meta", "history"), "conda_exe": "@_conda.exe", - "env_txt": "@env.txt", - "urls_file": "@urls", - "urls_txt_file": "@urls.txt", + "urls_file": "@" + join("pkgs", "urls"), + "urls_txt_file": "@" + join("pkgs", "urls.txt"), "pre_install": "@pre_install.bat", "post_install": "@post_install.bat", "pre_uninstall": "@pre_uninstall.bat", - "index_cache": "@cache", - "repodata_record": "@repodata_record.json", + "index_cache": "@" + join("pkgs", "cache"), + "repodata_record": "@" + join("pkgs", "repodata_record.json"), } conclusion_text = info.get("conclusion_text", "") diff --git a/news/1059-lockfiles b/news/1059-lockfiles new file mode 100644 index 000000000..e4586e8ad --- /dev/null +++ b/news/1059-lockfiles @@ -0,0 +1,19 @@ +### Enhancements + +* Ship `conda-meta/initial-state.explicit.txt` as a copy of the lockfile that provisions the initial state of each environment. (#1052 via #1059) + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/tests/test_examples.py b/tests/test_examples.py index 5a64c867a..e8b3168fa 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -465,7 +465,17 @@ def test_example_extra_pages_win(tmp_path, request, extra_pages, monkeypatch): def test_example_extra_envs(tmp_path, request): input_path = _example_path("extra_envs") for installer, install_dir in create_installer(input_path, tmp_path): - _run_installer(input_path, installer, install_dir, request=request) + _run_installer(input_path, installer, install_dir, request=request, uninstall=False) + assert ( + "@EXPLICIT" in (install_dir / "conda-meta" / "initial-state.explicit.txt").read_text() + ) + for env in install_dir.glob("envs/*/conda-meta/"): + envtxt = env / "initial-state.explicit.txt" + assert envtxt.exists() + assert "@EXPLICIT" in envtxt.read_text() + + if sys.platform.startswith("win"): + _run_uninstaller_exe(install_dir=install_dir) def test_example_extra_files(tmp_path, request):