From cac78893790588e4c93502b21f939912002f2af8 Mon Sep 17 00:00:00 2001 From: Emmett Butler Date: Sat, 20 May 2023 07:52:47 -0700 Subject: [PATCH 1/5] add --py argument to cli allowing user to specify python version to include in generated shell --- pynixify/command.py | 15 +++++++++++++-- pynixify/expression_builder.py | 6 +++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pynixify/command.py b/pynixify/command.py index 6b9959a..0a63468 100644 --- a/pynixify/command.py +++ b/pynixify/command.py @@ -153,6 +153,15 @@ def main(): "executed by pynixify. If it isn't specified, it will be set to " "the number of CPUs in the system." )) + parser.add_argument( + "-p", + "--py", + default="python3", + help=( + "Name of the nixpkgs python interpreter package to install in the " + "generated shell.nix. Defaults to 'python3'." + ), + ) args = parser.parse_args() asyncio.run(_main_async( @@ -166,6 +175,7 @@ def main(): ignore_test_requirements_for=args.ignore_tests.split(',') if args.ignore_tests else [], max_jobs=args.max_jobs, generate_only_overlay=args.overlay_only, + interpreter=args.py, )) async def _main_async( @@ -178,7 +188,8 @@ async def _main_async( ignore_test_requirements_for: List[str], load_all_test_requirements: bool, max_jobs: Optional[int], - generate_only_overlay:bool): + generate_only_overlay: bool, + interpreter: str): if nixpkgs is not None: pynixify.nixpkgs_sources.NIXPKGS_URL = nixpkgs @@ -277,7 +288,7 @@ async def write_package_expression(package: PyPIPackage): packages.append(p) with (base_path / 'shell.nix').open('w') as fp: - expr = build_shell_nix_expression(packages) + expr = build_shell_nix_expression(packages, interpreter) fp.write(await nixfmt(expr)) diff --git a/pynixify/expression_builder.py b/pynixify/expression_builder.py index 11c20a6..de6195a 100644 --- a/pynixify/expression_builder.py +++ b/pynixify/expression_builder.py @@ -123,7 +123,7 @@ """) shell_nix_template = Template("""${DISCLAIMER} - { python ? "python3" }: + { python ? "${interpreter | nix}" }: let pkgs = import ./nixpkgs.nix {}; pythonPkg = builtins.getAttr python pkgs; @@ -206,8 +206,8 @@ def build_overlayed_nixpkgs( return overlayed_nixpkgs_template.render(DISCLAIMER=DISCLAIMER, **locals()) -def build_shell_nix_expression(packages: List[Package]) -> str: - return shell_nix_template.render(DISCLAIMER=DISCLAIMER, packages=packages) +def build_shell_nix_expression(packages: List[Package], interpreter: str) -> str: + return shell_nix_template.render(DISCLAIMER=DISCLAIMER, packages=packages, interpreter=interpreter) async def nixfmt(expr: str) -> str: From 3117b168fa099f0e8871b4110f003f280b473be1 Mon Sep 17 00:00:00 2001 From: Emmett Butler Date: Mon, 22 May 2023 05:17:45 -0700 Subject: [PATCH 2/5] pass string escape function to shell.nix template --- pynixify/expression_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pynixify/expression_builder.py b/pynixify/expression_builder.py index de6195a..dd52537 100644 --- a/pynixify/expression_builder.py +++ b/pynixify/expression_builder.py @@ -123,7 +123,7 @@ """) shell_nix_template = Template("""${DISCLAIMER} - { python ? "${interpreter | nix}" }: + { python ? ${interpreter | nix} }: let pkgs = import ./nixpkgs.nix {}; pythonPkg = builtins.getAttr python pkgs; @@ -207,7 +207,7 @@ def build_overlayed_nixpkgs( def build_shell_nix_expression(packages: List[Package], interpreter: str) -> str: - return shell_nix_template.render(DISCLAIMER=DISCLAIMER, packages=packages, interpreter=interpreter) + return shell_nix_template.render(DISCLAIMER=DISCLAIMER, packages=packages, interpreter=interpreter, nix=escape_string) async def nixfmt(expr: str) -> str: From 7c777e6f8f1b4fc373ace646ae76de39f90fe230 Mon Sep 17 00:00:00 2001 From: Emmett Butler Date: Mon, 22 May 2023 13:03:00 -0700 Subject: [PATCH 3/5] prove that parameterizing python package version works --- pynixify/data/parse_setuppy_data.nix | 5 +++-- pynixify/package_requirements.py | 6 +++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pynixify/data/parse_setuppy_data.nix b/pynixify/data/parse_setuppy_data.nix index 7ffbe22..0a06e76 100644 --- a/pynixify/data/parse_setuppy_data.nix +++ b/pynixify/data/parse_setuppy_data.nix @@ -1,5 +1,6 @@ -{ file, stdenv ? (import { }).stdenv, lib ? (import { }).lib -, unzip ? (import { }).unzip, python ? (import { }).python3 +{ file, python, stdenv ? (import { }).stdenv, lib ? (import { }).lib +, unzip ? (import { }).unzip, git ? (import { }).git +, fetchFromGitLab ? (import { }).fetchFromGitLab }: let diff --git a/pynixify/package_requirements.py b/pynixify/package_requirements.py index 5a17cc5..333e6bb 100644 --- a/pynixify/package_requirements.py +++ b/pynixify/package_requirements.py @@ -58,13 +58,17 @@ async def eval_path_requirements(path: Path) -> PackageRequirements: runtime_requirements=[] ) assert nix_expression_path.exists() + interpreter = "python37" nix_store_path = await run_nix_build( str(nix_expression_path), '--no-out-link', '--no-build-output', '--arg', 'file', - str(path.resolve()) + str(path.resolve()), + '--arg', + 'python', + "(import { }).%s" % interpreter ) if (nix_store_path / 'failed').exists(): print(f'Error parsing requirements of {path}. Assuming it has no dependencies.') From 7f5dec080acc890c2a8282514ccccf1ba6474a55 Mon Sep 17 00:00:00 2001 From: Emmett Butler Date: Tue, 23 May 2023 06:49:06 -0700 Subject: [PATCH 4/5] pass interpreter string everywhere it is needed --- default.nix | 8 ++++---- pynixify/base.py | 7 +++++-- pynixify/command.py | 6 +++--- pynixify/package_requirements.py | 3 +-- pynixify/version_chooser.py | 9 +++++---- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/default.nix b/default.nix index 768eda5..e7663df 100644 --- a/default.nix +++ b/default.nix @@ -22,10 +22,10 @@ in python3.pkgs.toPythonApplication (python3.pkgs.pynixify.overridePythonAttrs checkInputs = drv.nativeBuildInputs ++ [ nix chosenNixfmt bats ]; }) // { - checkPhase = '' - ${if runMypy then "mypy pynixify/ tests/ acceptance_tests/" else ""} - pytest tests/ -m 'not usesnix' # We can't run Nix inside Nix builds - ''; + #checkPhase = '' + # ${if runMypy then "mypy pynixify/ tests/ acceptance_tests/" else ""} + # pytest tests/ -m 'not usesnix' # We can't run Nix inside Nix builds + #''; postInstall = '' # Add nixfmt to pynixify's PATH diff --git a/pynixify/base.py b/pynixify/base.py index 39eee63..6999e58 100644 --- a/pynixify/base.py +++ b/pynixify/base.py @@ -37,7 +37,7 @@ async def source(self, extra_args=[]) -> Path: def attr(self) -> str: raise NotImplementedError() - async def metadata(self) -> PackageMetadata: + async def metadata(self, interpreter) -> PackageMetadata: from pynixify.package_requirements import run_nix_build, NixBuildError source = await self.source() if source.name.endswith('.whl'): @@ -56,7 +56,10 @@ async def metadata(self) -> PackageMetadata: '--no-build-output', '--arg', 'file', - str(source.resolve()) + str(source.resolve()), + '--arg', + 'python', + "(import { }).%s" % interpreter ) if (nix_store_path / 'failed').exists(): print(f'Error parsing metadata of {source}. Assuming it has no metadata.') diff --git a/pynixify/command.py b/pynixify/command.py index 0a63468..fcb5970 100644 --- a/pynixify/command.py +++ b/pynixify/command.py @@ -214,7 +214,7 @@ async def _main_async( all_requirements.append(Requirement(req_)) await asyncio.gather(*( - version_chooser.require(req) + version_chooser.require(req, interpreter) for req in all_requirements )) @@ -229,13 +229,13 @@ async def _main_async( async def write_package_expression(package: PyPIPackage): reqs: ChosenPackageRequirements reqs = ChosenPackageRequirements.from_package_requirements( - await evaluate_package_requirements(package), + await evaluate_package_requirements(package, extra_args=[interpreter]), version_chooser=version_chooser, load_tests=version_chooser.should_load_tests(package.pypi_name), ) sha256 = await get_path_hash(await package.source()) - meta = await package.metadata() + meta = await package.metadata(interpreter) version = await load_nixpkgs_version() try: (pname, ext) = await get_pypi_data( diff --git a/pynixify/package_requirements.py b/pynixify/package_requirements.py index 333e6bb..5a8e4ce 100644 --- a/pynixify/package_requirements.py +++ b/pynixify/package_requirements.py @@ -45,7 +45,7 @@ def from_result_path(cls, result_path: Path): return cls(**kwargs) -async def eval_path_requirements(path: Path) -> PackageRequirements: +async def eval_path_requirements(path: Path, interpreter: str) -> PackageRequirements: nix_expression_path = Path(__file__).parent / "data" / "parse_setuppy_data.nix" if path.name.endswith('.whl'): # Some nixpkgs packages use a wheel as source, which don't have a @@ -58,7 +58,6 @@ async def eval_path_requirements(path: Path) -> PackageRequirements: runtime_requirements=[] ) assert nix_expression_path.exists() - interpreter = "python37" nix_store_path = await run_nix_build( str(nix_expression_path), '--no-out-link', diff --git a/pynixify/version_chooser.py b/pynixify/version_chooser.py index debe2ee..7dd2f12 100644 --- a/pynixify/version_chooser.py +++ b/pynixify/version_chooser.py @@ -46,7 +46,7 @@ def __init__(self, nixpkgs_data: NixpkgsData, pypi_data: PyPIData, self.evaluate_requirements = req_evaluate self.should_load_tests = should_load_tests - async def require(self, r: Requirement, coming_from: Optional[Package]=None): + async def require(self, r: Requirement, interpreter: str, coming_from: Optional[Package]=None): pkg: Package if r.marker and not r.marker.evaluate(): @@ -119,14 +119,14 @@ async def require(self, r: Requirement, coming_from: Optional[Package]=None): pkg = max(pkgs, key=operator.attrgetter('version')) self._choosed_packages[canonicalize_name(r.name)] = (pkg, r.specifier) - reqs: PackageRequirements = await self.evaluate_requirements(pkg) + reqs: PackageRequirements = await self.evaluate_requirements(pkg, extra_args=[interpreter]) if isinstance(pkg, NixPackage) or ( not self.should_load_tests(canonicalize_name(r.name))): reqs.test_requirements = [] await asyncio.gather(*( - self.require(req, coming_from=pkg) + self.require(req, interpreter, coming_from=pkg) for req in (reqs.runtime_requirements + reqs.test_requirements + reqs.build_requirements) )) @@ -160,8 +160,9 @@ def all_pypi_packages(self) -> List[PyPIPackage]: async def evaluate_package_requirements( pkg: Package, extra_args=[]) -> PackageRequirements: + interpreter = extra_args.pop() src = await pkg.source(extra_args) - return await eval_path_requirements(src) + return await eval_path_requirements(src, interpreter) @dataclass From a7451f14d4bb92f792710991907a360ebbec5b7b Mon Sep 17 00:00:00 2001 From: Emmett Butler Date: Tue, 23 May 2023 06:59:56 -0700 Subject: [PATCH 5/5] undo comment --- default.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/default.nix b/default.nix index e7663df..768eda5 100644 --- a/default.nix +++ b/default.nix @@ -22,10 +22,10 @@ in python3.pkgs.toPythonApplication (python3.pkgs.pynixify.overridePythonAttrs checkInputs = drv.nativeBuildInputs ++ [ nix chosenNixfmt bats ]; }) // { - #checkPhase = '' - # ${if runMypy then "mypy pynixify/ tests/ acceptance_tests/" else ""} - # pytest tests/ -m 'not usesnix' # We can't run Nix inside Nix builds - #''; + checkPhase = '' + ${if runMypy then "mypy pynixify/ tests/ acceptance_tests/" else ""} + pytest tests/ -m 'not usesnix' # We can't run Nix inside Nix builds + ''; postInstall = '' # Add nixfmt to pynixify's PATH