Merge pull request #50 from DavidCEllis/expand_cache_usage

Don't rely on uv/pyenv/windows registry for runtime details - use the script and cache.

Author: DavidCEllis

src/ducktools/pythonfinder/__init__.py

@@ -20,7 +20,7 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+from __future__ import annotations
 # Find platform python versions
 
 __all__ = [
@@ -43,10 +43,10 @@
     from .linux import get_python_installs
 
 
-def list_python_installs(*, query_executables: bool = True, finder: "DetailFinder | None" = None):
+def list_python_installs(*, finder: DetailFinder | None = None):
     finder = DetailFinder() if finder is None else finder
     return sorted(
-        get_python_installs(query_executables=query_executables, finder=finder),
+        get_python_installs(finder=finder),
         reverse=True,
         key=lambda x: (x.version[3], *x.version[:3], x.version[4])
     )

src/ducktools/pythonfinder/__main__.py

@@ -115,11 +115,6 @@ def get_parser():
         description="Discover base Python installs",
     )
     parser.add_argument("-V", "--version", action="version", version=__version__)
-    parser.add_argument(
-        "--fast",
-        action="store_true",
-        help="Skip Python installs that need to be launched to obtain metadata"
-    )
 
     subparsers = parser.add_subparsers(dest="command", required=False)
 
@@ -169,7 +164,6 @@ def display_local_installs(
     min_ver=None,
     max_ver=None,
     compatible=None,
-    query_executables=True,
 ):
     if min_ver:
         min_ver = tuple(int(i) for i in min_ver.split("."))
@@ -178,7 +172,7 @@ def display_local_installs(
     if compatible:
         compatible = tuple(int(i) for i in compatible.split("."))
 
-    installs = list_python_installs(query_executables=query_executables)
+    installs = list_python_installs()
 
     headings = ["Version", "Executable Location"]
 
@@ -333,7 +327,6 @@ def main():
                 min_ver=vals.min,
                 max_ver=vals.max,
                 compatible=vals.compatible,
-                query_executables=not vals.fast,
             )
     else:
         # No arguments to parse

src/ducktools/pythonfinder/details_script.py

@@ -90,7 +90,6 @@ def get_details():
                 freethreaded = bool(sysconfig.get_config_var("Py_GIL_DISABLED"))
                 metadata["freethreaded"] = freethreaded
 
-
     install = dict(
         version=list(sys.version_info),
         executable=sys.executable,

src/ducktools/pythonfinder/linux/__init__.py

@@ -73,20 +73,17 @@ def get_path_pythons(*, finder: DetailFinder | None = None) -> Iterator[PythonIn
 
 def get_python_installs(
     *,
-    query_executables: bool = True,
     finder: DetailFinder | None = None,
 ) -> Iterator[PythonInstall]:
     listed_pythons = set()
 
     finder = DetailFinder() if finder is None else finder
 
     chain_commands = [
-        get_pyenv_pythons(query_executables=query_executables, finder=finder),
-        get_uv_pythons(query_executables=query_executables, finder=finder),
+        get_pyenv_pythons(finder=finder),
+        get_uv_pythons(finder=finder),
+        get_path_pythons(finder=finder),
     ]
-    if query_executables:
-        chain_commands.append(get_path_pythons(finder=finder))
-
     with finder:
         for py in itertools.chain.from_iterable(chain_commands):
             if py.executable not in listed_pythons:

src/ducktools/pythonfinder/linux/pyenv_search.py

@@ -32,7 +32,7 @@
 
 from ducktools.lazyimporter import LazyImporter, FromImport, ModuleImport
 
-from ..shared import PythonInstall, DetailFinder, FULL_PY_VER_RE, INSTALLER_CACHE_PATH, version_str_to_tuple
+from ..shared import PythonInstall, DetailFinder, INSTALLER_CACHE_PATH
 
 _laz = LazyImporter(
     [
@@ -42,9 +42,6 @@
     ]
 )
 
-# pyenv folder name for pypy
-PYPY_VER_RE = r"^pypy(?P<pyversion>\d{1,2}\.\d+)-(?P<pypyversion>[\d\.]*)$"
-
 
 def get_pyenv_root() -> str | None:
     # Check if the environment variable exists, if so use that
@@ -77,7 +74,6 @@ def get_pyenv_root() -> str | None:
 def get_pyenv_pythons(
     versions_folder: str | os.PathLike | None = None,
     *,
-    query_executables: bool = True,
     finder: DetailFinder = None,
 ) -> Iterator[PythonInstall]:
     if versions_folder is None:
@@ -96,27 +92,7 @@ def get_pyenv_pythons(
     with finder:
         for p in sorted(os.scandir(versions_folder), key=lambda x: x.path):
             executable = os.path.join(p.path, "bin/python")
-
             if os.path.exists(executable):
-                if p.name.endswith("t"):
-                    freethreaded = True
-                    version = p.name[:-1]
-                else:
-                    freethreaded = False
-                    version = p.name
-                if _laz.re.fullmatch(FULL_PY_VER_RE, version):
-                    version_tuple = version_str_to_tuple(version)
-                    metadata = {}
-                    if version_tuple >= (3, 13):
-                        metadata["freethreaded"] = freethreaded
-                    yield PythonInstall(
-                        version=version_tuple,
-                        executable=executable,
-                        metadata=metadata,
-                        managed_by="pyenv",
-                    )
-                elif (
-                    query_executables
-                    and (install := finder.get_install_details(executable, managed_by="pyenv"))
-                ):
+                install = finder.get_install_details(executable, managed_by="pyenv")
+                if install:
                     yield install

src/ducktools/pythonfinder/shared.py

@@ -225,12 +225,18 @@ def clear_cache(self):
         self._raw_cache = {}
         self._dirty_cache = True
 
-    def query_install(self, exe_path: str, managed_by: str | None = None) -> PythonInstall | None:
+    def query_install(
+        self,
+        exe_path: str,
+        managed_by: str | None = None,
+        metadata: dict | None = None,
+    ) -> PythonInstall | None:
         """
         Query the details of a Python install directly
 
         :param exe_path: Path to the runtime .exe
         :param managed_by: Which tool manages this install (if any)
+        :param metadata: Dictionary of install metadata
         :return: a PythonInstall if one exists at the exe Path
         """
         try:
@@ -273,9 +279,19 @@ def query_install(self, exe_path: str, managed_by: str | None = None) -> PythonI
         except _laz.json.JSONDecodeError:
             return None
 
-        return PythonInstall.from_json(**output, managed_by=managed_by)
+        if metadata:
+            output["metadata"].update(metadata)
+
+        install = PythonInstall.from_json(**output, managed_by=managed_by)
 
-    def get_install_details(self, exe_path: str, managed_by=None) -> PythonInstall | None:
+        return install
+
+    def get_install_details(
+        self,
+        exe_path: str,
+        managed_by: str | None = None,
+        metadata: dict | None = None,
+    ) -> PythonInstall | None:
         exe_path = os.path.abspath(exe_path)
         mtime = os.stat(exe_path).st_mtime
 
@@ -287,7 +303,7 @@ def get_install_details(self, exe_path: str, managed_by=None) -> PythonInstall |
                 self.raw_cache.pop(exe_path)
 
         if install is None:
-            install = self.query_install(exe_path, managed_by)
+            install = self.query_install(exe_path, managed_by, metadata)
             if install:
                 self.raw_cache[exe_path] = {
                     "mtime": mtime,
@@ -416,6 +432,7 @@ def get_folder_pythons(
     base_folder: str | os.PathLike,
     basenames: tuple[str] = ("python", "pypy"),
     finder: DetailFinder | None = None,
+    managed_by: str | None = None,
 ):
     regexes = [_python_exe_regex(name) for name in basenames]
 
@@ -440,7 +457,7 @@ def get_folder_pythons(
                         continue
                 else:
                     p = file_path.path
-                install = finder.get_install_details(p)
+                install = finder.get_install_details(p, managed_by=managed_by)
                 if install:
                     yield install
 
@@ -483,7 +500,6 @@ def get_uv_python_path() -> str | None:
 
 def _implementation_from_uv_dir(
     direntry: os.DirEntry,
-    query_executables: bool = True,
     finder: DetailFinder | None = None,
 ) -> PythonInstall | None:
     python_exe = "python.exe" if sys.platform == "win32" else "bin/python"
@@ -493,35 +509,12 @@ def _implementation_from_uv_dir(
     finder = DetailFinder() if finder is None else finder
 
     if os.path.exists(python_path):
-        if match := _laz.re.fullmatch(UV_PYTHON_RE, direntry.name):
-            implementation, version, extra, platform, arch = match.groups()
-            metadata = {
-                "freethreaded": "freethreaded" in extra,
-            }
-
-            try:
-                if implementation in {"cpython"}:
-                    install = PythonInstall.from_str(
-                        version=version,
-                        executable=python_path,
-                        architecture="32bit" if arch in {"i686", "armv7"} else "64bit",
-                        implementation=implementation,
-                        metadata=metadata,
-                        managed_by="Astral",
-                    )
-            except ValueError:
-                pass
-
-        if install is None:
-            # Directory name format has changed or this is an alternate implementation
-            # Slow backup - ask python itself
-            if query_executables:
-                install = finder.get_install_details(python_path, managed_by="Astral")
+        install = finder.get_install_details(python_path, managed_by="Astral")
 
     return install
 
 
-def get_uv_pythons(query_executables=True, finder=None) -> Iterator[PythonInstall]:
+def get_uv_pythons(finder=None) -> Iterator[PythonInstall]:
     # This takes some shortcuts over the regular pythonfinder
     # As the UV folders give the python version and the implementation
     if uv_python_path := get_uv_python_path():
@@ -532,6 +525,6 @@ def get_uv_pythons(query_executables=True, finder=None) -> Iterator[PythonInstal
                 for f in fld:
                     if (
                         f.is_dir()
-                        and (install := _implementation_from_uv_dir(f, query_executables, finder=finder))
+                        and (install := _implementation_from_uv_dir(f, finder=finder))
                     ):
                         yield install

src/ducktools/pythonfinder/win32/__init__.py

@@ -32,7 +32,6 @@
 
 def get_python_installs(
     *,
-    query_executables: bool = True,
     finder: DetailFinder | None = None
 ) -> Iterator[PythonInstall]:
     listed_installs = set()
@@ -42,8 +41,8 @@ def get_python_installs(
     with finder:
         for py in itertools.chain(
             get_registered_pythons(),
-            get_pyenv_pythons(query_executables=query_executables, finder=finder),
-            get_uv_pythons(query_executables=query_executables, finder=finder),
+            get_pyenv_pythons(finder=finder),
+            get_uv_pythons(finder=finder),
         ):
             if py.executable not in listed_installs:
                 yield py

src/ducktools/pythonfinder/win32/pyenv_search.py

@@ -39,7 +39,6 @@ def get_pyenv_root() -> str | None:
 def get_pyenv_pythons(
     versions_folder: str | os.PathLike | None = None,
     *,
-    query_executables: bool = True,
     finder: DetailFinder | None = None,
 ) -> Iterator[PythonInstall]:
 
@@ -56,51 +55,14 @@ def get_pyenv_pythons(
         for p in os.scandir(versions_folder):
             path_base = os.path.basename(p.path)
 
-            if query_executables:
-                # Check for pypy/graalpy
-                if path_base.startswith("pypy"):
-                    executable = os.path.join(p.path, "pypy.exe")
-                    if os.path.exists(executable):
-                        yield finder.get_install_details(executable, managed_by="pyenv")
-                        continue
-                elif path_base.startswith("graalpy"):
-                    # Graalpy exe in bin subfolder
-                    executable = os.path.join(p.path, "bin", "graalpy.exe")
-                    if os.path.exists(executable):
-                        yield finder.get_install_details(executable, managed_by="pyenv")
-                        continue
-
-            # Regular CPython
-            executable = os.path.join(p.path, "python.exe")
+            if path_base.startswith("pypy"):
+                executable = os.path.join(p.path, "pypy.exe")
+            elif path_base.startswith("graalpy"):
+                # Graalpy exe in bin subfolder
+                executable = os.path.join(p.path, "bin", "graalpy.exe")
+            else:
+                # Try python.exe
+                executable = os.path.join(p.path, "python.exe")
 
             if os.path.exists(executable):
-                split_version = p.name.split("-")
-
-                # If there are 1 or 2 arguments this is a recognised version
-                # Otherwise it is unrecognised
-                if len(split_version) == 2:
-                    version, arch = split_version
-
-                    # win32 in pyenv name means 32 bit python install
-                    # 'arm' is the only alternative which will be 64bit
-                    arch = "32bit" if arch == "win32" else "64bit"
-                    try:
-                        yield PythonInstall.from_str(
-                            version=version,
-                            executable=executable,
-                            architecture=arch,
-                            managed_by="pyenv",
-                        )
-                    except ValueError:
-                        pass
-                elif len(split_version) == 1:
-                    version = split_version[0]
-                    try:
-                        yield PythonInstall.from_str(
-                            version=version,
-                            executable=executable,
-                            architecture="64bit",
-                            managed_by="pyenv",
-                        )
-                    except ValueError:
-                        pass
+                yield finder.get_install_details(executable, managed_by="pyenv")