fix(poly check): dependency/dists lookup lookup performance improvements (#359)

* fix(poly check): cached 'packages_distributions' from importlib.metadata, always run with same input

* fix(poly check): local cache for dist.files lookup when figuring out dependencies and sub-dependencies

* refactor(distributions): extract custom caching into separate module, also: unit tests resetting cached state before run

* bump Poetry plugin to 1.38.2

* bump CLI to 1.31.3

Author: DavidVujic

components/polylith/distributions/__init__.py

@@ -1,3 +1,4 @@
+from polylith.distributions import caching
 from polylith.distributions.collect import known_aliases_and_sub_dependencies
 from polylith.distributions.core import (
     distributions_packages,
@@ -6,6 +7,7 @@
 )
 
 __all__ = [
+    "caching",
     "distributions_packages",
     "distributions_sub_packages",
     "get_distributions",

components/polylith/distributions/caching.py

@@ -0,0 +1,17 @@
+_cache = {}
+
+
+def add(key: str, value) -> None:
+    _cache[key] = value
+
+
+def get(key: str):
+    return _cache[key]
+
+
+def exists(key: str) -> bool:
+    return key in _cache
+
+
+def clear() -> None:
+    _cache.clear()

components/polylith/distributions/core.py

@@ -1,8 +1,11 @@
 import importlib.metadata
+import pathlib
 import re
 from functools import lru_cache, reduce
 from typing import Dict, List
 
+from polylith.distributions import caching
+
 SUB_DEP_SEPARATORS = r"[\s!=;><\^~]"
 
 
@@ -25,9 +28,11 @@ def map_sub_packages(acc, dist) -> dict:
     return {**acc, **dist_subpackages(dist)}
 
 
-def parsed_namespaces_from_files(dist) -> List[str]:
-    name = dist.metadata["name"]
-    files = dist.files or []
+def parsed_namespaces_from_files(dist, name: str) -> List[str]:
+    if not caching.exists(name):
+        files = dist.files or []
+        python_files = [f for f in files if f.suffix == ".py"]
+        caching.add(name, python_files)
 
     normalized_name = str.replace(name, "-", "_")
     to_ignore = {
@@ -38,7 +43,7 @@ def parsed_namespaces_from_files(dist) -> List[str]:
         "..",
     }
 
-    filtered = [f for f in files if f.suffix == ".py"]
+    filtered: List[pathlib.PurePosixPath] = caching.get(name)
     top_folders = {f.parts[0] for f in filtered if len(f.parts) > 1}
     namespaces = {t for t in top_folders if t not in to_ignore}
 
@@ -49,17 +54,19 @@ def parsed_top_level_namespace(namespaces: List[str]) -> List[str]:
     return [str.replace(ns, "/", ".") for ns in namespaces]
 
 
-def top_level_packages(dist) -> List[str]:
+def top_level_packages(dist, name: str) -> List[str]:
     top_level = dist.read_text("top_level.txt")
 
     namespaces = str.split(top_level or "")
 
-    return parsed_top_level_namespace(namespaces) or parsed_namespaces_from_files(dist)
+    return parsed_top_level_namespace(namespaces) or parsed_namespaces_from_files(
+        dist, name
+    )
 
 
 def mapped_packages(dist) -> dict:
-    packages = top_level_packages(dist)
     name = dist.metadata["name"]
+    packages = top_level_packages(dist, name)
 
     return {name: packages} if packages else {}
 
@@ -83,6 +90,14 @@ def get_distributions() -> list:
     return list(importlib.metadata.distributions())
 
 
+@lru_cache
+def package_distributions_from_importlib() -> dict:
+    # added in Python 3.10
+    fn = getattr(importlib.metadata, "packages_distributions", None)
+
+    return fn() if fn else {}
+
+
 def get_packages_distributions(project_dependencies: set) -> set:
     """Return the mapped top namespace from an import
 
@@ -93,10 +108,7 @@ def get_packages_distributions(project_dependencies: set) -> set:
     Note: available for Python >= 3.10
     """
 
-    # added in Python 3.10
-    fn = getattr(importlib.metadata, "packages_distributions", None)
-
-    dists = fn() if fn else {}
+    dists = package_distributions_from_importlib()
 
     common = {k for k, v in dists.items() if project_dependencies.intersection(set(v))}
 

projects/poetry_polylith_plugin/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "poetry-polylith-plugin"
-version = "1.38.1"
+version = "1.38.2"
 description = "A Poetry plugin that adds tooling support for the Polylith Architecture"
 authors = ["David Vujic"]
 homepage = "https://davidvujic.github.io/python-polylith-docs/"

projects/polylith_cli/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "polylith-cli"
-version = "1.31.2"
+version = "1.31.3"
 description = "Python tooling support for the Polylith Architecture"
 authors = ['David Vujic']
 homepage = "https://davidvujic.github.io/python-polylith-docs/"

test/components/polylith/distributions/test_core.py

@@ -21,6 +21,12 @@ def read_text(self, *args):
         return self.read_text_data
 
 
+@pytest.fixture
+def setup():
+    distributions.caching.clear()
+    distributions.core.package_distributions_from_importlib.cache_clear()
+
+
 def test_distribution_packages():
     dists = list(importlib.metadata.distributions())
 
@@ -61,7 +67,7 @@ def test_distribution_packages_for_missing_metadata_is_handled():
     assert res == {}
 
 
-def test_distribution_packages_with_top_level_ns_information_in_files():
+def test_distribution_packages_with_top_level_ns_information_in_files(setup):
     files = [
         importlib.metadata.PackagePath("some_module.py"),
         importlib.metadata.PackagePath("hello/world.py"),
@@ -104,7 +110,7 @@ def test_distribution_sub_packages():
 
 
 @pytest.mark.skipif(sys.version_info < (3, 10), reason="requires python3.10 or higher")
-def test_package_distributions_returning_top_namespace(monkeypatch):
+def test_package_distributions_returning_top_namespace(setup, monkeypatch):
     fake_dists = {
         "something": ["something-subnamespace"],
         "opentelemetry": ["opentelemetry-instrumentation-fastapi"],