From 4e5611278118a46a6bbc8de30acb0446df1350b1 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Fri, 1 Aug 2025 19:11:08 +0200 Subject: [PATCH 1/7] Add Exclude option when building entrypoints --- plugin/entrypoint.py | 1 + plux/build/setuptools.py | 62 ++++++++++++-- plux/cli/cli.py | 2 + .../pyproject/mysrc/subpkg/__init__.py | 0 .../pyproject/mysrc/subpkg/plugins.py | 6 ++ .../setupcfg/mysrc/subpkg/__init__.py | 0 .../projects/setupcfg/mysrc/subpkg/plugins.py | 6 ++ tests/cli/test_entrypoints.py | 82 ++++++++++++++++++- 8 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 tests/cli/projects/pyproject/mysrc/subpkg/__init__.py create mode 100644 tests/cli/projects/pyproject/mysrc/subpkg/plugins.py create mode 100644 tests/cli/projects/setupcfg/mysrc/subpkg/__init__.py create mode 100644 tests/cli/projects/setupcfg/mysrc/subpkg/plugins.py diff --git a/plugin/entrypoint.py b/plugin/entrypoint.py index 52176b2..288c5b4 100644 --- a/plugin/entrypoint.py +++ b/plugin/entrypoint.py @@ -1,6 +1,7 @@ """ Deprecated bindings, use plux imports instead, but you shouldn't use the internals in the first place. """ + from plux.build.setuptools import find_plugins from plux.core.entrypoint import ( EntryPoint, diff --git a/plux/build/setuptools.py b/plux/build/setuptools.py index bc8a87e..05a89ad 100644 --- a/plux/build/setuptools.py +++ b/plux/build/setuptools.py @@ -8,7 +8,9 @@ import re import shutil import sys +import tomllib import typing as t +from fnmatch import fnmatchcase from pathlib import Path import setuptools @@ -39,20 +41,31 @@ def _ensure_dist_info(self, *args, **kwargs): class plugins(InfoCommon, setuptools.Command): description = "Discover plux plugins and store them in .egg_info" - user_options = [ - # TODO + user_options: t.ClassVar[list[tuple[str, str, str]]] = [ + ('exclude=', 'e', "exclude those files when discovering plugins"), + # TODO: add more ] egg_info: str def initialize_options(self) -> None: self.plux_json_path = None + self.exclude = None def finalize_options(self) -> None: self.plux_json_path = get_plux_json_path(self.distribution) + self.ensure_string_list('exclude') + if self.exclude is None: + self.exclude = [] + + project_config = read_configuration(self.distribution) + file_exclude = project_config.get("exclude") + if file_exclude: + self.exclude = set(self.exclude) | set(file_exclude) + self.exclude = [_path_to_module(item) for item in self.exclude] def run(self) -> None: - plugin_finder = PluginFromPackageFinder(DistributionPackageFinder(self.distribution)) + plugin_finder = PluginFromPackageFinder(DistributionPackageFinder(self.distribution, exclude=self.exclude)) ep = discover_entry_points(plugin_finder) self.debug_print(f"writing discovered plugins into {self.plux_json_path}") @@ -191,6 +204,20 @@ def get_plux_json_path(distribution): return os.path.join(egg_info_dir, "plux.json") +def read_configuration(distribution) -> dict: + dirs = distribution.package_dir + pyproject_base = (dirs or {}).get("", os.curdir) + pyproject_file = os.path.join(pyproject_base, "pyproject.toml") + if not os.path.exists(pyproject_file): + return {} + + with open(pyproject_file, "rb") as file: + pyproject_config = tomllib.load(file) + + tool_table = pyproject_config.get("tool", {}) + return tool_table.get("plux", {}) + + def update_entrypoints(distribution, ep: EntryPointDict): if distribution.entry_points is None: distribution.entry_points = {} @@ -375,6 +402,27 @@ def _to_filename(name): return name.replace("-", "_") +def _path_to_module(path): + """ + Convert a path to a Python module to its module representation + Example: plux/core/test -> plux.core.test + """ + return path.strip("/").replace("/", ".") + + +class _Filter: + """ + Given a list of patterns, create a callable that will be true only if + the input matches at least one of the patterns. + This is from `setuptools.discovery._Filter` + """ + def __init__(self, patterns: t.Iterable[str]): + self._patterns = patterns + + def __call__(self, item: str): + return any(fnmatchcase(item, pat) for pat in self._patterns) + + class _PackageFinder: """ Generate a list of Python packages. How these are generated depends on the implementation. @@ -397,11 +445,12 @@ class DistributionPackageFinder(_PackageFinder): correctly if configured. """ - def __init__(self, distribution: Distribution): + def __init__(self, distribution: Distribution, exclude: t.Optional[t.Iterable[str]] = None): self.distribution = distribution + self.exclude = _Filter(exclude or []) def find_packages(self) -> t.Iterable[str]: - return self.distribution.packages + return self.filter_packages(self.distribution.packages) @property def path(self) -> str: @@ -415,6 +464,9 @@ def path(self) -> str: where = "." return where + def filter_packages(self, packages: t.Iterable[str]) -> t.Iterable[str]: + return [item for item in packages if not self.exclude(item)] + class DefaultPackageFinder(_PackageFinder): def __init__(self, where=".", exclude=(), include=("*",), namespace=True) -> None: diff --git a/plux/cli/cli.py b/plux/cli/cli.py index 5a3bd97..a0fb230 100644 --- a/plux/cli/cli.py +++ b/plux/cli/cli.py @@ -20,6 +20,7 @@ def entrypoints(args): dist = get_distribution_from_workdir(os.getcwd()) print("discovering plugins ...") + dist.command_options["plugins"] = {"exclude": ("command line", args.exclude)} dist.run_command("plugins") print(f"building {dist.get_name().replace('-', '_')}.egg-info...") @@ -82,6 +83,7 @@ def main(argv=None): generate_parser = subparsers.add_parser( "entrypoints", help="Discover plugins and generate entry points" ) + generate_parser.add_argument("-e", "--exclude", help="path(s) to exclude") generate_parser.set_defaults(func=entrypoints) # Subparser for the 'discover' subcommand diff --git a/tests/cli/projects/pyproject/mysrc/subpkg/__init__.py b/tests/cli/projects/pyproject/mysrc/subpkg/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/projects/pyproject/mysrc/subpkg/plugins.py b/tests/cli/projects/pyproject/mysrc/subpkg/plugins.py new file mode 100644 index 0000000..163edf8 --- /dev/null +++ b/tests/cli/projects/pyproject/mysrc/subpkg/plugins.py @@ -0,0 +1,6 @@ +from plugin import Plugin + + +class MyNestedPlugin(Plugin): + namespace = "plux.test.plugins" + name = "mynestedplugin" diff --git a/tests/cli/projects/setupcfg/mysrc/subpkg/__init__.py b/tests/cli/projects/setupcfg/mysrc/subpkg/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/projects/setupcfg/mysrc/subpkg/plugins.py b/tests/cli/projects/setupcfg/mysrc/subpkg/plugins.py new file mode 100644 index 0000000..163edf8 --- /dev/null +++ b/tests/cli/projects/setupcfg/mysrc/subpkg/plugins.py @@ -0,0 +1,6 @@ +from plugin import Plugin + + +class MyNestedPlugin(Plugin): + namespace = "plux.test.plugins" + name = "mynestedplugin" diff --git a/tests/cli/test_entrypoints.py b/tests/cli/test_entrypoints.py index 9849425..6ae0d14 100644 --- a/tests/cli/test_entrypoints.py +++ b/tests/cli/test_entrypoints.py @@ -1,6 +1,6 @@ import os.path +import shutil import sys -from pathlib import Path import pytest @@ -26,10 +26,88 @@ def test_entrypoints(project_name): with open(os.path.join(project, "test_project.egg-info", "entry_points.txt"), "r") as f: lines = [line.strip() for line in f.readlines() if line.strip()] - assert lines == ["[plux.test.plugins]", "myplugin = mysrc.plugins:MyPlugin"] + assert lines == [ + "[plux.test.plugins]", + "mynestedplugin = mysrc.subpkg.plugins:MyNestedPlugin", + "myplugin = mysrc.plugins:MyPlugin", + ] # make sure that SOURCES.txt contain no absolute paths with open(os.path.join(project, "test_project.egg-info", "SOURCES.txt"), "r") as f: lines = [line.strip() for line in f.readlines() if line.strip()] for line in lines: assert not line.startswith("/") + + +@pytest.mark.parametrize("project_name", ["pyproject", "setupcfg"]) +def test_entrypoints_exclude(project_name): + if project_name == "pyproject" and sys.version_info < (3, 10): + pytest.xfail("reading pyproject.toml requires Python 3.10 or above") + + from plux.__main__ import main + + project = os.path.join(os.path.dirname(__file__), "projects", project_name) + os.chdir(project) + + sys.path.append(project) + try: + try: + main(["--workdir", project, "entrypoints", "--exclude", "**/subpkg*"]) + except SystemExit: + pass + finally: + sys.path.remove(project) + + with open(os.path.join(project, "test_project.egg-info", "entry_points.txt"), "r") as f: + lines = [line.strip() for line in f.readlines() if line.strip()] + assert lines == [ + "[plux.test.plugins]", + "myplugin = mysrc.plugins:MyPlugin", + ] + + # make sure that SOURCES.txt contain no absolute paths + with open(os.path.join(project, "test_project.egg-info", "SOURCES.txt"), "r") as f: + lines = [line.strip() for line in f.readlines() if line.strip()] + for line in lines: + assert not line.startswith("/") + + +def test_entrypoints_exclude_from_pyproject_config(tmp_path): + if sys.version_info < (3, 10): + pytest.xfail("reading pyproject.toml requires Python 3.10 or above") + + from plux.__main__ import main + + src_project = os.path.join(os.path.dirname(__file__), "projects", "pyproject") + dest_project = os.path.join(str(tmp_path), "pyproject") + + shutil.copytree(src_project, dest_project) + + pyproject_toml_path = os.path.join(dest_project, "pyproject.toml") + + with open(pyproject_toml_path, "a") as fp: + fp.write('\n[tool.plux]\nexclude = ["**subpkg*"]\n') + + os.chdir(dest_project) + + sys.path.append(dest_project) + try: + try: + main(["--workdir", dest_project, "entrypoints"]) + except SystemExit: + pass + finally: + sys.path.remove(dest_project) + + with open(os.path.join(dest_project, "test_project.egg-info", "entry_points.txt"), "r") as f: + lines = [line.strip() for line in f.readlines() if line.strip()] + assert lines == [ + "[plux.test.plugins]", + "myplugin = mysrc.plugins:MyPlugin", + ] + + # make sure that SOURCES.txt contain no absolute paths + with open(os.path.join(dest_project, "test_project.egg-info", "SOURCES.txt"), "r") as f: + lines = [line.strip() for line in f.readlines() if line.strip()] + for line in lines: + assert not line.startswith("/") From 79222e4312123fd771df197ac383535165f6e921 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Fri, 1 Aug 2025 19:22:04 +0200 Subject: [PATCH 2/7] fix tomllib import --- plux/build/setuptools.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plux/build/setuptools.py b/plux/build/setuptools.py index 05a89ad..33bfead 100644 --- a/plux/build/setuptools.py +++ b/plux/build/setuptools.py @@ -8,9 +8,9 @@ import re import shutil import sys -import tomllib import typing as t from fnmatch import fnmatchcase +from importlib.util import find_spec from pathlib import Path import setuptools @@ -205,6 +205,10 @@ def get_plux_json_path(distribution): def read_configuration(distribution) -> dict: + if not find_spec("tomllib"): + return {} + + import tomllib dirs = distribution.package_dir pyproject_base = (dirs or {}).get("", os.curdir) pyproject_file = os.path.join(pyproject_base, "pyproject.toml") From 7613128b33d896393721f40e3b24924180b95c48 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Fri, 1 Aug 2025 19:32:18 +0200 Subject: [PATCH 3/7] fix typing for 3.10 --- plux/build/setuptools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plux/build/setuptools.py b/plux/build/setuptools.py index 33bfead..2aaad9c 100644 --- a/plux/build/setuptools.py +++ b/plux/build/setuptools.py @@ -41,7 +41,7 @@ def _ensure_dist_info(self, *args, **kwargs): class plugins(InfoCommon, setuptools.Command): description = "Discover plux plugins and store them in .egg_info" - user_options: t.ClassVar[list[tuple[str, str, str]]] = [ + user_options: t.ClassVar[t.List[t.Tuple[str, str, str]]] = [ ('exclude=', 'e', "exclude those files when discovering plugins"), # TODO: add more ] From 4ce9f5d75015317a6f2b0d868e079e14e2999d6d Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Fri, 1 Aug 2025 19:46:26 +0200 Subject: [PATCH 4/7] use internal vendored tomli from setuptools as fallback --- plux/build/setuptools.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plux/build/setuptools.py b/plux/build/setuptools.py index 2aaad9c..68277ec 100644 --- a/plux/build/setuptools.py +++ b/plux/build/setuptools.py @@ -205,10 +205,13 @@ def get_plux_json_path(distribution): def read_configuration(distribution) -> dict: - if not find_spec("tomllib"): + if find_spec("tomllib"): + from tomllib import load as load_toml + elif find_spec("tomli"): + from tomli import load as load_toml + else: return {} - import tomllib dirs = distribution.package_dir pyproject_base = (dirs or {}).get("", os.curdir) pyproject_file = os.path.join(pyproject_base, "pyproject.toml") @@ -216,7 +219,7 @@ def read_configuration(distribution) -> dict: return {} with open(pyproject_file, "rb") as file: - pyproject_config = tomllib.load(file) + pyproject_config = load_toml(file) tool_table = pyproject_config.get("tool", {}) return tool_table.get("plux", {}) From 354b912a907dacf4bc8c9d6997b3c4859a978d24 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Fri, 1 Aug 2025 20:33:46 +0200 Subject: [PATCH 5/7] update README.md --- README.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 55c9aac..3168002 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ It provides tools to load plugins from entry points at run time, and to discover * `PluginManager`: manages the run time lifecycle of a Plugin, which has three states: * resolved: the entrypoint pointing to the PluginSpec was imported and the `PluginSpec` instance was created * init: the `PluginFactory` of the `PluginSpec` was successfully invoked - * loaded: the `load` method of the `Plugin` was successfully invoked + * loaded: the `load` method of the `Plugin` was successfully i[nvoked ![architecture](https://raw.githubusercontent.com/localstack/plux/main/docs/plux-architecture.png) @@ -198,6 +198,29 @@ build-backend = "setuptools.build_meta" # ... ``` +Additional configuration +------------------------ + +You can pass additional configuration to Plux, either via the command line or your project `pyproject.toml`. + +### Excluding Python packages during discovery + +When [discovering entrypoints](#discovering-entrypoints), Plux will try importing your code to discover Plugins. +Some parts of your codebase might have side effects, or raise errors when imported outside a specific context like some database +migration scripts. + +You can ignore those Python packages by specifying the `--exclude` flag to the entrypoints discovery commands (`python -m plux entrypoints` or `python setup.py plugins`). +The option takes a list of comma-separated values that can be paths or package names. + +You can also specify those values in the `tool.plux` section of your `pyproject.toml`: + +```toml +# ... + +[tool.plux] +exclude = ["**/database/alembic*"] +``` + Install ------- From f3e5cc69e2e4941fd9f675d01c79f5147547ce8d Mon Sep 17 00:00:00 2001 From: Ben Simon Hartung <42031100+bentsku@users.noreply.github.com> Date: Mon, 4 Aug 2025 22:34:12 +0200 Subject: [PATCH 6/7] PR comments Co-authored-by: Thomas Rausch --- README.md | 2 +- plux/build/setuptools.py | 4 ++-- plux/cli/cli.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3168002..3cfa7e7 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ It provides tools to load plugins from entry points at run time, and to discover * `PluginManager`: manages the run time lifecycle of a Plugin, which has three states: * resolved: the entrypoint pointing to the PluginSpec was imported and the `PluginSpec` instance was created * init: the `PluginFactory` of the `PluginSpec` was successfully invoked - * loaded: the `load` method of the `Plugin` was successfully i[nvoked + * loaded: the `load` method of the `Plugin` was successfully invoked ![architecture](https://raw.githubusercontent.com/localstack/plux/main/docs/plux-architecture.png) diff --git a/plux/build/setuptools.py b/plux/build/setuptools.py index 68277ec..35533e9 100644 --- a/plux/build/setuptools.py +++ b/plux/build/setuptools.py @@ -42,7 +42,7 @@ class plugins(InfoCommon, setuptools.Command): description = "Discover plux plugins and store them in .egg_info" user_options: t.ClassVar[t.List[t.Tuple[str, str, str]]] = [ - ('exclude=', 'e', "exclude those files when discovering plugins"), + ('exclude=', 'e', "a sequence of paths to exclude; '*' can be used as a wildcard in the names. 'foo.*' will exclude all subpackages of 'foo' (but not 'foo' itself)."), # TODO: add more ] @@ -414,7 +414,7 @@ def _path_to_module(path): Convert a path to a Python module to its module representation Example: plux/core/test -> plux.core.test """ - return path.strip("/").replace("/", ".") + return '.'.join(Path(path).with_suffix('').parts) class _Filter: diff --git a/plux/cli/cli.py b/plux/cli/cli.py index a0fb230..c2fb7e3 100644 --- a/plux/cli/cli.py +++ b/plux/cli/cli.py @@ -83,7 +83,7 @@ def main(argv=None): generate_parser = subparsers.add_parser( "entrypoints", help="Discover plugins and generate entry points" ) - generate_parser.add_argument("-e", "--exclude", help="path(s) to exclude") + generate_parser.add_argument("-e", "--exclude", help="a sequence of paths to exclude; '*' can be used as a wildcard in the names. 'foo.*' will exclude all subpackages of 'foo' (but not 'foo' itself).") generate_parser.set_defaults(func=entrypoints) # Subparser for the 'discover' subcommand From 94c694e873f7a74d2f3f43ed4011f035bc115b92 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Mon, 4 Aug 2025 23:05:08 +0200 Subject: [PATCH 7/7] address PR comments --- plux/build/setuptools.py | 22 +++++++++++++++++----- plux/cli/cli.py | 6 +++++- tests/cli/test_entrypoints.py | 4 ++-- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/plux/build/setuptools.py b/plux/build/setuptools.py index 35533e9..e8b8ebb 100644 --- a/plux/build/setuptools.py +++ b/plux/build/setuptools.py @@ -58,7 +58,9 @@ def finalize_options(self) -> None: if self.exclude is None: self.exclude = [] - project_config = read_configuration(self.distribution) + # we merge the configuration from the CLI arguments with the configuration read from the `pyproject.toml` + # [tool.plux] section + project_config = read_plux_configuration(self.distribution) file_exclude = project_config.get("exclude") if file_exclude: self.exclude = set(self.exclude) | set(file_exclude) @@ -204,12 +206,19 @@ def get_plux_json_path(distribution): return os.path.join(egg_info_dir, "plux.json") -def read_configuration(distribution) -> dict: +def read_plux_configuration(distribution) -> dict: + """ + Try reading the [tool.plux] section of the `pyproject.toml` TOML file of the Distribution, and returns it as a + dictionary. + """ if find_spec("tomllib"): + # the tomllib library is part of the standard library since 3.11 from tomllib import load as load_toml elif find_spec("tomli"): + # setuptools vendors the tomli library in 3.10 from tomli import load as load_toml else: + # if we cannot find a TOML lib, we do not return any configuration return {} dirs = distribution.package_dir @@ -257,8 +266,8 @@ def load_entry_points( instead, acting as sort of a cache. :param where: the file path to look for plugins (default, the current working dir) - :param exclude: the glob patterns to exclude - :param include: the glob patterns to include + :param exclude: the shell style wildcard patterns to exclude + :param include: the shell style wildcard patterns to include :param merge: a map of entry points that are always added """ should_read, meta_dir = _should_read_existing_egg_info() @@ -414,7 +423,7 @@ def _path_to_module(path): Convert a path to a Python module to its module representation Example: plux/core/test -> plux.core.test """ - return '.'.join(Path(path).with_suffix('').parts) + return ".".join(Path(path).with_suffix("").parts) class _Filter: @@ -450,6 +459,9 @@ class DistributionPackageFinder(_PackageFinder): then the ``[tool.setuptools.package.find]`` config will be interpreted, resolved, and then ``distribution.packages`` will contain the resolved packages. This already contains namespace packages correctly if configured. + You can additionally pass a sequence of values to the `exclude` parameters to provide a list of Unix shell style + patterns that will be matched against the Python packages to exclude them from the resolved packages. + Wildcards are allowed in the patterns with '*'. 'foo.*' will exclude all subpackages of 'foo' (but not 'foo' itself) """ def __init__(self, distribution: Distribution, exclude: t.Optional[t.Iterable[str]] = None): diff --git a/plux/cli/cli.py b/plux/cli/cli.py index c2fb7e3..9420f82 100644 --- a/plux/cli/cli.py +++ b/plux/cli/cli.py @@ -83,7 +83,11 @@ def main(argv=None): generate_parser = subparsers.add_parser( "entrypoints", help="Discover plugins and generate entry points" ) - generate_parser.add_argument("-e", "--exclude", help="a sequence of paths to exclude; '*' can be used as a wildcard in the names. 'foo.*' will exclude all subpackages of 'foo' (but not 'foo' itself).") + generate_parser.add_argument( + "-e", + "--exclude", + help="a sequence of paths to exclude; '*' can be used as a wildcard in the names. 'foo.*' will exclude all subpackages of 'foo' (but not 'foo' itself).", + ) generate_parser.set_defaults(func=entrypoints) # Subparser for the 'discover' subcommand diff --git a/tests/cli/test_entrypoints.py b/tests/cli/test_entrypoints.py index 6ae0d14..8835667 100644 --- a/tests/cli/test_entrypoints.py +++ b/tests/cli/test_entrypoints.py @@ -52,7 +52,7 @@ def test_entrypoints_exclude(project_name): sys.path.append(project) try: try: - main(["--workdir", project, "entrypoints", "--exclude", "**/subpkg*"]) + main(["--workdir", project, "entrypoints", "--exclude", "*/subpkg*"]) except SystemExit: pass finally: @@ -86,7 +86,7 @@ def test_entrypoints_exclude_from_pyproject_config(tmp_path): pyproject_toml_path = os.path.join(dest_project, "pyproject.toml") with open(pyproject_toml_path, "a") as fp: - fp.write('\n[tool.plux]\nexclude = ["**subpkg*"]\n') + fp.write('\n[tool.plux]\nexclude = ["*.subpkg.*"]\n') os.chdir(dest_project)