diff --git a/README.md b/README.md index 55c9aac..3cfa7e7 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,29 @@ build-backend = "setuptools.build_meta" # ... ``` +Additional configuration +------------------------ + +You can pass additional configuration to Plux, either via the command line or your project `pyproject.toml`. + +### Excluding Python packages during discovery + +When [discovering entrypoints](#discovering-entrypoints), Plux will try importing your code to discover Plugins. +Some parts of your codebase might have side effects, or raise errors when imported outside a specific context like some database +migration scripts. + +You can ignore those Python packages by specifying the `--exclude` flag to the entrypoints discovery commands (`python -m plux entrypoints` or `python setup.py plugins`). +The option takes a list of comma-separated values that can be paths or package names. + +You can also specify those values in the `tool.plux` section of your `pyproject.toml`: + +```toml +# ... + +[tool.plux] +exclude = ["**/database/alembic*"] +``` + Install ------- diff --git a/plugin/entrypoint.py b/plugin/entrypoint.py index 52176b2..288c5b4 100644 --- a/plugin/entrypoint.py +++ b/plugin/entrypoint.py @@ -1,6 +1,7 @@ """ Deprecated bindings, use plux imports instead, but you shouldn't use the internals in the first place. """ + from plux.build.setuptools import find_plugins from plux.core.entrypoint import ( EntryPoint, diff --git a/plux/build/setuptools.py b/plux/build/setuptools.py index bc8a87e..e8b8ebb 100644 --- a/plux/build/setuptools.py +++ b/plux/build/setuptools.py @@ -9,6 +9,8 @@ import shutil import sys import typing as t +from fnmatch import fnmatchcase +from importlib.util import find_spec from pathlib import Path import setuptools @@ -39,20 +41,33 @@ def _ensure_dist_info(self, *args, **kwargs): class plugins(InfoCommon, setuptools.Command): description = "Discover plux plugins and store them in .egg_info" - user_options = [ - # TODO + user_options: t.ClassVar[t.List[t.Tuple[str, str, str]]] = [ + ('exclude=', 'e', "a sequence of paths to exclude; '*' can be used as a wildcard in the names. 'foo.*' will exclude all subpackages of 'foo' (but not 'foo' itself)."), + # TODO: add more ] egg_info: str def initialize_options(self) -> None: self.plux_json_path = None + self.exclude = None def finalize_options(self) -> None: self.plux_json_path = get_plux_json_path(self.distribution) + self.ensure_string_list('exclude') + if self.exclude is None: + self.exclude = [] + + # we merge the configuration from the CLI arguments with the configuration read from the `pyproject.toml` + # [tool.plux] section + project_config = read_plux_configuration(self.distribution) + file_exclude = project_config.get("exclude") + if file_exclude: + self.exclude = set(self.exclude) | set(file_exclude) + self.exclude = [_path_to_module(item) for item in self.exclude] def run(self) -> None: - plugin_finder = PluginFromPackageFinder(DistributionPackageFinder(self.distribution)) + plugin_finder = PluginFromPackageFinder(DistributionPackageFinder(self.distribution, exclude=self.exclude)) ep = discover_entry_points(plugin_finder) self.debug_print(f"writing discovered plugins into {self.plux_json_path}") @@ -191,6 +206,34 @@ def get_plux_json_path(distribution): return os.path.join(egg_info_dir, "plux.json") +def read_plux_configuration(distribution) -> dict: + """ + Try reading the [tool.plux] section of the `pyproject.toml` TOML file of the Distribution, and returns it as a + dictionary. + """ + if find_spec("tomllib"): + # the tomllib library is part of the standard library since 3.11 + from tomllib import load as load_toml + elif find_spec("tomli"): + # setuptools vendors the tomli library in 3.10 + from tomli import load as load_toml + else: + # if we cannot find a TOML lib, we do not return any configuration + return {} + + dirs = distribution.package_dir + pyproject_base = (dirs or {}).get("", os.curdir) + pyproject_file = os.path.join(pyproject_base, "pyproject.toml") + if not os.path.exists(pyproject_file): + return {} + + with open(pyproject_file, "rb") as file: + pyproject_config = load_toml(file) + + tool_table = pyproject_config.get("tool", {}) + return tool_table.get("plux", {}) + + def update_entrypoints(distribution, ep: EntryPointDict): if distribution.entry_points is None: distribution.entry_points = {} @@ -223,8 +266,8 @@ def load_entry_points( instead, acting as sort of a cache. :param where: the file path to look for plugins (default, the current working dir) - :param exclude: the glob patterns to exclude - :param include: the glob patterns to include + :param exclude: the shell style wildcard patterns to exclude + :param include: the shell style wildcard patterns to include :param merge: a map of entry points that are always added """ should_read, meta_dir = _should_read_existing_egg_info() @@ -375,6 +418,27 @@ def _to_filename(name): return name.replace("-", "_") +def _path_to_module(path): + """ + Convert a path to a Python module to its module representation + Example: plux/core/test -> plux.core.test + """ + return ".".join(Path(path).with_suffix("").parts) + + +class _Filter: + """ + Given a list of patterns, create a callable that will be true only if + the input matches at least one of the patterns. + This is from `setuptools.discovery._Filter` + """ + def __init__(self, patterns: t.Iterable[str]): + self._patterns = patterns + + def __call__(self, item: str): + return any(fnmatchcase(item, pat) for pat in self._patterns) + + class _PackageFinder: """ Generate a list of Python packages. How these are generated depends on the implementation. @@ -395,13 +459,17 @@ class DistributionPackageFinder(_PackageFinder): then the ``[tool.setuptools.package.find]`` config will be interpreted, resolved, and then ``distribution.packages`` will contain the resolved packages. This already contains namespace packages correctly if configured. + You can additionally pass a sequence of values to the `exclude` parameters to provide a list of Unix shell style + patterns that will be matched against the Python packages to exclude them from the resolved packages. + Wildcards are allowed in the patterns with '*'. 'foo.*' will exclude all subpackages of 'foo' (but not 'foo' itself) """ - def __init__(self, distribution: Distribution): + def __init__(self, distribution: Distribution, exclude: t.Optional[t.Iterable[str]] = None): self.distribution = distribution + self.exclude = _Filter(exclude or []) def find_packages(self) -> t.Iterable[str]: - return self.distribution.packages + return self.filter_packages(self.distribution.packages) @property def path(self) -> str: @@ -415,6 +483,9 @@ def path(self) -> str: where = "." return where + def filter_packages(self, packages: t.Iterable[str]) -> t.Iterable[str]: + return [item for item in packages if not self.exclude(item)] + class DefaultPackageFinder(_PackageFinder): def __init__(self, where=".", exclude=(), include=("*",), namespace=True) -> None: diff --git a/plux/cli/cli.py b/plux/cli/cli.py index 5a3bd97..9420f82 100644 --- a/plux/cli/cli.py +++ b/plux/cli/cli.py @@ -20,6 +20,7 @@ def entrypoints(args): dist = get_distribution_from_workdir(os.getcwd()) print("discovering plugins ...") + dist.command_options["plugins"] = {"exclude": ("command line", args.exclude)} dist.run_command("plugins") print(f"building {dist.get_name().replace('-', '_')}.egg-info...") @@ -82,6 +83,11 @@ def main(argv=None): generate_parser = subparsers.add_parser( "entrypoints", help="Discover plugins and generate entry points" ) + generate_parser.add_argument( + "-e", + "--exclude", + help="a sequence of paths to exclude; '*' can be used as a wildcard in the names. 'foo.*' will exclude all subpackages of 'foo' (but not 'foo' itself).", + ) generate_parser.set_defaults(func=entrypoints) # Subparser for the 'discover' subcommand diff --git a/tests/cli/projects/pyproject/mysrc/subpkg/__init__.py b/tests/cli/projects/pyproject/mysrc/subpkg/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/projects/pyproject/mysrc/subpkg/plugins.py b/tests/cli/projects/pyproject/mysrc/subpkg/plugins.py new file mode 100644 index 0000000..163edf8 --- /dev/null +++ b/tests/cli/projects/pyproject/mysrc/subpkg/plugins.py @@ -0,0 +1,6 @@ +from plugin import Plugin + + +class MyNestedPlugin(Plugin): + namespace = "plux.test.plugins" + name = "mynestedplugin" diff --git a/tests/cli/projects/setupcfg/mysrc/subpkg/__init__.py b/tests/cli/projects/setupcfg/mysrc/subpkg/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/projects/setupcfg/mysrc/subpkg/plugins.py b/tests/cli/projects/setupcfg/mysrc/subpkg/plugins.py new file mode 100644 index 0000000..163edf8 --- /dev/null +++ b/tests/cli/projects/setupcfg/mysrc/subpkg/plugins.py @@ -0,0 +1,6 @@ +from plugin import Plugin + + +class MyNestedPlugin(Plugin): + namespace = "plux.test.plugins" + name = "mynestedplugin" diff --git a/tests/cli/test_entrypoints.py b/tests/cli/test_entrypoints.py index 9849425..8835667 100644 --- a/tests/cli/test_entrypoints.py +++ b/tests/cli/test_entrypoints.py @@ -1,6 +1,6 @@ import os.path +import shutil import sys -from pathlib import Path import pytest @@ -26,10 +26,88 @@ def test_entrypoints(project_name): with open(os.path.join(project, "test_project.egg-info", "entry_points.txt"), "r") as f: lines = [line.strip() for line in f.readlines() if line.strip()] - assert lines == ["[plux.test.plugins]", "myplugin = mysrc.plugins:MyPlugin"] + assert lines == [ + "[plux.test.plugins]", + "mynestedplugin = mysrc.subpkg.plugins:MyNestedPlugin", + "myplugin = mysrc.plugins:MyPlugin", + ] # make sure that SOURCES.txt contain no absolute paths with open(os.path.join(project, "test_project.egg-info", "SOURCES.txt"), "r") as f: lines = [line.strip() for line in f.readlines() if line.strip()] for line in lines: assert not line.startswith("/") + + +@pytest.mark.parametrize("project_name", ["pyproject", "setupcfg"]) +def test_entrypoints_exclude(project_name): + if project_name == "pyproject" and sys.version_info < (3, 10): + pytest.xfail("reading pyproject.toml requires Python 3.10 or above") + + from plux.__main__ import main + + project = os.path.join(os.path.dirname(__file__), "projects", project_name) + os.chdir(project) + + sys.path.append(project) + try: + try: + main(["--workdir", project, "entrypoints", "--exclude", "*/subpkg*"]) + except SystemExit: + pass + finally: + sys.path.remove(project) + + with open(os.path.join(project, "test_project.egg-info", "entry_points.txt"), "r") as f: + lines = [line.strip() for line in f.readlines() if line.strip()] + assert lines == [ + "[plux.test.plugins]", + "myplugin = mysrc.plugins:MyPlugin", + ] + + # make sure that SOURCES.txt contain no absolute paths + with open(os.path.join(project, "test_project.egg-info", "SOURCES.txt"), "r") as f: + lines = [line.strip() for line in f.readlines() if line.strip()] + for line in lines: + assert not line.startswith("/") + + +def test_entrypoints_exclude_from_pyproject_config(tmp_path): + if sys.version_info < (3, 10): + pytest.xfail("reading pyproject.toml requires Python 3.10 or above") + + from plux.__main__ import main + + src_project = os.path.join(os.path.dirname(__file__), "projects", "pyproject") + dest_project = os.path.join(str(tmp_path), "pyproject") + + shutil.copytree(src_project, dest_project) + + pyproject_toml_path = os.path.join(dest_project, "pyproject.toml") + + with open(pyproject_toml_path, "a") as fp: + fp.write('\n[tool.plux]\nexclude = ["*.subpkg.*"]\n') + + os.chdir(dest_project) + + sys.path.append(dest_project) + try: + try: + main(["--workdir", dest_project, "entrypoints"]) + except SystemExit: + pass + finally: + sys.path.remove(dest_project) + + with open(os.path.join(dest_project, "test_project.egg-info", "entry_points.txt"), "r") as f: + lines = [line.strip() for line in f.readlines() if line.strip()] + assert lines == [ + "[plux.test.plugins]", + "myplugin = mysrc.plugins:MyPlugin", + ] + + # make sure that SOURCES.txt contain no absolute paths + with open(os.path.join(dest_project, "test_project.egg-info", "SOURCES.txt"), "r") as f: + lines = [line.strip() for line in f.readlines() if line.strip()] + for line in lines: + assert not line.startswith("/")