Add ruff as default linting + formatting option

ccds-help.json

@@ -202,6 +202,29 @@
             }
         ]
     },
+    {
+        "field": "linting_and_formatting",
+        "help": {
+            "description": "How to handle linting and formatting on your code.",
+            "more_information": ""
+        },
+        "choices": [
+            {
+                "choice": "ruff",
+                "help": {
+                    "description": "Use ruff for linting and formatting.",
+                    "more_information": ""
+                } 
+            },
+            {
+                "choice": "flake8+black+isort",
+                "help": {
+                    "description": "Use flake8 for linting and black+isort for formatting.",
+                    "more_information": ""
+                } 
+            }
+        ]
+    },
     {
         "field": "open_source_license",
         "help": {

ccds.json

@@ -26,6 +26,10 @@
         "none",
         "basic"
     ],
+    "linting_and_formatting": [
+        "ruff",
+        "flake8+black+isort"
+    ],
     "open_source_license": ["No license file", "MIT", "BSD-3-Clause"],
     "docs": ["mkdocs", "none"],
     "include_code_scaffold": ["Yes", "No"]

ccds/hook_utils/dependencies.py

@@ -1,11 +1,16 @@
 packages = [
+    "pip",
+    "python-dotenv",
+]
+
+flake8_black_isort = [
     "black",
     "flake8",
     "isort",
-    "pip",
-    "python-dotenv",
 ]
 
+ruff = ["ruff"]
+
 basic = [
     "ipython",
     "jupyterlab",

docs/scripts/generate-termynal.py

@@ -1,3 +1,4 @@
+import json
 import shutil
 from pathlib import Path
 
@@ -49,14 +50,16 @@ def execute_command_and_get_output(command, input_script):
     ("author_name", "Dat A. Scientist"),
     ("description", "This is my analysis of the data."),
     ("python_version_number", "3.12"),
-    ("Choose from", "3"),
+    ("Choose from", "3"),  # dataset_storage
     ("bucket", "s3://my-aws-bucket"),
     ("aws_profile", ""),
-    ("Choose from", "2"),
-    ("Choose from", "1"),
-    ("Choose from", "2"),
-    ("Choose from", "2"),
-    ("Choose from", "1"),
+    ("Choose from", "2"),  # environment_manager
+    ("Choose from", "1"),  # dependency_file
+    ("Choose from", "2"),  # pydata_packages
+    ("Choose from", "1"),  # linting_and_formatting
+    ("Choose from", "2"),  # open_source_license
+    ("Choose from", "1"),  # docs
+    ("Choose from", "2"),  # include_code_scaffold
 ]
 
 
@@ -128,6 +131,11 @@ def render_termynal():
     html_lines.append("</div>")
     output = "\n".join(html_lines)
 
+    # Ensure that all options are contained in the output
+    options = json.load((CCDS_ROOT / "ccds.json").open("r")).keys()
+    for option in options:
+        assert option in output, f'Option "{option}" not found in termynal output.'
+
     # replace local directory in ccds call with URL so it can be used for documentation
     output = output.replace(
         str(CCDS_ROOT), "https://github.com/drivendataorg/cookiecutter-data-science"

hooks/post_gen_project.py

@@ -5,7 +5,14 @@
 # https://github.com/cookiecutter/cookiecutter/issues/824
 #   our workaround is to include these utility functions in the CCDS package
 from ccds.hook_utils.custom_config import write_custom_config
-from ccds.hook_utils.dependencies import basic, packages, scaffold, write_dependencies
+from ccds.hook_utils.dependencies import (
+    basic,
+    flake8_black_isort,
+    packages,
+    ruff,
+    scaffold,
+    write_dependencies,
+)
 
 #
 #  TEMPLATIZED VARIABLES FILLED IN BY COOKIECUTTER
@@ -24,6 +31,13 @@
 packages_to_install += basic
 # {% endif %}
 
+# {% if cookiecutter.linting_and_formatting == "ruff" %}
+packages_to_install += ruff
+# Remove setup.cfg
+Path("setup.cfg").unlink()
+# {% elif cookiecutter.linting_and_formatting == "flake8+black+isort" %}
+packages_to_install += flake8_black_isort
+# {% endif %}
 # track packages that are not available through conda
 pip_only_packages = [
     "awscli",

tests/conda_harness.sh

@@ -34,5 +34,7 @@ make
 make create_environment
 conda activate $PROJECT_NAME
 make requirements
+make lint
+make format
 
 run_tests $PROJECT_NAME $MODULE_NAME

tests/conftest.py

@@ -57,22 +57,40 @@ def _is_valid(config):
     # remove invalid configs
     configs = [c for c in configs if _is_valid(c)]
 
-    # cycle over all values other multi-select fields that should be inter-operable
+    # ensure linting and formatting options are run on code scaffold
+    # otherwise, linting "passes" because one linter never runs on any code during tests
+    code_format_cycler = cycle(
+        product(
+            [
+                ("include_code_scaffold", opt)
+                for opt in cookiecutter_json["include_code_scaffold"]
+            ],
+            [
+                ("linting_and_formatting", opt)
+                for opt in cookiecutter_json["linting_and_formatting"]
+            ],
+        )
+    )
+
+    # cycle over values for multi-select fields that should be inter-operable
     # and that we don't need to handle with combinatorics
     cycle_fields = [
         "dataset_storage",
         "open_source_license",
-        "include_code_scaffold",
         "docs",
     ]
-    cyclers = {k: cycle(cookiecutter_json[k]) for k in cycle_fields}
+    multi_select_cyclers = {k: cycle(cookiecutter_json[k]) for k in cycle_fields}
 
     for ind, c in enumerate(configs):
         config = dict(c)
         config.update(default_args)
-        # Alternate including the code scaffold
-        for field, cycler in cyclers.items():
+
+        code_format_settings = dict(next(code_format_cycler))
+        config.update(code_format_settings)
+
+        for field, cycler in multi_select_cyclers.items():
             config[field] = next(cycler)
+
         config["repo_name"] += f"-{ind}"
         yield config
 

tests/pipenv_harness.sh

@@ -26,6 +26,10 @@ make create_environment
 # can happen outside of environment since pipenv knows based on Pipfile
 make requirements
 
+# linting + formatting must happen inside environment
+pipenv run make lint
+pipenv run make format
+
 # test with pipenv run
 pipenv run python -c "import sys; assert \"$PROJECT_NAME\" in sys.executable"
 

tests/test_creation.py

@@ -49,7 +49,6 @@ def test_baking_configs(config, fast):
     with bake_project(config) as project_directory:
         verify_folders(project_directory, config)
         verify_files(project_directory, config)
-        lint(project_directory)
 
         if fast < 2:
             verify_makefile_commands(project_directory, config)
@@ -100,7 +99,6 @@ def verify_files(root, config):
         "Makefile",
         "README.md",
         "pyproject.toml",
-        "setup.cfg",
         ".env",
         ".gitignore",
         "data/external/.gitkeep",
@@ -120,6 +118,9 @@ def verify_files(root, config):
     if not config["open_source_license"].startswith("No license"):
         expected_files.append("LICENSE")
 
+    if config["linting_and_formatting"] == "flake8+black+isort":
+        expected_files.append("setup.cfg")
+
     if config["include_code_scaffold"] == "Yes":
         expected_files += [
             f"{config['module_name']}/config.py",
@@ -156,6 +157,8 @@ def verify_makefile_commands(root, config):
     - blank command listing commands
     - create_environment
     - requirements
+    - linting
+    - formatting
     Ensure that these use the proper environment.
     """
     test_path = Path(__file__).parent
@@ -184,23 +187,20 @@ def verify_makefile_commands(root, config):
         stdout=PIPE,
     )
 
-    stdout_output, _ = _decode_print_stdout_stderr(result)
+    stdout_output, stderr_output = _decode_print_stdout_stderr(result)
 
     # Check that makefile help ran successfully
     assert "Available rules:" in stdout_output
     assert "clean                    Delete all compiled Python files" in stdout_output
 
-    assert result.returncode == 0
-
-
-def lint(root):
-    """Run the linters on the project."""
-    result = run(
-        ["make", "lint"],
-        cwd=root,
-        stderr=PIPE,
-        stdout=PIPE,
-    )
-    _, _ = _decode_print_stdout_stderr(result)
+    # Check that linting and formatting ran successfully
+    if config["linting_and_formatting"] == "ruff":
+        assert "All checks passed!" in stdout_output
+        assert "left unchanged" in stdout_output
+        assert "reformatted" not in stdout_output
+    elif config["linting_and_formatting"] == "flake8+black+isort":
+        assert "All done!" in stderr_output
+        assert "left unchanged" in stderr_output
+        assert "reformatted" not in stderr_output
 
     assert result.returncode == 0

tests/virtualenv_harness.sh

@@ -55,6 +55,8 @@ else
 fi
 
 make requirements
+make lint
+make format
 
 run_tests $PROJECT_NAME $MODULE_NAME
 

{{ cookiecutter.repo_name }}/.gitignore

@@ -8,7 +8,7 @@
 *.swp
 *.swo
 
-## https://github.com/github/gitignore/blob/4488915eec0b3a45b5c63ead28f286819c0917de/Python.gitignore
+## https://github.com/github/gitignore/blob/e8554d85bf62e38d6db966a50d2064ac025fd82a/Python.gitignore
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -106,6 +106,12 @@ ipython_config.py
 #   install all needed dependencies.
 #Pipfile.lock
 
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
 # poetry
 #   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
 #   This is especially recommended for binary packages to ensure reproducibility, and is more
@@ -118,8 +124,10 @@ ipython_config.py
 #pdm.lock
 #   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
 #   in version control.
-#   https://pdm.fming.dev/#use-with-ide
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
 .pdm.toml
+.pdm-python
+.pdm-build/
 
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
 __pypackages__/
@@ -170,3 +178,9 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc

{{ cookiecutter.repo_name }}/Makefile

@@ -11,7 +11,7 @@ PYTHON_INTERPRETER = python
 #################################################################################
 
 {% if cookiecutter.dependency_file != 'none' %}
-## Install Python Dependencies
+## Install Python dependencies
 .PHONY: requirements
 requirements:
 	{% if "requirements.txt" == cookiecutter.dependency_file -%}
@@ -31,17 +31,32 @@ clean:
 	find . -type f -name "*.py[co]" -delete
 	find . -type d -name "__pycache__" -delete
 
-## Lint using flake8 and black (use `make format` to do formatting)
+{% if cookiecutter.linting_and_formatting == 'ruff' %}
+## Lint using ruff (use `make format` to do formatting)
+.PHONY: lint
+lint:
+	ruff format --check
+	ruff check
+
+## Format source code with ruff
+.PHONY: format
+format:
+	ruff check --fix
+	ruff format
+{% elif cookiecutter.linting_and_formatting == 'flake8+black+isort' %}
+## Lint using flake8, black, and isort (use `make format` to do formatting)
 .PHONY: lint
 lint:
 	flake8 {{ cookiecutter.module_name }}
-	isort --check --diff --profile black {{ cookiecutter.module_name }}
-	black --check --config pyproject.toml {{ cookiecutter.module_name }}
+	isort --check --diff {{ cookiecutter.module_name }}
+	black --check {{ cookiecutter.module_name }}
 
 ## Format source code with black
 .PHONY: format
 format:
-	black --config pyproject.toml {{ cookiecutter.module_name }}
+	isort {{ cookiecutter.module_name }}
+	black {{ cookiecutter.module_name }}
+{% endif %}
 
 {% if not cookiecutter.dataset_storage.none %}
 ## Download Data from storage system
@@ -72,7 +87,7 @@ sync_data_up:
 {% endif %}
 
 {% if cookiecutter.environment_manager != 'none' %}
-## Set up python interpreter environment
+## Set up Python interpreter environment
 .PHONY: create_environment
 create_environment:
 	{% if cookiecutter.environment_manager == 'conda' -%}
@@ -97,7 +112,7 @@ create_environment:
 #################################################################################
 
 {% if cookiecutter.include_code_scaffold == 'Yes' %}
-## Make Dataset
+## Make dataset
 .PHONY: data
 data: requirements
 	$(PYTHON_INTERPRETER) {{ cookiecutter.module_name }}/dataset.py