diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..d8872e9 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,49 @@ +name: Deploy docs + +on: + push: + branches: + - main + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install docs dependencies + run: pip install ".[docs]" + + - name: Build docs + run: mkdocs build + + - name: Upload pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: site/ + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 91c24bd..c37f387 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,4 +34,4 @@ jobs: run: make test - name: Run example - run: resqui -c configurations/basic.json -t ${{ secrets.GITHUB_TOKEN }} + run: venv/bin/resqui -c configurations/basic.json -t ${{ secrets.GITHUB_TOKEN }} diff --git a/Makefile b/Makefile index f22b5bb..daa4e98 100644 --- a/Makefile +++ b/Makefile @@ -1,25 +1,37 @@ PKGNAME=resqui +VENV=venv +PYTHON=$(VENV)/bin/python +PIP=$(VENV)/bin/pip -install: - pip install . +$(VENV)/bin/activate: + python3 -m venv $(VENV) + $(PIP) install -e ".[dev,docs]" -install-dev: - pip install -e ".[dev]" +install: $(VENV)/bin/activate -venv: - python3 -m venv venv +install-dev: $(VENV)/bin/activate -test: - python3 run_tests.py +install-docs: $(VENV)/bin/activate -example: - resqui -c configurations/basic.json +venv: $(VENV)/bin/activate -black: - black src/$(PKGNAME) - black tests +test: $(VENV)/bin/activate + $(PYTHON) run_tests.py + +example: $(VENV)/bin/activate + $(VENV)/bin/resqui -c configurations/basic.json + +black: $(VENV)/bin/activate + $(VENV)/bin/black src/$(PKGNAME) + $(VENV)/bin/black tests + +docs: $(VENV)/bin/activate + $(VENV)/bin/mkdocs build + +docs-serve: $(VENV)/bin/activate + $(VENV)/bin/mkdocs serve clean: - rm -rf venv + rm -rf venv site -.PHONY: install install-dev venv test example black clean +.PHONY: install install-dev install-docs venv test example black docs docs-serve clean diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md new file mode 100644 index 0000000..be8fe4a --- /dev/null +++ b/docs/explanation/architecture.md @@ -0,0 +1,63 @@ +# Architecture + +## Overview + +resqui is built around three concepts: **plugins**, **executors**, and a +**configuration-driven pipeline**. The CLI wires them together but the +components are independent. + +``` +CLI (cli.py) + │ + ├─ Configuration (config.py) ← JSON file or built-in defaults + │ + ├─ GitInspector (cli.py) ← extracts metadata from the repo + │ + ├─ IndicatorPlugin subclasses ← one per tool (HowFairIs, Gitleaks, …) + │ └─ uses Executor ← PythonExecutor or DockerExecutor + │ + └─ Summary (core.py) ← aggregates CheckResults → JSON-LD +``` + +## Plugin system + +Every quality check is a subclass of `IndicatorPlugin`. The CLI discovers +plugins at runtime via `__subclasses__()` — no registration step is needed. +Each plugin declares a list of indicator names in its `indicators` class +attribute; each name corresponds to a method of the same name on the class. + +A single plugin instance is reused for all of its indicators within one run, +which means Docker images are pulled and Python venvs are created only once +per plugin class. + +## Executor design + +Plugins delegate subprocess execution to one of two executors: + +**`PythonExecutor`** creates a temporary `venv` via the stdlib `venv` module, +installs the required packages with pip, and runs Python snippets inside it. +The venv is torn down in `__del__`. This approach keeps plugin dependencies +completely isolated from the resqui installation. + +**`DockerExecutor`** pulls a Docker image on construction and runs commands +inside containers via `docker run`. The Docker daemon is the only external +dependency. + +Both raise `ExecutorInitError` when they cannot initialise (Docker unavailable, +pip install failure, etc.). The CLI catches this and skips all indicators +belonging to that plugin with a warning, allowing the rest of the run to +continue. + +## Why not extend IndicatorPlugin with Python magic? + +Subclassing is used purely as a discovery mechanism — `__subclasses__()` gives +a flat list of all available plugins without any import-side-effect registration. +There is no `__init_subclass__` hook or metaclass. This keeps plugin authorship +simple: write a class, import it, and it is automatically available. + +## Output format + +`Summary.to_json()` serialises results as JSON-LD conforming to the EVERSE +Research Software Quality Assessment schema +(`https://w3id.org/everse/rsqa/0.0.1/`). Each `CheckResult` maps to one entry +in the `checks` array with linked indicator, software, and status IRIs. diff --git a/docs/explanation/indicators.md b/docs/explanation/indicators.md new file mode 100644 index 0000000..2a2a98f --- /dev/null +++ b/docs/explanation/indicators.md @@ -0,0 +1,65 @@ +# Indicators + +An **indicator** is a measurable property of a software repository. resqui maps +each indicator to a plugin method that performs the check automatically. + +## Built-in indicators + +### `has_license` — HowFairIs + +Looks for a file named `LICENSE` or `LICENSE.md` in the repository root using +the [howfairis](https://github.com/fair-software/howfairis) library. Requires a +GitHub token. + +W3ID: `https://w3id.org/everse/i/indicators/license` + +### `has_citation` — CFFConvert + +Checks for a valid `CITATION.cff` file using +[cffconvert](https://github.com/citation-file-format/cffconvert). Both +presence and schema validity are verified. + +W3ID: `https://w3id.org/everse/i/indicators/citation` + +### `has_ci_tests` — OpenSSFScorecard + +Checks whether the project has a functioning CI test setup, as determined by +the [OpenSSF Scorecard](https://github.com/ossf/scorecard). Runs via Docker. +Requires a GitHub token. + +### `human_code_review_requirement` — OpenSSFScorecard + +Checks whether pull requests require human review before merging, per the +OpenSSF Scorecard "Code-Review" check. + +### `has_published_package` — OpenSSFScorecard + +Checks whether the project publishes a package to a registry such as PyPI or +npm, per the OpenSSF Scorecard "Packaging" check. + +### `has_no_security_leak` — Gitleaks + +Scans the repository history for accidentally committed secrets (API keys, +tokens, passwords) using [Gitleaks](https://github.com/gitleaks/gitleaks). +Runs via Docker. + +## Interpreting results + +Each indicator produces a `CheckResult` with: + +| Field | Values | +|---|---| +| `output` | `valid` — indicator satisfied; `missing` — not found; `failed` — check error | +| `status` | Schema.org action status IRI | +| `evidence` | Human-readable finding from the underlying tool | + +An indicator returning `missing` or `failed` does **not** abort the run — all +configured indicators are always attempted. + +## Status IDs + +| Status IRI | Meaning | +|---|---| +| `schema:CompletedActionStatus` | Check passed | +| `schema:FailedActionStatus` | Check ran but found a problem | +| `missing` | Check could not be completed (plugin skipped) | diff --git a/docs/how-to/add-a-plugin.md b/docs/how-to/add-a-plugin.md new file mode 100644 index 0000000..9da6ce8 --- /dev/null +++ b/docs/how-to/add-a-plugin.md @@ -0,0 +1,103 @@ +# Add an Indicator Plugin + +This guide shows how to add a new quality indicator to resqui. + +## 1. Create the plugin module + +Add a new file under `src/resqui/plugins/`, for example `src/resqui/plugins/myplugin.py`. + +## 2. Subclass `IndicatorPlugin` + +```python +from resqui.plugins.base import IndicatorPlugin +from resqui.core import CheckResult + + +class MyPlugin(IndicatorPlugin): + name = "MyPlugin" + version = "1.0.0" + id = "https://w3id.org/everse/software/MyPlugin" + indicators = ["has_readme"] + + def __init__(self, context): + # context.github_token and context.dashverse_token are available + # Raise PluginInitError here if required credentials are missing + pass + + def has_readme(self, url, branch_or_commit): + # run your check here and return a CheckResult + found = ... # your check logic + return CheckResult( + process="Looks for a README file in the repository root.", + status_id="schema:CompletedActionStatus" if found else "schema:FailedActionStatus", + output="valid" if found else "missing", + evidence="Found README." if found else "No README found.", + success=found, + ) +``` + +### Using PythonExecutor + +```python +from resqui.executors.python import PythonExecutor + +class MyPlugin(IndicatorPlugin): + def __init__(self, context): + self.executor = PythonExecutor(packages=["some-package==1.2.3"]) + + def my_indicator(self, url, branch_or_commit): + result = self.executor.execute(f""" +import some_package +output = some_package.check("{url}") +print(output) +""") + ... +``` + +### Using DockerExecutor + +```python +from resqui.executors.docker import DockerExecutor + +class MyPlugin(IndicatorPlugin): + def __init__(self, context): + self.executor = DockerExecutor("ghcr.io/org/image:latest") + + def my_indicator(self, url, branch_or_commit): + result = self.executor.run(["check", url]) + ... +``` + +Both executors raise `ExecutorInitError` on startup failure (e.g. Docker not +available, pip install failed). resqui catches this and skips the plugin with a +warning rather than aborting the whole run. + +## 3. Export from the plugins package + +Add an import to `src/resqui/plugins/__init__.py`: + +```python +from resqui.plugins.myplugin import MyPlugin +``` + +## 4. Reference it in a configuration file + +```json +{ + "indicators": [ + { + "name": "has_readme", + "plugin": "MyPlugin", + "@id": "missing" + } + ] +} +``` + +## 5. Verify + +```bash +resqui indicators +``` + +Your plugin and its indicators should appear in the list. diff --git a/docs/how-to/ci-integration.md b/docs/how-to/ci-integration.md new file mode 100644 index 0000000..b73e1df --- /dev/null +++ b/docs/how-to/ci-integration.md @@ -0,0 +1,70 @@ +# Integrate in GitHub Actions + +resqui ships as a reusable GitHub Action. Add it to any repository to run +quality assessments automatically on every push. + +## Prerequisites + +1. Go to your repository → **Settings → Actions → General** and ensure + "Allow all actions and reusable workflows" is selected. + +2. Create an environment called **resqui** at + `https://github.com/USER_OR_GROUP/PROJECT/settings/environments`. + +3. Add a secret named `DASHVERSE_TOKEN` to that environment (obtain it from + your DashVerse instance). If you do not have a DashVerse token the step + will report a failure but the assessment itself still runs. + +## Minimal workflow + +Create `.github/workflows/resqui.yml` in your repository: + +```yaml +name: Run resqui + +on: + push: + +jobs: + run-resqui: + runs-on: ubuntu-latest + environment: resqui + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run resqui action + uses: EVERSE-ResearchSoftware/QualityPipelines@main + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + dashverse_token: ${{ secrets.DASHVERSE_TOKEN }} +``` + +## Using a custom configuration + +Place a config file (e.g. `.resqui.json`) at the root of your repository and +pass its path to the action: + +```yaml + - name: Run resqui action + uses: EVERSE-ResearchSoftware/QualityPipelines@main + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + dashverse_token: ${{ secrets.DASHVERSE_TOKEN }} + config: .resqui.json +``` + +## Running the CLI directly + +If you prefer managing the installation yourself: + +```yaml + - name: Install resqui + run: pip install resqui + + - name: Run resqui + run: resqui -t ${{ secrets.GITHUB_TOKEN }} -d ${{ secrets.DASHVERSE_TOKEN }} +``` diff --git a/docs/how-to/custom-configuration.md b/docs/how-to/custom-configuration.md new file mode 100644 index 0000000..ba2a606 --- /dev/null +++ b/docs/how-to/custom-configuration.md @@ -0,0 +1,57 @@ +# Write a Custom Configuration + +By default resqui runs the built-in indicator set. A JSON configuration file +lets you choose exactly which indicators to run and which plugins to use. + +## Configuration file format + +```json +{ + "indicators": [ + { + "name": "has_license", + "plugin": "HowFairIs", + "@id": "https://w3id.org/everse/i/indicators/license" + }, + { + "name": "has_citation", + "plugin": "CFFConvert", + "@id": "https://w3id.org/everse/i/indicators/citation" + } + ] +} +``` + +| Field | Description | +|---|---| +| `name` | Must match a method name on the plugin class | +| `plugin` | Class name of the plugin (see `resqui indicators`) | +| `@id` | W3ID URI for the indicator (use `"missing"` if no URI exists yet) | + +## Using a custom config + +Pass the file with `-c`: + +```bash +resqui -c my-config.json -t $GITHUB_TOKEN +``` + +## Example configurations + +The `configurations/` directory in the repository contains ready-made configs +for different use cases: + +| File | Purpose | +|---|---| +| `basic.json` | License and citation only | +| `indicators_analysis_code.json` | Indicators for analysis code | +| `indicators_prototype_tools.json` | Indicators for prototype tools | +| `indicators_rs_infrastructure.json` | Indicators for RS infrastructure | + +## Discovering available plugins and indicators + +```bash +resqui indicators +``` + +This prints every plugin class, its version, and the indicator names it supports. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..e02cd55 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,40 @@ +# resqui + +**resqui** is a command-line tool that checks research software quality indicators. +It analyses a Git repository against a configurable set of indicators and produces +a [JSON-LD](https://json-ld.org/) assessment report conforming to the +[EVERSE Research Software Quality Assessment schema](https://github.com/EVERSE-ResearchSoftware/schemas). + +## Requirements + +- Python 3.9+ +- Docker (required by several indicator plugins) + +## Installation + +```bash +pip install resqui +``` + +## Quick start + +Run against the current repository using the default indicator set: + +```bash +resqui -t $GITHUB_TOKEN +``` + +Run against a remote repository and write output to a custom file: + +```bash +resqui -u https://github.com/org/repo -t $GITHUB_TOKEN -o report.json +``` + +## Navigation + +| Section | What you'll find | +|---|---| +| **Tutorials** | Step-by-step lessons — start here if you're new | +| **How-to guides** | Recipes for specific tasks (custom configs, adding plugins, CI setup) | +| **Reference** | Complete CLI, configuration, and API documentation | +| **Explanation** | Architecture decisions and indicator background | diff --git a/docs/reference/api.md b/docs/reference/api.md new file mode 100644 index 0000000..3e4c02d --- /dev/null +++ b/docs/reference/api.md @@ -0,0 +1,42 @@ +# API Reference + +## Core + +::: resqui.core + options: + members: + - Context + - CheckResult + - Summary + +## Configuration + +::: resqui.config + options: + members: + - Configuration + +## Plugins + +::: resqui.plugins.base + options: + members: + - IndicatorPlugin + - PluginInitError + +## Executors + +::: resqui.executors.python + options: + members: + - PythonExecutor + +::: resqui.executors.docker + options: + members: + - DockerExecutor + +::: resqui.executors.base + options: + members: + - ExecutorInitError diff --git a/docs/reference/cli.md b/docs/reference/cli.md new file mode 100644 index 0000000..9aef867 --- /dev/null +++ b/docs/reference/cli.md @@ -0,0 +1,40 @@ +# CLI Reference + +## Synopsis + +``` +resqui [options] +resqui indicators +``` + +## Options + +| Flag | Argument | Default | Description | +|---|---|---|---| +| `-u` | `` | current repo | URL of the repository to assess. If omitted, resqui uses the remote URL of the current working directory. | +| `-c` | `` | built-in default | Path to a JSON configuration file. | +| `-o` | `` | `resqui_summary.json` | Path for the JSON-LD output report. | +| `-t` | `` | — | GitHub personal access token. Required by `HowFairIs` and `OpenSSFScorecard`. | +| `-d` | `` | — | DashVerse API token. When provided, the summary is uploaded after assessment. | +| `-b` | `` | HEAD commit | Git branch, tag, or commit hash to assess. | +| `-v` | — | off | Verbose output: prints full evidence text for each indicator. | +| `--version` | — | — | Print the installed version and exit. | +| `--help` | — | — | Print usage and exit. | + +## Subcommands + +### `indicators` + +```bash +resqui indicators +``` + +Prints all available plugin classes, their versions, and the indicator names +they expose. Useful for discovering what can go into a configuration file. + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Assessment completed (individual indicator failures do not affect the exit code) | +| `1` | Fatal error (not a Git repository, clone failed, etc.) | diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md new file mode 100644 index 0000000..2b2a6d0 --- /dev/null +++ b/docs/reference/configuration.md @@ -0,0 +1,79 @@ +# Configuration Format + +Configuration files are JSON documents that select which indicators to run +and which plugins to use for each. + +## Schema + +```json +{ + "indicators": [ + { + "name": "", + "plugin": "", + "@id": "" + } + ] +} +``` + +### `name` + +The method name on the plugin class that implements the check. +Must match exactly (case-sensitive). + +### `plugin` + +The class name of the plugin. Use `resqui indicators` to list all available +class names. + +### `@id` + +The W3ID URI that identifies this indicator in the EVERSE vocabulary. +Use the string `"missing"` if no URI has been assigned yet. + +## Default configuration + +When no `-c` flag is provided, resqui uses this built-in configuration: + +```json +{ + "indicators": [ + { + "name": "has_license", + "plugin": "HowFairIs", + "@id": "https://w3id.org/everse/i/indicators/license" + }, + { + "name": "has_citation", + "plugin": "CFFConvert", + "@id": "https://w3id.org/everse/i/indicators/citation" + }, + { + "name": "has_ci_tests", + "plugin": "OpenSSFScorecard", + "@id": "missing" + }, + { + "name": "human_code_review_requirement", + "plugin": "OpenSSFScorecard", + "@id": "missing" + }, + { + "name": "has_published_package", + "plugin": "OpenSSFScorecard", + "@id": "missing" + }, + { + "name": "has_no_security_leak", + "plugin": "Gitleaks", + "@id": "missing" + } + ] +} +``` + +## Example configurations + +Ready-made configurations are included in the `configurations/` directory of +the repository and can be passed directly with `-c`. diff --git a/docs/tutorials/first-assessment.md b/docs/tutorials/first-assessment.md new file mode 100644 index 0000000..0422313 --- /dev/null +++ b/docs/tutorials/first-assessment.md @@ -0,0 +1,81 @@ +# Your First Assessment + +This tutorial walks you through running your first quality assessment from scratch. +By the end you will have a signed JSON-LD report listing the quality indicators +for a real repository. + +## Prerequisites + +- Python 3.9+ installed +- Docker running (needed by most plugins) +- A GitHub personal access token (read-only scope is enough) + +You can create a token at . + +## 1. Install resqui + +Create a virtual environment and install: + +```bash +python -m venv .venv +source .venv/bin/activate +pip install resqui +``` + +## 2. Check your current repository + +Navigate to any Git repository on your machine and run: + +```bash +resqui -t $GITHUB_TOKEN +``` + +resqui detects the remote URL automatically and runs all default indicators. +You will see a live progress line per indicator: + +``` +Loading default configuration. +GitHub API token ✔ +Repository URL: https://github.com/org/myproject.git +Project name: myproject +Author: Your Name +Version: v1.2.0 +Checking indicators ... + has_license/HowFairIs (0.9s): ✔ + has_citation/CFFConvert (0.3s): ✔ + has_ci_tests/OpenSSFScorecard (12.4s): ✔ + ... +Summary has been written to resqui_summary.json +``` + +## 3. Inspect the report + +Open `resqui_summary.json`. Each entry in `checks` corresponds to one indicator: + +```json +{ + "@type": "CheckResult", + "assessesIndicator": { "@id": "https://w3id.org/everse/i/indicators/license" }, + "checkingSoftware": { "name": "HowFairIs", "version": "0.14.2" }, + "evidence": "Found license file: 'LICENSE'.", + "output": "valid", + "status": { "@id": "schema:CompletedActionStatus" } +} +``` + +- **evidence** – the human-readable finding from the plugin +- **output** – `valid`, `missing`, or `failed` +- **status** – linked schema.org action status + +## 4. Check a remote repository + +You can point resqui at any public repository without cloning it yourself: + +```bash +resqui -u https://github.com/org/other-project -t $GITHUB_TOKEN -o other-report.json +``` + +## Next steps + +- [Write a custom configuration](../how-to/custom-configuration.md) to choose which indicators to run +- [Integrate resqui in GitHub Actions](../how-to/ci-integration.md) to run it automatically on every push diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..10d80ab --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,35 @@ +site_name: resqui +site_description: EVERSE Research Software Quality Indicators +site_url: https://everse-researchsoftware.github.io/QualityPipelines +repo_url: https://github.com/EVERSE-ResearchSoftware/QualityPipelines +repo_name: EVERSE-ResearchSoftware/QualityPipelines + +theme: + name: readthedocs + +plugins: + - search + - mkdocstrings: + handlers: + python: + paths: [src] + options: + show_source: true + show_root_heading: true + docstring_style: google + +nav: + - Home: index.md + - Tutorials: + - Your First Assessment: tutorials/first-assessment.md + - How-to Guides: + - Write a Custom Configuration: how-to/custom-configuration.md + - Add an Indicator Plugin: how-to/add-a-plugin.md + - Integrate in GitHub Actions: how-to/ci-integration.md + - Reference: + - CLI: reference/cli.md + - Configuration Format: reference/configuration.md + - API: reference/api.md + - Explanation: + - Architecture: explanation/architecture.md + - Indicators: explanation/indicators.md diff --git a/pyproject.toml b/pyproject.toml index 9fd28c6..4b94843 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dynamic = ["version"] [project.optional-dependencies] dev = ["black"] +docs = ["mkdocs", "mkdocstrings[python]"] [project.scripts] resqui = "resqui.cli:resqui"