diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000..446faec --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,72 @@ +name: CD +on: + push: + tags: + - 'v*' + +env: + PY_VERSION: 3.12 + +jobs: + pypi-build: + name: Build package for PyPI + if: github.repository == 'ACCESS-NRI/access-experiment-runner' # exclude forks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PY_VERSION }} + + - run: | + python3 -m pip install --upgrade build && python3 -m build + + - uses: actions/upload-artifact@v4 + with: + name: release + path: dist + + pypi-publish: + # Split build and publish to restrict trusted publishing to just this workflow + needs: ['pypi-build'] + name: Publish to PyPI.org + runs-on: ubuntu-latest + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + name: release + path: dist + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4 + + conda: + name: Build with conda and upload + if: github.repository == 'ACCESS-NRI/access-experiment-runner' # exclude forks + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Setup conda environment + uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0 + with: + miniconda-version: "latest" + python-version: ${{ env.PY_VERSION }} + environment-file: conda/environment.yml + auto-update-conda: false + auto-activate-base: false + show-channel-urls: true + + - name: Build and upload conda package + uses: ACCESS-NRI/action-build-and-upload-conda-packages@v3.0.0 + with: + meta_yaml_dir: conda + user: ${{ vars.ANACONDA_USER }} + label: main + token: ${{ secrets.ANACONDA_TOKEN }} + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..983e41c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,105 @@ +name: CI +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +env: + PY_VERSION_BUILD: 3.12 + +jobs: + formatting: + name: Code formatting + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: black + uses: psf/black@stable + with: + options: "--check --verbose --diff" + src: "./src/experiment_runner ./tests" + + - name: flake8 + uses: py-actions/flake8@v2 + with: + args: "--extend-ignore=E203 --max-line-length=120" + + pypa-build: + name: PyPA build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PY_VERSION_BUILD }} + cache: 'pip' # caching pip dependencies + + - run: | + python3 -m pip install --upgrade build && python3 -m build + + conda-build: + name: Conda Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup conda environment + uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0 + with: + miniconda-version: "latest" + python-version: ${{ env.PY_VERSION_BUILD }} + environment-file: conda/environment.yml + auto-update-conda: false + auto-activate-base: false + show-channel-urls: true + + - name: Build conda package + uses: ACCESS-NRI/action-build-and-upload-conda-packages@v3.0.0 + with: + meta_yaml_dir: conda + label: main + upload: false + + tests: + name: Tests + runs-on: ubuntu-latest + + # Run the job for different versions of python + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install '.[devel,test,access]' + + - name: Run tests with coverage + run: | + python3 -m pytest -s \ + --cov=experiment_runner \ + --cov-branch \ + --cov-report=term-missing \ + --cov-report=xml:coverage.xml \ + tests + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 811385d..793ccc2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,4 +4,8 @@ repos: hooks: - id: black language_version: python3 - + - repo: https://github.com/pycqa/flake8 + rev: 7.3.0 + hooks: + - id: flake8 + additional_dependencies: [Flake8-pyproject] diff --git a/conda/environment.yml b/conda/environment.yml new file mode 100644 index 0000000..85daf10 --- /dev/null +++ b/conda/environment.yml @@ -0,0 +1,10 @@ +channels: + - conda-forge + - accessnri + - default + +dependencies: + - anaconda-client + - conda-build + - conda-verify + - setuptools_scm diff --git a/conda/meta.yaml b/conda/meta.yaml new file mode 100644 index 0000000..debdc52 --- /dev/null +++ b/conda/meta.yaml @@ -0,0 +1,42 @@ +{% set data = load_setup_py_data(setup_file='../setup.py', from_recipe_dir=True) %} +{% set version = data.get('version') %} +{% set pyproj = load_file_data('../pyproject.toml', from_recipe_dir=True) %} +{% set project = pyproj.get('project') %} + +package: + name: experiment-runner + version: "{{ version }}" + +build: + noarch: python + number: 0 + script: "{{ PYTHON }} -m pip install . -vv" + entry_points: + {% for name, script in project.get('scripts', {}).items() %} + - {{ name }} = {{ script }} + {% endfor %} + +source: + path: ../ + +requirements: + host: + - python + - pip + - setuptools >=61.0.0 + - setuptools_scm + run: + - python >=3.10 + {% for dep in project.get('dependencies', []) %} + - {{ dep }} + {% endfor %} + +test: + imports: + - experiment_runner + +about: + home: https://github.com/ACCESS-NRI/access-experiment-runner/ + license: Apache Software + license_family: APACHE + summary: "A tool to orchestrate branch-based workflows and automate job submission for ACCESS experiments." diff --git a/pyproject.toml b/pyproject.toml index cdaf93e..e930b7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,12 @@ [project] name = "experiment-runner" -version = "0.1.0" -description = "A tool to run parameter sensitivity experiments." +dynamic = ["version"] +description = "A tool to orchestrate branch-based workflows and automate job submission for ACCESS experiments." authors = [ { name = "Minghang Li", email = "Minghang.Li1@anu.edu.au" } ] readme = "README.md" -keywords = ["experiment runner", "workflow", "payu"] +keywords = ["experiment runner", "workflow", "access", "payu"] license = { text = "Apache-2.0" } classifiers = [ "License :: OSI Approved :: Apache Software License", @@ -14,12 +14,52 @@ classifiers = [ "Programming Language :: Python :: 3", "Topic :: Utilities", ] -# As payu is pre-installed (via modules), this can be omitted. -dependencies = [] -[project.scripts] -experiment-runner = "experiment_runner.main:main" +dependencies = [ + "ruamel.yaml", + "f90nml" +] [build-system] -requires = ["setuptools>=61.0", "wheel"] +requires = ["setuptools>=80", "setuptools_scm[toml]>=8", "wheel"] build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] + +[project.scripts] +experiment-runner = "experiment_runner.main:main" + +[project.urls] +Homepage = "https://github.com/ACCESS-NRI/access-experiment-runner" + +[project.optional-dependencies] +devel = [ + "flake8", + "black", + "pre-commit", +] +test = [ + "pytest", + "pytest-cov", +] + +access = ["payu"] + +[tool.pytest.ini_options] +addopts = [ + "--cov=experiment_runner", + "--cov-report=term", + "--cov-report=html", + "--cov-report=xml" +] +testpaths = ["tests"] + +[tool.coverage.run] + +[tool.black] +line-length = 120 + +[tool.flake8] +max-line-length = 120 +extend-ignore = ["E203"] diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..709f663 --- /dev/null +++ b/setup.py @@ -0,0 +1,5 @@ +# This file is only needed for the automatic versioning in the conda recipe +from setuptools import setup +import setuptools_scm + +setup(version=setuptools_scm.get_version()) diff --git a/src/experiment_runner/experiment_runner.py b/src/experiment_runner/experiment_runner.py index fe7da6c..1cd3c66 100644 --- a/src/experiment_runner/experiment_runner.py +++ b/src/experiment_runner/experiment_runner.py @@ -2,7 +2,6 @@ from payu.branch import clone, list_branches from .base_experiment import BaseExperiment from .pbs_job_manager import PBSJobManager -import subprocess import git @@ -36,18 +35,13 @@ def _create_cloned_directory(self) -> None: if not self.running_branches: raise ValueError("No running branches provided!") - all_cloned_directories = [ - Path(self.test_path) / b / self.repository_directory - for b in self.running_branches - ] + all_cloned_directories = [Path(self.test_path) / b / self.repository_directory for b in self.running_branches] for clone_dir, branch in zip(all_cloned_directories, self.running_branches): if clone_dir.exists(): print(f"-- Test dir: {clone_dir} already exists, skipping cloning.") if not self._update_existing_repo(clone_dir, branch): - print( - f"Failed to update existing repo {clone_dir}, leaving as it is." - ) + print(f"Failed to update existing repo {clone_dir}, leaving as it is.") else: print(f"-- Cloning branch '{branch}' into {clone_dir}...") self._do_clone(clone_dir, branch) @@ -92,7 +86,7 @@ def _update_existing_repo(self, clone_dir: Path, target_ref: str) -> bool: # try pulling with rebase try: repo.git.pull("--rebase", "--autostash", "origin", target_ref) - except git.exc.GitCommandError as e: + except git.exc.GitCommandError: repo.git.reset("--keep", f"origin/{target_ref}") # save new HEAD after update @@ -106,9 +100,7 @@ def _update_existing_repo(self, clone_dir: Path, target_ref: str) -> bool: print( f"-- Repo {rel_path} updated from {current_commit[:7]} to {new_commit[:7]} on branch {target_ref}." ) - changed = repo.git.diff( - "--name-only", current_commit, new_commit - ).splitlines() + changed = repo.git.diff("--name-only", current_commit, new_commit).splitlines() if changed: print("-- Changed files:") for file in changed: diff --git a/src/experiment_runner/pbs_job_manager.py b/src/experiment_runner/pbs_job_manager.py index 8898576..b058870 100644 --- a/src/experiment_runner/pbs_job_manager.py +++ b/src/experiment_runner/pbs_job_manager.py @@ -37,7 +37,7 @@ def output_existing_pbs_jobs() -> dict: pbs_job_file = f.read() def _flush_pair(): - nonlocal current_key, current_value, job_id + nonlocal current_key, current_value if current_key and job_id: pbs_jobs[job_id][current_key] = current_value.strip() current_key = None @@ -136,9 +136,7 @@ def _check_duplicated_jobs(self, path: Path, pbs_jobs: dict) -> bool: duplicated = False for _, job_info in pbs_jobs.items(): - folder_path, parent_path = _extract_current_and_parent_path( - job_info["Error_Path"] - ) + folder_path, parent_path = _extract_current_and_parent_path(job_info["Error_Path"]) job_state = job_info["job_state"] if job_state not in ("F", "S"): if parent_path not in parent_paths: @@ -178,10 +176,7 @@ def _start_experiment_runs(self, path: Path, nruns: int, duplicated: bool) -> No subprocess.run(command, shell=True, check=True) print("\n") else: - print( - f"-- `{Path(*path.parts[-2:])}` already completed " - f"{doneruns} runs, hence no new runs.\n" - ) + print(f"-- `{Path(*path.parts[-2:])}` already completed " f"{doneruns} runs, hence no new runs.\n") def _clean_workspace(self, path: Path) -> None: """ diff --git a/tests/conftest.py b/tests/conftest.py index 74d8608..4c5c248 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -148,9 +148,7 @@ def patch_runner(monkeypatch, tmp_path, payu_calls, pbs_job_recorder): """ monkeypatch.setattr(exp_runner, "clone", _dummy_clone, raising=True) monkeypatch.setattr(exp_runner, "list_branches", _dummy_list_branches, raising=True) - monkeypatch.setattr( - exp_runner, "PBSJobManager", lambda: pbs_job_recorder, raising=True - ) + monkeypatch.setattr(exp_runner, "PBSJobManager", lambda: pbs_job_recorder, raising=True) class _Exc: class GitCommandError(Exception): diff --git a/tests/test_experiment_runner.py b/tests/test_experiment_runner.py index 91f998d..e0f20f5 100644 --- a/tests/test_experiment_runner.py +++ b/tests/test_experiment_runner.py @@ -16,9 +16,7 @@ def test_error_when_no_running_branches(indata, monkeypatch, patch_runner): er.run() -def test_update_existing_repo_creates_branch_if_missing( - tmp_path, indata, monkeypatch, patch_runner -): +def test_update_existing_repo_creates_branch_if_missing(tmp_path, indata, monkeypatch, patch_runner): for branch in indata["running_branches"]: dir_path = tmp_path / "tests" / branch / indata["repository_directory"] dir_path.mkdir(parents=True, exist_ok=True) @@ -42,9 +40,7 @@ def make_repo(path): assert len(patch_runner.pbs.calls) == 2 -def test_update_existing_repo_already_up_to_date( - tmp_path, indata, monkeypatch, patch_runner, capsys -): +def test_update_existing_repo_already_up_to_date(tmp_path, indata, monkeypatch, patch_runner, capsys): for branch in indata["running_branches"]: dir_path = tmp_path / "tests" / branch / indata["repository_directory"] dir_path.mkdir(parents=True, exist_ok=True) @@ -112,16 +108,8 @@ def test_run_clones_and_runs_jobs(indata, monkeypatch, patch_runner): assert len(patch_runner.payu.clone_calls) == len(indata["running_branches"]) - expt1 = ( - Path(indata["test_path"]) - / indata["running_branches"][0] - / indata["repository_directory"] - ) - expt2 = ( - Path(indata["test_path"]) - / indata["running_branches"][1] - / indata["repository_directory"] - ) + expt1 = Path(indata["test_path"]) / indata["running_branches"][0] / indata["repository_directory"] + expt2 = Path(indata["test_path"]) / indata["running_branches"][1] / indata["repository_directory"] assert patch_runner.pbs.calls == [(expt1, 1), (expt2, 2)] @@ -151,9 +139,7 @@ def make_repo(path): assert len(patch_runner.pbs.calls) == 2 -def test_run_existing_dirs_pull_failure_uses_reset( - tmp_path, indata, monkeypatch, patch_runner -): +def test_run_existing_dirs_pull_failure_uses_reset(tmp_path, indata, monkeypatch, patch_runner): for branch in indata["running_branches"]: dir_path = tmp_path / "tests" / branch / indata["repository_directory"] diff --git a/tests/test_main.py b/tests/test_main.py index ad28397..14795c4 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -74,9 +74,7 @@ def run(self): assert called["indata"]["keep_uuid"] is True -def test_main_errors_when_no_yaml_provided_and_default_missing( - tmp_path, monkeypatch, capsys -): +def test_main_errors_when_no_yaml_provided_and_default_missing(tmp_path, monkeypatch, capsys): monkeypatch.chdir(tmp_path) monkeypatch.setattr(sys, "argv", ["prog"]) diff --git a/tests/test_pbs_job_manager.py b/tests/test_pbs_job_manager.py index b2e2094..eada514 100644 --- a/tests/test_pbs_job_manager.py +++ b/tests/test_pbs_job_manager.py @@ -1,5 +1,3 @@ -from pathlib import Path -import pytest from experiment_runner.pbs_job_manager import output_existing_pbs_jobs from experiment_runner.pbs_job_manager import ( _extract_current_and_parent_path, @@ -33,15 +31,9 @@ def dummy_run(*args, **kwargs): jobs = output_existing_pbs_jobs() assert "123.gadi" in jobs and "999.gadi" in jobs - assert ( - jobs["123.gadi"]["Error_Path"] - == "gadi.nci.org.au:/g/data/group/parentA/expA/job.e123" - ) + assert jobs["123.gadi"]["Error_Path"] == "gadi.nci.org.au:/g/data/group/parentA/expA/job.e123" assert jobs["123.gadi"]["job_state"] == "R" - assert ( - jobs["999.gadi"]["Error_Path"] - == "gadi.nci.org.au:/g/data/group/parentB/expB/job.e999" - ) + assert jobs["999.gadi"]["Error_Path"] == "gadi.nci.org.au:/g/data/group/parentB/expB/job.e999" assert jobs["999.gadi"]["job_state"] == "Q" assert not (tmp_path / "current_job_status").exists() @@ -80,12 +72,8 @@ def dummy_check(self, path, jobs): def dummy_start(self, path, nruns, duplicated): called["args"] = (path, nruns, duplicated) - monkeypatch.setattr( - PBSJobManager, "_check_duplicated_jobs", dummy_check, raising=True - ) - monkeypatch.setattr( - PBSJobManager, "_start_experiment_runs", dummy_start, raising=True - ) + monkeypatch.setattr(PBSJobManager, "_check_duplicated_jobs", dummy_check, raising=True) + monkeypatch.setattr(PBSJobManager, "_start_experiment_runs", dummy_start, raising=True) pbs_job_manager = PBSJobManager() pbs_job_manager.pbs_job_runs(current_path, nruns=3) @@ -118,12 +106,8 @@ def dummy_check(self, path, jobs): def dummy_start(self, path, nruns, duplicated): called_start["args"] = (path, nruns, duplicated) - monkeypatch.setattr( - PBSJobManager, "_check_duplicated_jobs", dummy_check, raising=True - ) - monkeypatch.setattr( - PBSJobManager, "_start_experiment_runs", dummy_start, raising=True - ) + monkeypatch.setattr(PBSJobManager, "_check_duplicated_jobs", dummy_check, raising=True) + monkeypatch.setattr(PBSJobManager, "_start_experiment_runs", dummy_start, raising=True) pbs_job_manager = PBSJobManager() pbs_job_manager.pbs_job_runs(current_path, nruns=2) @@ -131,9 +115,7 @@ def dummy_start(self, path, nruns, duplicated): assert called_start["args"] == (current_path, 2, True) -def test_start_experiment_runs_return_early_if_duplicated( - tmp_path, monkeypatch, capsys -): +def test_start_experiment_runs_return_early_if_duplicated(tmp_path, monkeypatch, capsys): pbs_job_manager = PBSJobManager() current_path = tmp_path / "parentA" / "expA" current_path.mkdir(parents=True, exist_ok=True)