Cherry-pick python package related files

.github/workflows/cd.yml

@@ -0,0 +1,72 @@
+name: CD
+on:
+  push:
+    tags:
+      - 'v*'
+
+env:
+  PY_VERSION: 3.12
+
+jobs:
+  pypi-build:
+    name: Build package for PyPI
+    if: github.repository == 'ACCESS-NRI/access-experiment-runner'  # exclude forks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PY_VERSION }}
+
+      - run: |
+          python3 -m pip install --upgrade build && python3 -m build
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: release
+          path: dist
+
+  pypi-publish:
+    # Split build and publish to restrict trusted publishing to just this workflow
+    needs: ['pypi-build']
+    name: Publish to PyPI.org
+    runs-on: ubuntu-latest
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: release
+          path: dist
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4
+
+  conda:
+    name: Build with conda and upload
+    if: github.repository == 'ACCESS-NRI/access-experiment-runner'  # exclude forks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+
+      - name: Setup conda environment
+        uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0
+        with:
+          miniconda-version: "latest"
+          python-version: ${{ env.PY_VERSION }}
+          environment-file: conda/environment.yml
+          auto-update-conda: false
+          auto-activate-base: false
+          show-channel-urls: true
+
+      - name: Build and upload conda package
+        uses: ACCESS-NRI/action-build-and-upload-conda-packages@v3.0.0
+        with:
+          meta_yaml_dir: conda
+          user: ${{ vars.ANACONDA_USER }}
+          label: main
+          token: ${{ secrets.ANACONDA_TOKEN }}
+

.github/workflows/ci.yml

@@ -0,0 +1,105 @@
+name: CI
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+env:
+  PY_VERSION_BUILD: 3.12
+
+jobs:
+  formatting:
+    name: Code formatting
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: black
+        uses: psf/black@stable
+        with:
+          options: "--check --verbose --diff"
+          src: "./src/experiment_runner ./tests"
+
+      - name: flake8
+        uses: py-actions/flake8@v2
+        with:
+          args: "--extend-ignore=E203 --max-line-length=120"
+
+  pypa-build:
+    name: PyPA build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PY_VERSION_BUILD }}
+          cache: 'pip' # caching pip dependencies
+
+      - run: |
+          python3 -m pip install --upgrade build && python3 -m build
+
+  conda-build:
+    name: Conda Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup conda environment
+        uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0
+        with:
+          miniconda-version: "latest"
+          python-version: ${{ env.PY_VERSION_BUILD }}
+          environment-file: conda/environment.yml
+          auto-update-conda: false
+          auto-activate-base: false
+          show-channel-urls: true
+
+      - name: Build conda package
+        uses: ACCESS-NRI/action-build-and-upload-conda-packages@v3.0.0
+        with:
+          meta_yaml_dir: conda
+          label: main
+          upload: false
+
+  tests:
+    name: Tests
+    runs-on: ubuntu-latest
+
+    # Run the job for different versions of python
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install '.[devel,test,access]'
+
+      - name: Run tests with coverage
+        run: |
+          python3 -m pytest -s \
+            --cov=experiment_runner \
+            --cov-branch \
+            --cov-report=term-missing \
+            --cov-report=xml:coverage.xml \
+            tests
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+         token: ${{ secrets.CODECOV_TOKEN }}
+         files: ./coverage.xml

.pre-commit-config.yaml

@@ -4,4 +4,8 @@ repos:
     hooks:
       - id: black
         language_version: python3
-
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.3.0
+    hooks:
+    -   id: flake8
+        additional_dependencies: [Flake8-pyproject]

conda/environment.yml

@@ -0,0 +1,10 @@
+channels:
+  - conda-forge
+  - accessnri
+  - default
+
+dependencies:
+  - anaconda-client
+  - conda-build
+  - conda-verify
+  - setuptools_scm

conda/meta.yaml

@@ -0,0 +1,42 @@
+{% set data = load_setup_py_data(setup_file='../setup.py', from_recipe_dir=True) %}
+{% set version = data.get('version') %}
+{% set pyproj = load_file_data('../pyproject.toml', from_recipe_dir=True) %}
+{% set project = pyproj.get('project') %}
+
+package:
+    name: experiment-runner
+    version: "{{ version }}"
+
+build:
+    noarch: python
+    number: 0
+    script: "{{ PYTHON }}  -m pip install . -vv"
+    entry_points:
+        {% for name, script in project.get('scripts', {}).items() %}
+        - {{ name }} = {{ script }}
+        {% endfor %}
+
+source:
+    path: ../
+
+requirements:
+    host:
+        - python
+        - pip
+        - setuptools >=61.0.0
+        - setuptools_scm
+    run:
+        - python >=3.10
+        {% for dep in project.get('dependencies', []) %}
+        - {{ dep }}
+        {% endfor %}
+
+test:
+    imports:
+        - experiment_runner
+
+about:
+    home: https://github.com/ACCESS-NRI/access-experiment-runner/
+    license: Apache Software
+    license_family: APACHE
+    summary: "A tool to orchestrate branch-based workflows and automate job submission for ACCESS experiments."

pyproject.toml

@@ -1,25 +1,65 @@
 [project]
 name = "experiment-runner"
-version = "0.1.0"
-description = "A tool to run parameter sensitivity experiments."
+dynamic = ["version"]
+description = "A tool to orchestrate branch-based workflows and automate job submission for ACCESS experiments."
 authors = [
   { name = "Minghang Li", email = "Minghang.Li1@anu.edu.au" }
 ]
 readme = "README.md"
-keywords = ["experiment runner", "workflow", "payu"]
+keywords = ["experiment runner", "workflow", "access", "payu"]
 license = { text = "Apache-2.0" }
 classifiers = [
     "License :: OSI Approved :: Apache Software License",
     "Operating System :: POSIX :: Linux",
     "Programming Language :: Python :: 3",
     "Topic :: Utilities",
 ]
-# As payu is pre-installed (via modules), this can be omitted.
-dependencies = []
 
-[project.scripts]
-experiment-runner = "experiment_runner.main:main"
+dependencies = [
+  "ruamel.yaml",
+  "f90nml"
+]
 
 [build-system]
-requires = ["setuptools>=61.0", "wheel"]
+requires = ["setuptools>=80", "setuptools_scm[toml]>=8", "wheel"]
 build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[project.scripts]
+experiment-runner = "experiment_runner.main:main"
+
+[project.urls]
+Homepage = "https://github.com/ACCESS-NRI/access-experiment-runner"
+
+[project.optional-dependencies]
+devel = [
+    "flake8",
+    "black",
+    "pre-commit",
+]
+test = [
+    "pytest",
+    "pytest-cov",
+]
+
+access = ["payu"]
+
+[tool.pytest.ini_options]
+addopts = [
+  "--cov=experiment_runner",
+  "--cov-report=term",
+  "--cov-report=html",
+  "--cov-report=xml"
+]
+testpaths = ["tests"]
+
+[tool.coverage.run]
+
+[tool.black]
+line-length = 120
+
+[tool.flake8]
+max-line-length = 120
+extend-ignore = ["E203"]

setup.py

@@ -0,0 +1,5 @@
+# This file is only needed for the automatic versioning in the conda recipe
+from setuptools import setup
+import setuptools_scm
+
+setup(version=setuptools_scm.get_version())

src/experiment_runner/experiment_runner.py

@@ -2,7 +2,6 @@
 from payu.branch import clone, list_branches
 from .base_experiment import BaseExperiment
 from .pbs_job_manager import PBSJobManager
-import subprocess
 import git
 
 
@@ -36,18 +35,13 @@ def _create_cloned_directory(self) -> None:
         if not self.running_branches:
             raise ValueError("No running branches provided!")
 
-        all_cloned_directories = [
-            Path(self.test_path) / b / self.repository_directory
-            for b in self.running_branches
-        ]
+        all_cloned_directories = [Path(self.test_path) / b / self.repository_directory for b in self.running_branches]
 
         for clone_dir, branch in zip(all_cloned_directories, self.running_branches):
             if clone_dir.exists():
                 print(f"-- Test dir: {clone_dir} already exists, skipping cloning.")
                 if not self._update_existing_repo(clone_dir, branch):
-                    print(
-                        f"Failed to update existing repo {clone_dir}, leaving as it is."
-                    )
+                    print(f"Failed to update existing repo {clone_dir}, leaving as it is.")
             else:
                 print(f"-- Cloning branch '{branch}' into {clone_dir}...")
                 self._do_clone(clone_dir, branch)
@@ -92,7 +86,7 @@ def _update_existing_repo(self, clone_dir: Path, target_ref: str) -> bool:
             # try pulling with rebase
             try:
                 repo.git.pull("--rebase", "--autostash", "origin", target_ref)
-            except git.exc.GitCommandError as e:
+            except git.exc.GitCommandError:
                 repo.git.reset("--keep", f"origin/{target_ref}")
 
             # save new HEAD after update
@@ -106,9 +100,7 @@ def _update_existing_repo(self, clone_dir: Path, target_ref: str) -> bool:
                 print(
                     f"-- Repo {rel_path} updated from {current_commit[:7]} to {new_commit[:7]} on branch {target_ref}."
                 )
-                changed = repo.git.diff(
-                    "--name-only", current_commit, new_commit
-                ).splitlines()
+                changed = repo.git.diff("--name-only", current_commit, new_commit).splitlines()
                 if changed:
                     print("-- Changed files:")
                     for file in changed: