chore: backport CI improvements to v2.3

.github/actions/default/action.yml

@@ -9,19 +9,24 @@ runs:
   steps:
     - name: Install Nix
       uses: cachix/install-nix-action@v31
-    - name: Cache dependencies
-      uses: nix-community/cache-nix-action@v6
-      with:
-        primary-key: nix-${{ runner.os }}-${{ hashFiles('**/flake.nix', '**/flake.lock') }}
-        restore-prefixes-first-match: nix-${{ runner.os }}-
-    - name: Load dependencies
+    ## Disabling cache for now, as it's slowing down the build. Downloading nix
+    ## dependencies from cache.nixos.org is faster than restoring from the cache.
+    #- name: Cache dependencies
+    #  uses: nix-community/cache-nix-action@v6
+    #  with:
+    #    primary-key: nix-${{ runner.os }}-${{ hashFiles('**/flake.nix', '**/flake.lock') }}
+    #    restore-prefixes-first-match: nix-${{ runner.os }}-
+    - name: go env
+      id: go-env
       shell: bash
-      run: nix develop --install
+      run: |
+        nix develop --command bash -c "go env | sed -E \"s/^([^=]+)='(.*)'\$/\1=\2/\"" >> "$GITHUB_OUTPUT"
     - uses: actions/cache@v4
       with:
         path: |
-          ~/.cache/go-build
-          /tmp/go/pkg/mod/
+          ${{ steps.go-env.outputs.GOCACHE }}
+          ${{ steps.go-env.outputs.GOMODCACHE }}
+          ~/.cache/golangci-lint
         key: ${{ runner.os }}-${{ github.job }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
-          ${{ runner.os }}-${{ github.job }}-go-
+          ${{ runner.os }}-${{ github.job }}-go-

.github/workflows/benchmark-comparison.yml

@@ -3,23 +3,23 @@ on:
   workflow_dispatch:
     inputs:
       bench:
-        description: 'Benchmarks to run'
+        description: "Benchmarks to run"
         required: false
-        default: '.'
+        default: "."
       parallelism:
-        description: 'Number of parallel benchmarks to run'
+        description: "Number of parallel benchmarks to run"
         required: false
-        default: 5
+        default: "5"
       duration:
-        description: 'Duration of each benchmark'
+        description: "Duration of each benchmark"
         required: false
-        default: '10s'
+        default: "10s"
       count:
-        description: 'Number of times to run each benchmark '
+        description: "Number of times to run each benchmark "
         required: false
-        default: 1
+        default: "1"
   pull_request:
-    types: [ assigned, opened, synchronize, reopened, labeled ]
+    types: [assigned, opened, synchronize, reopened, labeled]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
@@ -28,11 +28,12 @@ concurrency:
 jobs:
   BenchmarkCompare:
     runs-on: "github-001"
-    if: contains(github.event.pull_request.labels.*.name, 'benchmarks')
+    if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'benchmarks')
     steps:
-      - uses: 'actions/checkout@v4'
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          filter: tree:0 # treeless clone, faster to clone as history and blobs are only fetched when needed.
       - name: Setup Env
         uses: ./.github/actions/env
         with:
@@ -41,15 +42,15 @@ jobs:
           /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes"
           develop --impure --command just
           --justfile ./test/performance/justfile
-          --working-directory ./test/performance 
+          --working-directory ./test/performance
           writes compare ${{ inputs.bench }} ${{ inputs.parallelism }} ${{ inputs.duration }} ${{ inputs.count }}
       - run: >
           /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes"
-          develop --impure --command just 
-          --justfile ./test/performance/justfile 
-          --working-directory ./test/performance 
+          develop --impure --command just
+          --justfile ./test/performance/justfile
+          --working-directory ./test/performance
           writes graphs
       - uses: actions/upload-artifact@v4
         with:
           name: graphs
-          path: test/performance/report
+          path: test/performance/report

.github/workflows/benchmark.yml

@@ -3,17 +3,17 @@ on:
   workflow_dispatch:
     inputs:
       bench:
-        description: 'Benchmarks to run'
+        description: "Benchmarks to run"
         required: false
-        default: '.'
+        default: "."
       parallelism:
-        description: 'Number of parallel benchmarks to run'
+        description: "Number of parallel benchmarks to run"
         required: false
-        default: 5
+        default: "5"
       duration:
-        description: 'Duration of each benchmark'
+        description: "Duration of each benchmark"
         required: false
-        default: '10s'
+        default: "10s"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -23,9 +23,10 @@ jobs:
   Benchmark:
     runs-on: "github-001"
     steps:
-      - uses: 'actions/checkout@v4'
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          filter: tree:0 # treeless clone, faster to clone as history and blobs are only fetched when needed.
       - name: Setup Env
         uses: ./.github/actions/env
         with:
@@ -34,15 +35,15 @@ jobs:
           /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes"
           develop --impure --command just
           --justfile ./test/performance/justfile
-          --working-directory ./test/performance 
+          --working-directory ./test/performance
           writes run ${{ inputs.bench }} ${{ inputs.parallelism }} ${{ inputs.duration }} 1
       - run: >
           /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes"
-          develop --impure --command just 
-          --justfile ./test/performance/justfile 
-          --working-directory ./test/performance 
+          develop --impure --command just
+          --justfile ./test/performance/justfile
+          --working-directory ./test/performance
           writes graphs
       - uses: actions/upload-artifact@v4
         with:
           name: graphs
-          path: test/performance/report
+          path: test/performance/report

.github/workflows/main.yml

@@ -30,9 +30,10 @@ jobs:
       GOPATH: /tmp/go
       GOLANGCI_LINT_CACHE: /tmp/golangci-lint
     steps:
-      - uses: 'actions/checkout@v4'
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          filter: tree:0 # treeless clone, faster to clone as history and blobs are only fetched when needed.
       - name: Setup Env
         uses: ./.github/actions/default
         with:
@@ -45,7 +46,7 @@ jobs:
         id: changed-files
         shell: bash
         run: |
-          hasChanged=$(git status --porcelain) 
+          hasChanged=$(git status --porcelain)
           if (( $(echo ${#hasChanged}) != 0 )); then
             git status
             echo "There are changes in the repository"
@@ -58,9 +59,10 @@ jobs:
     env:
       GOPATH: /tmp/go
     steps:
-      - uses: 'actions/checkout@v4'
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          filter: tree:0 # treeless clone, faster to clone as history and blobs are only fetched when needed.
       - name: Setup Env
         uses: ./.github/actions/default
         with:
@@ -76,6 +78,9 @@ jobs:
 
   GoReleaser:
     runs-on: "shipfox-4vcpu-ubuntu-2404"
+    permissions:
+      id-token: write
+      attestations: write
     if: contains(github.event.pull_request.labels.*.name, 'build-images') || github.ref == 'refs/heads/main' || github.event_name == 'merge_group'
     steps:
       - name: Set up QEMU
@@ -86,9 +91,10 @@ jobs:
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           version: "latest"
-      - uses: 'actions/checkout@v4'
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          filter: tree:0 # treeless clone, faster to clone as history and blobs are only fetched when needed.
           ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       - name: Setup Env
         uses: ./.github/actions/default
@@ -112,37 +118,78 @@ jobs:
           FURY_TOKEN: ${{ secrets.FURY_TOKEN }}
           GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
 
+      - uses: actions/upload-artifact@v4
+        with:
+          name: goreleaser-metadata
+          path: |
+            dist/*.json
+            dist/ledger_checksums.txt
+          retention-days: 7
+          compression-level: 0
+
+      # Generate attestations for the goreleaser output archives
+      - uses: actions/attest-build-provenance@v2
+        with:
+          subject-checksums: ./dist/ledger_checksums.txt
+      # Generate attestations for the goreleaser output binaries
+      - uses: actions/attest-build-provenance@v2
+        with:
+          subject-path: ./dist/*/**
+      # Extract image metadata from the artifacts.json file
+      - run: |
+          jq -r '[ .[]|select(.type=="Docker Manifest") | .extra.Digest         ] | to_entries | .[] | ( "digest"+ (.key | tostring) + "=" + .value )' < dist/artifacts.json >> "$GITHUB_OUTPUT"
+          jq -r '[ .[]|select(.type=="Docker Manifest") | .name | split(":")[0] ] | to_entries | .[] | ( "name"+ (.key | tostring) + "=" + .value )'   < dist/artifacts.json >> "$GITHUB_OUTPUT"
+        id: image_metadata
+      # Generate attestations for docker images
+      - uses: actions/attest-build-provenance@v2
+        with:
+          subject-digest: ${{ steps.image_metadata.outputs.digest0 }}
+          subject-name: ${{ steps.image_metadata.outputs.name0 }}
+          push-to-registry: true
+      - uses: actions/attest-build-provenance@v2
+        with:
+          subject-digest: ${{ steps.image_metadata.outputs.digest1 }}
+          subject-name: ${{ steps.image_metadata.outputs.name1 }}
+          push-to-registry: true
 
   Deploy:
-    runs-on: "shipfox-2vcpu-ubuntu-2404"
+    runs-on: ubuntu-24.04
     if: github.ref == 'refs/heads/main'
     environment: staging
+    permissions:
+      id-token: write
     needs:
       - GoReleaser
       - Tests
     steps:
       - name: Tailscale
-        uses: tailscale/github-action@v3
+        uses: tailscale/github-action@v4
         with:
-          oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
-          oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
-          tags: tag:ci
+          oauth-client-id: ${{ secrets.TS_OIDC_OAUTH_CLIENT_ID }}
+          audience: ${{ secrets.TS_OIDC_AUDIENCE }}
+          tags: ${{ vars.TS_TAGS }}
+          version: ${{ vars.TS_VERSION }}
+          args: ${{ vars.TS_ARGS }}
+          retry: ${{ vars.TS_RETRY }}
+          timeout: ${{ vars.TS_TIMEOUT }}
+          ping: ${{ vars.TS_PING }}
       - uses: earthly/actions-setup@v1
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           version: "latest"
-      - uses: 'actions/checkout@v4'
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          filter: tree:0 # treeless clone, faster to clone as history and blobs are only fetched when needed.
       - name: "Deploy in staging"
         env:
           TAG: ${{ github.sha }}
           COMPONENT: ledger
           ARGOCD_REGION_AUTH_TOKEN: ${{ secrets.ARGOCD_REGION_AUTH_TOKEN }}
         run: >
-          earthly  
-          --no-output 
+          earthly
+          --no-output
           --secret AUTH_TOKEN=$ARGOCD_REGION_AUTH_TOKEN
           +deploy-staging
           --TAG=$TAG
-          --COMPONENT=$COMPONENT
+          --COMPONENT=$COMPONENT

.github/workflows/releases.yml

@@ -10,9 +10,10 @@ jobs:
   GoReleaser:
     runs-on: "shipfox-4vcpu-ubuntu-2404"
     steps:
-      - uses: 'actions/checkout@v4'
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          filter: tree:0 # treeless clone, faster to clone as history and blobs are only fetched when needed.
       - name: Setup Env
         uses: ./.github/actions/default
         with:
@@ -33,4 +34,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.NUMARY_GITHUB_TOKEN }}
           SPEAKEASY_API_KEY: ${{ secrets.SPEAKEASY_API_KEY }}
           FURY_TOKEN: ${{ secrets.FURY_TOKEN }}
-          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
+          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}

Justfile

@@ -3,11 +3,12 @@ set dotenv-load
 default:
   @just --list
 
-pre-commit: tidy generate lint export-docs-events openapi generate-client
+pre-commit: tidy generate generate-client lint export-docs-events openapi
 pc: pre-commit
 
 lint:
-    golangci-lint run --fix --build-tags it --timeout 5m
+    golangci-lint --version
+    golangci-lint run --fix --build-tags it,local --timeout 5m
     for d in $(ls tools); do \
         pushd tools/$d; \
         golangci-lint run --fix --build-tags it --timeout 5m; \
@@ -41,12 +42,15 @@ tests:
     cat coverage.txt | grep -v debug.go | grep -v "/machine/" | grep -v "pb.go" > coverage2.txt
     mv coverage2.txt coverage.txt
 
+fmt:
+    @golangci-lint fmt
+
 openapi:
     yq eval-all '. as $item ireduce ({}; . * $item)' openapi/v1.yaml openapi/v2.yaml openapi/overlay.yaml > openapi.yaml
     npx -y widdershins {{justfile_directory()}}/openapi/v2.yaml -o {{justfile_directory()}}/docs/api/README.md --search false --language_tabs 'http:HTTP' --summary --omitHeader
 
 generate-client: openapi
-    @cd pkg/client && speakeasy run --skip-versioning
+    if [ ! -z "${SPEAKEASY_API_KEY:-}" ]; then cd pkg/client && speakeasy run --skip-versioning; fi
 
 release-local:
     @goreleaser release --nightly --skip=publish --clean
@@ -60,4 +64,4 @@ release:
 generate-grpc-replication:
     protoc --go_out=. --go_opt=paths=source_relative \
         --go-grpc_out=. --go-grpc_opt=paths=source_relative \
-        ./internal/replication/grpc/replication_service.proto
+        ./internal/replication/grpc/replication_service.proto