diff --git a/.github/actions/default/action.yml b/.github/actions/default/action.yml index ff9b4ddc1a..ed1894495e 100644 --- a/.github/actions/default/action.yml +++ b/.github/actions/default/action.yml @@ -9,19 +9,24 @@ runs: steps: - name: Install Nix uses: cachix/install-nix-action@v31 - - name: Cache dependencies - uses: nix-community/cache-nix-action@v6 - with: - primary-key: nix-${{ runner.os }}-${{ hashFiles('**/flake.nix', '**/flake.lock') }} - restore-prefixes-first-match: nix-${{ runner.os }}- - - name: Load dependencies + ## Disabling cache for now, as it's slowing down the build. Downloading nix + ## dependencies from cache.nixos.org is faster than restoring from the cache. + #- name: Cache dependencies + # uses: nix-community/cache-nix-action@v6 + # with: + # primary-key: nix-${{ runner.os }}-${{ hashFiles('**/flake.nix', '**/flake.lock') }} + # restore-prefixes-first-match: nix-${{ runner.os }}- + - name: go env + id: go-env shell: bash - run: nix develop --install + run: | + nix develop --command bash -c "go env | sed -E \"s/^([^=]+)='(.*)'\$/\1=\2/\"" >> "$GITHUB_OUTPUT" - uses: actions/cache@v4 with: path: | - ~/.cache/go-build - /tmp/go/pkg/mod/ + ${{ steps.go-env.outputs.GOCACHE }} + ${{ steps.go-env.outputs.GOMODCACHE }} + ~/.cache/golangci-lint key: ${{ runner.os }}-${{ github.job }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | - ${{ runner.os }}-${{ github.job }}-go- \ No newline at end of file + ${{ runner.os }}-${{ github.job }}-go- diff --git a/.github/workflows/benchmark-comparison.yml b/.github/workflows/benchmark-comparison.yml index 58d28ceb78..9d8f923fd0 100644 --- a/.github/workflows/benchmark-comparison.yml +++ b/.github/workflows/benchmark-comparison.yml @@ -3,23 +3,23 @@ on: workflow_dispatch: inputs: bench: - description: 'Benchmarks to run' + description: "Benchmarks to run" required: false - default: '.' + default: "." parallelism: - description: 'Number of parallel benchmarks to run' + description: "Number of parallel benchmarks to run" required: false - default: 5 + default: "5" duration: - description: 'Duration of each benchmark' + description: "Duration of each benchmark" required: false - default: '10s' + default: "10s" count: - description: 'Number of times to run each benchmark ' + description: "Number of times to run each benchmark " required: false - default: 1 + default: "1" pull_request: - types: [ assigned, opened, synchronize, reopened, labeled ] + types: [assigned, opened, synchronize, reopened, labeled] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} @@ -28,11 +28,12 @@ concurrency: jobs: BenchmarkCompare: runs-on: "github-001" - if: contains(github.event.pull_request.labels.*.name, 'benchmarks') + if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'benchmarks') steps: - - uses: 'actions/checkout@v4' + - uses: actions/checkout@v4 with: fetch-depth: 0 + filter: tree:0 # treeless clone, faster to clone as history and blobs are only fetched when needed. - name: Setup Env uses: ./.github/actions/env with: @@ -41,15 +42,15 @@ jobs: /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes" develop --impure --command just --justfile ./test/performance/justfile - --working-directory ./test/performance + --working-directory ./test/performance writes compare ${{ inputs.bench }} ${{ inputs.parallelism }} ${{ inputs.duration }} ${{ inputs.count }} - run: > /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes" - develop --impure --command just - --justfile ./test/performance/justfile - --working-directory ./test/performance + develop --impure --command just + --justfile ./test/performance/justfile + --working-directory ./test/performance writes graphs - uses: actions/upload-artifact@v4 with: name: graphs - path: test/performance/report \ No newline at end of file + path: test/performance/report diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 88e47a4bf5..b810c80279 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -3,17 +3,17 @@ on: workflow_dispatch: inputs: bench: - description: 'Benchmarks to run' + description: "Benchmarks to run" required: false - default: '.' + default: "." parallelism: - description: 'Number of parallel benchmarks to run' + description: "Number of parallel benchmarks to run" required: false - default: 5 + default: "5" duration: - description: 'Duration of each benchmark' + description: "Duration of each benchmark" required: false - default: '10s' + default: "10s" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -23,9 +23,10 @@ jobs: Benchmark: runs-on: "github-001" steps: - - uses: 'actions/checkout@v4' + - uses: actions/checkout@v4 with: fetch-depth: 0 + filter: tree:0 # treeless clone, faster to clone as history and blobs are only fetched when needed. - name: Setup Env uses: ./.github/actions/env with: @@ -34,15 +35,15 @@ jobs: /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes" develop --impure --command just --justfile ./test/performance/justfile - --working-directory ./test/performance + --working-directory ./test/performance writes run ${{ inputs.bench }} ${{ inputs.parallelism }} ${{ inputs.duration }} 1 - run: > /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes" - develop --impure --command just - --justfile ./test/performance/justfile - --working-directory ./test/performance + develop --impure --command just + --justfile ./test/performance/justfile + --working-directory ./test/performance writes graphs - uses: actions/upload-artifact@v4 with: name: graphs - path: test/performance/report \ No newline at end of file + path: test/performance/report diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 43ace650e5..13cc9271a0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,9 +30,10 @@ jobs: GOPATH: /tmp/go GOLANGCI_LINT_CACHE: /tmp/golangci-lint steps: - - uses: 'actions/checkout@v4' + - uses: actions/checkout@v4 with: fetch-depth: 0 + filter: tree:0 # treeless clone, faster to clone as history and blobs are only fetched when needed. - name: Setup Env uses: ./.github/actions/default with: @@ -45,7 +46,7 @@ jobs: id: changed-files shell: bash run: | - hasChanged=$(git status --porcelain) + hasChanged=$(git status --porcelain) if (( $(echo ${#hasChanged}) != 0 )); then git status echo "There are changes in the repository" @@ -58,9 +59,10 @@ jobs: env: GOPATH: /tmp/go steps: - - uses: 'actions/checkout@v4' + - uses: actions/checkout@v4 with: fetch-depth: 0 + filter: tree:0 # treeless clone, faster to clone as history and blobs are only fetched when needed. - name: Setup Env uses: ./.github/actions/default with: @@ -76,6 +78,9 @@ jobs: GoReleaser: runs-on: "shipfox-4vcpu-ubuntu-2404" + permissions: + id-token: write + attestations: write if: contains(github.event.pull_request.labels.*.name, 'build-images') || github.ref == 'refs/heads/main' || github.event_name == 'merge_group' steps: - name: Set up QEMU @@ -86,9 +91,10 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} version: "latest" - - uses: 'actions/checkout@v4' + - uses: actions/checkout@v4 with: fetch-depth: 0 + filter: tree:0 # treeless clone, faster to clone as history and blobs are only fetched when needed. ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - name: Setup Env uses: ./.github/actions/default @@ -112,37 +118,78 @@ jobs: FURY_TOKEN: ${{ secrets.FURY_TOKEN }} GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + - uses: actions/upload-artifact@v4 + with: + name: goreleaser-metadata + path: | + dist/*.json + dist/ledger_checksums.txt + retention-days: 7 + compression-level: 0 + + # Generate attestations for the goreleaser output archives + - uses: actions/attest-build-provenance@v2 + with: + subject-checksums: ./dist/ledger_checksums.txt + # Generate attestations for the goreleaser output binaries + - uses: actions/attest-build-provenance@v2 + with: + subject-path: ./dist/*/** + # Extract image metadata from the artifacts.json file + - run: | + jq -r '[ .[]|select(.type=="Docker Manifest") | .extra.Digest ] | to_entries | .[] | ( "digest"+ (.key | tostring) + "=" + .value )' < dist/artifacts.json >> "$GITHUB_OUTPUT" + jq -r '[ .[]|select(.type=="Docker Manifest") | .name | split(":")[0] ] | to_entries | .[] | ( "name"+ (.key | tostring) + "=" + .value )' < dist/artifacts.json >> "$GITHUB_OUTPUT" + id: image_metadata + # Generate attestations for docker images + - uses: actions/attest-build-provenance@v2 + with: + subject-digest: ${{ steps.image_metadata.outputs.digest0 }} + subject-name: ${{ steps.image_metadata.outputs.name0 }} + push-to-registry: true + - uses: actions/attest-build-provenance@v2 + with: + subject-digest: ${{ steps.image_metadata.outputs.digest1 }} + subject-name: ${{ steps.image_metadata.outputs.name1 }} + push-to-registry: true Deploy: - runs-on: "shipfox-2vcpu-ubuntu-2404" + runs-on: ubuntu-24.04 if: github.ref == 'refs/heads/main' environment: staging + permissions: + id-token: write needs: - GoReleaser - Tests steps: - name: Tailscale - uses: tailscale/github-action@v3 + uses: tailscale/github-action@v4 with: - oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} - oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} - tags: tag:ci + oauth-client-id: ${{ secrets.TS_OIDC_OAUTH_CLIENT_ID }} + audience: ${{ secrets.TS_OIDC_AUDIENCE }} + tags: ${{ vars.TS_TAGS }} + version: ${{ vars.TS_VERSION }} + args: ${{ vars.TS_ARGS }} + retry: ${{ vars.TS_RETRY }} + timeout: ${{ vars.TS_TIMEOUT }} + ping: ${{ vars.TS_PING }} - uses: earthly/actions-setup@v1 with: github-token: ${{ secrets.GITHUB_TOKEN }} version: "latest" - - uses: 'actions/checkout@v4' + - uses: actions/checkout@v4 with: fetch-depth: 0 + filter: tree:0 # treeless clone, faster to clone as history and blobs are only fetched when needed. - name: "Deploy in staging" env: TAG: ${{ github.sha }} COMPONENT: ledger ARGOCD_REGION_AUTH_TOKEN: ${{ secrets.ARGOCD_REGION_AUTH_TOKEN }} run: > - earthly - --no-output + earthly + --no-output --secret AUTH_TOKEN=$ARGOCD_REGION_AUTH_TOKEN +deploy-staging --TAG=$TAG - --COMPONENT=$COMPONENT \ No newline at end of file + --COMPONENT=$COMPONENT diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index dc288841c9..b9cab676a3 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -10,9 +10,10 @@ jobs: GoReleaser: runs-on: "shipfox-4vcpu-ubuntu-2404" steps: - - uses: 'actions/checkout@v4' + - uses: actions/checkout@v4 with: fetch-depth: 0 + filter: tree:0 # treeless clone, faster to clone as history and blobs are only fetched when needed. - name: Setup Env uses: ./.github/actions/default with: @@ -33,4 +34,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.NUMARY_GITHUB_TOKEN }} SPEAKEASY_API_KEY: ${{ secrets.SPEAKEASY_API_KEY }} FURY_TOKEN: ${{ secrets.FURY_TOKEN }} - GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} \ No newline at end of file + GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} diff --git a/Justfile b/Justfile index fcd266edec..701f48e4a2 100644 --- a/Justfile +++ b/Justfile @@ -3,11 +3,12 @@ set dotenv-load default: @just --list -pre-commit: tidy generate lint export-docs-events openapi generate-client +pre-commit: tidy generate generate-client lint export-docs-events openapi pc: pre-commit lint: - golangci-lint run --fix --build-tags it --timeout 5m + golangci-lint --version + golangci-lint run --fix --build-tags it,local --timeout 5m for d in $(ls tools); do \ pushd tools/$d; \ golangci-lint run --fix --build-tags it --timeout 5m; \ @@ -41,12 +42,15 @@ tests: cat coverage.txt | grep -v debug.go | grep -v "/machine/" | grep -v "pb.go" > coverage2.txt mv coverage2.txt coverage.txt +fmt: + @golangci-lint fmt + openapi: yq eval-all '. as $item ireduce ({}; . * $item)' openapi/v1.yaml openapi/v2.yaml openapi/overlay.yaml > openapi.yaml npx -y widdershins {{justfile_directory()}}/openapi/v2.yaml -o {{justfile_directory()}}/docs/api/README.md --search false --language_tabs 'http:HTTP' --summary --omitHeader generate-client: openapi - @cd pkg/client && speakeasy run --skip-versioning + if [ ! -z "${SPEAKEASY_API_KEY:-}" ]; then cd pkg/client && speakeasy run --skip-versioning; fi release-local: @goreleaser release --nightly --skip=publish --clean @@ -60,4 +64,4 @@ release: generate-grpc-replication: protoc --go_out=. --go_opt=paths=source_relative \ --go-grpc_out=. --go-grpc_opt=paths=source_relative \ - ./internal/replication/grpc/replication_service.proto \ No newline at end of file + ./internal/replication/grpc/replication_service.proto diff --git a/flake.lock b/flake.lock index b1418761d8..61ac31d38d 100644 --- a/flake.lock +++ b/flake.lock @@ -23,16 +23,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1769461804, - "narHash": "sha256-msG8SU5WsBUfVVa/9RPLaymvi5bI8edTavbIq3vRlhI=", - "rev": "bfc1b8a4574108ceef22f02bafcf6611380c100d", - "revCount": 935279, + "lastModified": 1772047000, + "narHash": "sha256-7DaQVv4R97cii/Qdfy4tmDZMB2xxtyIvNGSwXBBhSmo=", + "rev": "1267bb4920d0fc06ea916734c11b0bf004bbe17e", + "revCount": 908388, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.1.935279%2Brev-bfc1b8a4574108ceef22f02bafcf6611380c100d/019c02ef-f13d-717e-8527-f1603ec205db/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2511.908388%2Brev-1267bb4920d0fc06ea916734c11b0bf004bbe17e/019ca036-f9d8-73b9-81f5-c14b9b593dcc/source.tar.gz" }, "original": { "type": "tarball", - "url": "https://flakehub.com/f/NixOS/nixpkgs/0.1.%2A.tar.gz" + "url": "https://flakehub.com/f/NixOS/nixpkgs/0.2511" } }, "nur": { @@ -43,11 +43,11 @@ ] }, "locked": { - "lastModified": 1769768855, - "narHash": "sha256-OuUk+skANEQBNDKLiXvZOcdv2vfWsrbxvwNPSw0iKjE=", + "lastModified": 1772478921, + "narHash": "sha256-jkXpp1vpQwFQ3sjp0I+8t3qmFciJUgGK21MxtCAtr1c=", "owner": "nix-community", "repo": "NUR", - "rev": "cf3ece1c8c4108d7e5940ba7f1b01048407fff10", + "rev": "4fafc922aed8225d9ae24af94f01a2a793630b83", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 69e4588efc..e6729f884e 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-flake-based Go 1.24 development environment"; inputs = { - nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.1.*.tar.gz"; + nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.2511"; nur = { url = "github:nix-community/NUR"; @@ -12,8 +12,6 @@ outputs = { self, nixpkgs, nur }: let - goVersion = 24; - supportedSystems = [ "x86_64-linux" "aarch64-linux" @@ -26,8 +24,10 @@ let pkgs = import nixpkgs { inherit system; - overlays = [ self.overlays.default nur.overlays.default ]; - config.allowUnfree = true; + overlays = [ nur.overlays.default ]; + config.allowUnfreePredicate = pkg: builtins.elem (nixpkgs.lib.getName pkg) [ + "goreleaser-pro" + ]; }; in f { pkgs = pkgs; system = system; } @@ -42,17 +42,13 @@ }; speakeasyHashes = { "x86_64-linux" = "632559a6bdc765ef42b81b8404fd0a3e5023919a0bb70ff7e40a8cc259545afd"; - "aarch64-linux" = "c74c502df3a05a2d69e6b282886df23354a319d0510d2c1a21fcc124b7ad00efGOROOT"; + "aarch64-linux" = "c74c502df3a05a2d69e6b282886df23354a319d0510d2c1a21fcc124b7ad00ef"; "x86_64-darwin" = "8814be1fdd4eaf6dcc7fb251ede5693e1d3d4c8e03986f8d37bfd59e049698b9"; "aarch64-darwin" = "12c20fa485de4725c9730cb2e8936cab4b0961d0a956e9f4c45534371f2a6148"; }; in { - overlays.default = final: prev: { - go = final."go_1_${toString goVersion}"; - }; - packages = forEachSupportedSystem ({ pkgs, system }: { speakeasy = pkgs.stdenv.mkDerivation { @@ -85,30 +81,35 @@ defaultPackage.aarch64-darwin = self.packages.aarch64-darwin.speakeasy; devShells = forEachSupportedSystem ({ pkgs, system }: + let + stablePackages = with pkgs; [ + ginkgo + go_1_24 + go-tools + golangci-lint + gomarkdoc + goperf + gotools + jdk11 + jq + just + mockgen + nodejs_22 + protobuf_27 + protoc-gen-go + protoc-gen-go-grpc + yq-go + ]; + otherPackages = [ + pkgs.nur.repos.goreleaser.goreleaser-pro + self.packages.${system}.speakeasy + ]; + in { default = pkgs.mkShell { - packages = with pkgs; [ - go - gotools - go-tools - golangci-lint - ginkgo - yq-go - jq - pkgs.nur.repos.goreleaser.goreleaser-pro - mockgen - gomarkdoc - jdk11 - just - nodejs_22 - self.packages.${system}.speakeasy - goperf - protobuf_27 - protoc-gen-go-grpc - protoc-gen-go - ]; + packages = stablePackages ++ otherPackages; }; } ); }; -} \ No newline at end of file +} diff --git a/test/performance/pkg/write/local_env_test.go b/test/performance/pkg/write/local_env_test.go index a271d886e7..04433d23dc 100644 --- a/test/performance/pkg/write/local_env_test.go +++ b/test/performance/pkg/write/local_env_test.go @@ -6,9 +6,9 @@ import ( "context" "github.com/formancehq/go-libs/v3/logging" "github.com/formancehq/go-libs/v3/otlp/otlpmetrics" + "github.com/formancehq/go-libs/v3/testing/deferred" "github.com/formancehq/go-libs/v3/testing/docker" "github.com/formancehq/go-libs/v3/testing/testservice" - . "github.com/formancehq/go-libs/v3/testing/utils" ledgerclient "github.com/formancehq/ledger/pkg/client" "github.com/formancehq/ledger/test/performance/pkg/env" "io" @@ -62,11 +62,11 @@ func (f *TestServerEnvFactory) Create(ctx context.Context, b *testing.B) env.Env } testServer := testserver.NewTestServer( - NewValuedDeferred(connectionOptions), + deferred.FromValue(connectionOptions), testservice.WithInstruments( testservice.DebugInstrumentation(os.Getenv("DEBUG") == "true"), testservice.OutputInstrumentation(output), - testservice.OTLPInstrumentation(NewValuedDeferred(testservice.OTLPConfig{ + testservice.OTLPInstrumentation(deferred.FromValue(testservice.OTLPConfig{ Metrics: &otlpmetrics.ModuleConfig{ KeepInMemory: true, RuntimeMetrics: true,