backend-selective.yml

# Parallel validation workflow: runs sentry backend tests using getsentry's
# cross-repo coverage DB for selective test selection. Runs alongside the
# existing backend.yml — not required for merge.
#
# Once validated, this will replace prepare-selective-tests in backend.yml.
name: backend (NOT REQUIRED - selective via getsentry)

on:
  pull_request:
    types: [opened, synchronize, reopened, labeled]

# Cancel in progress workflows on pull_requests.
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash -euo pipefail {0}

# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359
env:
  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3
  SNUBA_NO_WORKERS: 1
  SENTRY_SKIP_SELENIUM_PLUGIN: '1'

permissions:
  contents: read
  id-token: write
  actions: read

jobs:
  files-changed:
    name: detect what files changed
    runs-on: ubuntu-24.04
    timeout-minutes: 3
    outputs:
      backend: ${{ steps.changes.outputs.backend_all_without_acceptance }}
      skip_selective_testing: "${{ contains(github.event.pull_request.labels.*.name, 'Trigger: Override Selective Testing') }}"
    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

      - name: Check for backend file changes
        uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0
        id: changes
        with:
          token: ${{ github.token }}
          filters: .github/file-filters.yml

  select-tests:
    if: >-
      needs.files-changed.outputs.backend == 'true' &&
      needs.files-changed.outputs.skip_selective_testing != 'true' &&
      github.event.pull_request.head.repo.full_name == github.repository
    needs: files-changed
    name: select tests via getsentry coverage
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    outputs:
      has-selected-tests: ${{ steps.compute-tests.outputs.has-selected-tests }}
      test-count: ${{ steps.compute-tests.outputs.test-count }}
    steps:
      - name: Get changed files
        id: changed
        env:
          GH_TOKEN: ${{ github.token }}
        run: |

          CHANGED_FILES=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files --paginate --jq '.[].filename' | tr '\n' ' ')
          echo "Changed files: $CHANGED_FILES"
          echo "files=$CHANGED_FILES" >> "$GITHUB_OUTPUT"

      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

      - name: Setup Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version-file: '.python-version'

      - name: Authenticate to Google Cloud
        uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2.1.3
        with:
          project_id: sentry-dev-tooling
          workload_identity_provider: ${{ secrets.SENTRY_GCP_DEV_WORKLOAD_IDENTITY_POOL }}
          service_account: ${{ secrets.COLLECT_TEST_DATA_SERVICE_ACCOUNT_EMAIL }}

      - name: Download coverage database
        id: download-coverage
        run: |

          mkdir -p .artifacts/coverage

          GCS_PATH="gs://getsentry-coverage-data/latest/.coverage.combined"
          echo "Fetching coverage DB from: $GCS_PATH"
          gcloud storage ls -l "$GCS_PATH" 2>/dev/null || true

          if ! gcloud storage cp "$GCS_PATH" \
            .artifacts/coverage/.coverage.combined 2>/dev/null; then
            echo "Warning: Failed to download coverage from GCS, will run full test suite"
            echo "coverage-file=" >> "$GITHUB_OUTPUT"
          else
            ls -lh .artifacts/coverage/.coverage.combined
            echo "coverage-file=.artifacts/coverage/.coverage.combined" >> "$GITHUB_OUTPUT"
          fi

      - name: Compute selected tests
        id: compute-tests
        if: steps.download-coverage.outputs.coverage-file != ''
        env:
          COVERAGE_DB: ${{ steps.download-coverage.outputs.coverage-file }}
          CHANGED_FILES: ${{ steps.changed.outputs.files }}
        run: |


          python3 .github/workflows/scripts/compute-sentry-selected-tests.py \
            --coverage-db "$COVERAGE_DB" \
            --changed-files "$CHANGED_FILES" \
            --output .artifacts/selected-tests.txt \
            --github-output

      - name: Upload selected tests artifact
        if: steps.compute-tests.outputs.has-selected-tests == 'true'
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: selected-tests-${{ github.run_id }}
          path: .artifacts/selected-tests.txt
          retention-days: 1

  calculate-shards:
    if: >-
      needs.files-changed.outputs.backend == 'true' &&
      needs.files-changed.outputs.skip_selective_testing != 'true'
    needs: [files-changed, select-tests]
    name: calculate test shards (selective)
    runs-on: ubuntu-24.04
    timeout-minutes: 5
    outputs:
      shard-count: ${{ steps.calculate-shards.outputs.shard-count }}
      shard-indices: ${{ steps.calculate-shards.outputs.shard-indices }}
    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

      - name: Setup sentry env
        uses: ./.github/actions/setup-sentry
        id: setup
        with:
          mode: backend-ci
          skip-devservices: true

      - name: Download selected tests artifact
        if: needs.select-tests.outputs.has-selected-tests == 'true'
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
        with:
          name: selected-tests-${{ github.run_id }}
          path: .artifacts/

      - name: Calculate test shards
        id: calculate-shards
        env:
          SELECTED_TESTS_FILE: ${{ needs.select-tests.outputs.has-selected-tests == 'true' && '.artifacts/selected-tests.txt' || '' }}
          SELECTED_TEST_COUNT: ${{ needs.select-tests.outputs.test-count || '' }}
        run: |
          python3 .github/workflows/scripts/calculate-backend-test-shards.py

  backend-test:
    if: >-
      needs.files-changed.outputs.backend == 'true' &&
      needs.files-changed.outputs.skip_selective_testing != 'true' &&
      needs.calculate-shards.outputs.shard-count != '0'
    needs: [files-changed, select-tests, calculate-shards]
    name: backend test (selective)
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        instance: ${{ fromJSON(needs.calculate-shards.outputs.shard-indices) }}
    env:
      MATRIX_INSTANCE_TOTAL: ${{ needs.calculate-shards.outputs.shard-count }}
      TEST_GROUP_STRATEGY: roundrobin
      PYTHONHASHSEED: '0'
      XDIST_PER_WORKER_SNUBA: '1'
      XDIST_WORKERS: '2'
    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

      - name: Setup sentry env
        uses: ./.github/actions/setup-sentry
        id: setup
        with:
          mode: backend-ci

      - name: Download odiff binary
        run: |
          curl -sL https://registry.npmjs.org/odiff-bin/-/odiff-bin-4.3.2.tgz \
            | tar -xz --strip-components=2 package/raw_binaries/odiff-linux-x64
          sudo install -m 755 odiff-linux-x64 /usr/local/bin/odiff
          rm odiff-linux-x64

      - name: Bootstrap per-worker Snuba instances
        run: |
          set -eo pipefail
          SNUBA_IMAGE=$(docker inspect snuba-snuba-1 --format '{{.Config.Image}}')
          SNUBA_NETWORK=$(docker inspect snuba-snuba-1 --format '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}')
          if [ -z "$SNUBA_IMAGE" ] || [ -z "$SNUBA_NETWORK" ]; then
            echo "ERROR: Could not inspect snuba-snuba-1 container. Is devservices running?"
            exit 1
          fi

          docker stop snuba-snuba-1 || true

          PIDS=()
          for i in $(seq 0 $(( ${XDIST_WORKERS} - 1 ))); do
            (
              WORKER_DB="default_gw${i}"
              WORKER_PORT=$((1230 + i))
              curl -sf 'http://localhost:8123/' --data-binary "CREATE DATABASE IF NOT EXISTS ${WORKER_DB}"
              docker run --rm --network "$SNUBA_NETWORK" \
                -e "CLICKHOUSE_DATABASE=${WORKER_DB}" -e "CLICKHOUSE_HOST=clickhouse" \
                -e "CLICKHOUSE_PORT=9000" -e "CLICKHOUSE_HTTP_PORT=8123" \
                -e "DEFAULT_BROKERS=kafka:9093" -e "REDIS_HOST=redis" \
                -e "REDIS_PORT=6379" -e "REDIS_DB=1" -e "SNUBA_SETTINGS=docker" \
                "$SNUBA_IMAGE" bootstrap --force 2>&1 | tail -3
              docker run -d --name "snuba-gw${i}" --network "$SNUBA_NETWORK" \
                -p "${WORKER_PORT}:1218" \
                -e "CLICKHOUSE_DATABASE=${WORKER_DB}" -e "CLICKHOUSE_HOST=clickhouse" \
                -e "CLICKHOUSE_PORT=9000" -e "CLICKHOUSE_HTTP_PORT=8123" \
                -e "DEFAULT_BROKERS=kafka:9093" -e "REDIS_HOST=redis" \
                -e "REDIS_PORT=6379" -e "REDIS_DB=1" -e "SNUBA_SETTINGS=docker" \
                -e "DEBUG=1" "$SNUBA_IMAGE" api

              for attempt in $(seq 1 30); do
                if curl -sf "http://127.0.0.1:${WORKER_PORT}/health" > /dev/null 2>&1; then
                  echo "snuba-gw${i} healthy on port ${WORKER_PORT}"
                  break
                fi
                if [ "$attempt" -eq 30 ]; then
                  echo "ERROR: snuba-gw${i} failed health check after 30 attempts"
                  docker logs "snuba-gw${i}" 2>&1 | tail -20 || true
                  exit 1
                fi
                sleep 2
              done
            ) &
            PIDS+=($!)
          done

          for pid in "${PIDS[@]}"; do
            wait "$pid" || { echo "ERROR: Snuba bootstrap subshell (PID $pid) failed"; exit 1; }
          done

      - name: Download selected tests artifact
        if: needs.select-tests.outputs.has-selected-tests == 'true'
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
        with:
          name: selected-tests-${{ github.run_id }}
          path: .artifacts/

      - name: Run backend test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }})
        env:
          SELECTED_TESTS_FILE: ${{ needs.select-tests.outputs.has-selected-tests == 'true' && '.artifacts/selected-tests.txt' || '' }}
        run: |
          export PYTEST_ADDOPTS="$PYTEST_ADDOPTS -n ${XDIST_WORKERS} --dist=loadfile"
          make test-python-ci

      - name: Inspect failure
        if: failure()
        run: |
          if command -v devservices; then
            devservices logs
          fi

          for i in $(seq 0 $(( ${XDIST_WORKERS} - 1 ))); do
            echo "--- snuba-gw${i} logs ---"
            docker logs "snuba-gw${i}" 2>&1 | tail -30 || true
          done