deployment-tests.yml

# End-to-end deployment tests that deploy Aspire applications to real Azure infrastructure
#
# Triggers:
# - workflow_dispatch: Manual trigger with scenario selection
# - schedule: Nightly at 03:00 UTC
# - /deployment-test command on PRs (via deployment-test-command.yml)
#
# Security:
# - Uses OIDC (Workload Identity Federation) for Azure authentication
# - No stored Azure secrets
# - Only dotnet org members can trigger via PR command
#
name: Deployment E2E Tests

on:
  workflow_dispatch:
    inputs:
      pr_number:
        description: 'PR number to test (for testing PR builds)'
        required: false
        type: string
        default: ''

  schedule:
    # Run nightly at 03:00 UTC
    - cron: '0 3 * * *'

# Limit concurrent runs to avoid Azure quota issues
concurrency:
  group: deployment-e2e-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # Post "starting" comment to PR when triggered via /deployment-test command
  notify-start:
    name: Notify PR
    runs-on: ubuntu-latest
    if: ${{ github.repository_owner == 'dotnet' && inputs.pr_number != '' }}
    permissions:
      pull-requests: write
    steps:
      - name: Post starting comment
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          PR_NUMBER="${{ inputs.pr_number }}"
          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

          gh pr comment "${PR_NUMBER}" --repo "${{ github.repository }}" --body \
            "🚀 **Deployment tests starting** on PR #${PR_NUMBER}...

          This will deploy to real Azure infrastructure. Results will be posted here when complete.

          [View workflow run](${RUN_URL})"

  # Enumerate test classes to build the matrix
  enumerate:
    name: Enumerate Tests
    runs-on: ubuntu-latest
    if: ${{ github.repository_owner == 'dotnet' }}
    permissions:
      contents: read
    outputs:
      matrix: ${{ steps.enumerate.outputs.all_tests }}
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - uses: ./.github/actions/enumerate-tests
        id: enumerate
        with:
          buildArgs: '/p:OnlyDeploymentTests=true'

      - name: Display test matrix
        run: |
          echo "Deployment test matrix:"
          echo '${{ steps.enumerate.outputs.all_tests }}' | jq .

  # Build solution and CLI once, share via artifacts
  build:
    name: Build
    runs-on: 8-core-ubuntu-latest
    if: ${{ github.repository_owner == 'dotnet' }}
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - name: Setup .NET
        uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1
        with:
          global-json-file: global.json

      - name: Restore solution
        run: ./restore.sh

      - name: Build solution and pack CLI
        run: |
          # Build the full solution and pack CLI for local testing
          ./build.sh --build --pack -c Release
        env:
          # Skip native build to save time - we'll use the non-native CLI
          SkipNativeBuild: true

      - name: Prepare CLI artifacts
        run: |
          # Create a clean artifact directory with CLI and packages
          ARTIFACT_DIR="${{ github.workspace }}/cli-artifacts"
          mkdir -p "$ARTIFACT_DIR/bin"
          mkdir -p "$ARTIFACT_DIR/packages"

          # Copy CLI binary and dependencies
          cp -r "${{ github.workspace }}/artifacts/bin/Aspire.Cli/Release/net10.0/"* "$ARTIFACT_DIR/bin/"

          # Copy NuGet packages
          PACKAGES_DIR="${{ github.workspace }}/artifacts/packages/Release/Shipping"
          if [ -d "$PACKAGES_DIR" ]; then
            find "$PACKAGES_DIR" -name "*.nupkg" -exec cp {} "$ARTIFACT_DIR/packages/" \;
          fi

          echo "CLI artifacts prepared:"
          ls -la "$ARTIFACT_DIR/bin/"
          echo "Package count: $(find "$ARTIFACT_DIR/packages" -name "*.nupkg" | wc -l)"

      - name: Upload CLI artifacts
        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
        with:
          name: aspire-cli-artifacts
          path: ${{ github.workspace }}/cli-artifacts/
          retention-days: 1

  # Run each test class in parallel
  deploy-test:
    name: Deploy (${{ matrix.shortname }})
    needs: [enumerate, build]
    if: ${{ needs.enumerate.outputs.matrix != '{"include":[]}' && needs.enumerate.outputs.matrix != '' }}
    runs-on: 8-core-ubuntu-latest
    environment: deployment-testing
    permissions:
      id-token: write  # For OIDC Azure login
      contents: read
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.enumerate.outputs.matrix) }}
    env:
      ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION: ${{ secrets.AZURE_DEPLOYMENT_TEST_SUBSCRIPTION_ID }}
      ASPIRE_DEPLOYMENT_TEST_RG_PREFIX: ${{ vars.ASPIRE_DEPLOYMENT_TEST_RG_PREFIX || 'aspire-e2e' }}

    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - name: Setup .NET
        uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1
        with:
          global-json-file: global.json

      - name: Restore and build test project
        run: |
          ./restore.sh
          ./build.sh -restore -ci -build -projects ${{ github.workspace }}/tests/Aspire.Deployment.EndToEnd.Tests/Aspire.Deployment.EndToEnd.Tests.csproj -c Release
        env:
          SkipNativeBuild: true

      - name: Download CLI artifacts
        uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
        with:
          name: aspire-cli-artifacts
          path: ${{ github.workspace }}/cli-artifacts

      - name: Install Aspire CLI from artifacts
        run: |
          ASPIRE_HOME="$HOME/.aspire"
          mkdir -p "$ASPIRE_HOME/bin"

          # Copy CLI binary and dependencies
          cp -r "${{ github.workspace }}/cli-artifacts/bin/"* "$ASPIRE_HOME/bin/"
          chmod +x "$ASPIRE_HOME/bin/aspire"

          # Add to PATH for this job
          echo "$ASPIRE_HOME/bin" >> $GITHUB_PATH

          # Set up NuGet hive for local packages
          HIVE_DIR="$ASPIRE_HOME/hives/local/packages"
          mkdir -p "$HIVE_DIR"
          cp "${{ github.workspace }}/cli-artifacts/packages/"*.nupkg "$HIVE_DIR/" 2>/dev/null || true

          # Configure CLI to use local channel
          "$ASPIRE_HOME/bin/aspire" config set channel local --global || true

          echo "✅ Aspire CLI installed:"
          "$ASPIRE_HOME/bin/aspire" --version

      - name: Azure Login (OIDC)
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
        env:
          AZURE_CLIENT_ID: ${{ secrets.AZURE_DEPLOYMENT_TEST_CLIENT_ID }}
          AZURE_TENANT_ID: ${{ secrets.AZURE_DEPLOYMENT_TEST_TENANT_ID }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_DEPLOYMENT_TEST_SUBSCRIPTION_ID }}
        with:
          script: |
            const token = await core.getIDToken('api://AzureADTokenExchange');
            core.setSecret(token);

            // Login directly - token never leaves this step
            await exec.exec('az', [
              'login', '--service-principal',
              '--username', process.env.AZURE_CLIENT_ID,
              '--tenant', process.env.AZURE_TENANT_ID,
              '--federated-token', token,
              '--allow-no-subscriptions'
            ]);

            await exec.exec('az', [
              'account', 'set',
              '--subscription', process.env.AZURE_SUBSCRIPTION_ID
            ]);

      - name: Verify Azure authentication
        run: |
          echo "Verifying Azure authentication..."
          az account show --query "{subscriptionId:id, tenantId:tenantId, user:user.name}" -o table
          echo "✅ Azure authentication successful"

      - name: Verify Docker is running
        run: |
          echo "Verifying Docker daemon..."
          docker version
          docker info | head -20
          echo "✅ Docker is available"

      - name: Run deployment test (${{ matrix.shortname }})
        id: run_tests
        env:
          GITHUB_PR_NUMBER: ${{ inputs.pr_number || '' }}
          GITHUB_PR_HEAD_SHA: ${{ github.sha }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_DEPLOYMENT_TEST_SUBSCRIPTION_ID }}
          AZURE_TENANT_ID: ${{ secrets.AZURE_DEPLOYMENT_TEST_TENANT_ID }}
          AZURE_CLIENT_ID: ${{ secrets.AZURE_DEPLOYMENT_TEST_CLIENT_ID }}
          Azure__SubscriptionId: ${{ secrets.AZURE_DEPLOYMENT_TEST_SUBSCRIPTION_ID }}
          Azure__Location: westus3
          GH_TOKEN: ${{ github.token }}
        run: |
          ./dotnet.sh test tests/Aspire.Deployment.EndToEnd.Tests/Aspire.Deployment.EndToEnd.Tests.csproj \
            -c Release \
            --logger "trx;LogFileName=${{ matrix.shortname }}.trx" \
            --results-directory ${{ github.workspace }}/testresults \
            -- \
            --filter-not-trait "quarantined=true" \
            ${{ matrix.extraTestArgs }} \
            || echo "test_failed=true" >> $GITHUB_OUTPUT

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
        with:
          name: deployment-test-results-${{ matrix.shortname }}
          path: |
            ${{ github.workspace }}/testresults/
          retention-days: 30

      - name: Upload recordings
        if: always()
        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
        with:
          name: deployment-test-recordings-${{ matrix.shortname }}
          path: |
            ${{ github.workspace }}/testresults/recordings/
          retention-days: 30
          if-no-files-found: ignore

      - name: Check for test failures
        if: steps.run_tests.outputs.test_failed == 'true'
        run: |
          echo "::error::Deployment test ${{ matrix.shortname }} failed. Check the test results artifact for details."
          exit 1

  # Create GitHub issue on nightly failure
  create_issue_on_failure:
    name: Create Issue on Failure
    needs: [deploy-test]
    runs-on: ubuntu-latest
    if: ${{ failure() && github.event_name == 'schedule' }}
    permissions:
      issues: write
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - name: Create GitHub Issue
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const date = new Date().toISOString().split('T')[0];

            const issueTitle = `[Deployment E2E] Nightly test failure - ${date}`;
            const issueBody = `## Deployment E2E Test Failure

            The nightly deployment E2E tests failed on ${date}.

            **Workflow Run:** ${runUrl}

            ### Next Steps
            1. Check the workflow run for detailed error logs
            2. Download test artifacts for asciinema recordings
            3. Investigate and fix the failing tests

            ### Labels
            This issue was automatically created by the deployment E2E test workflow.

            /cc @dotnet/aspire-team
            `;

            // Check if a similar issue already exists (created today)
            const existingIssues = await github.rest.issues.listForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              state: 'open',
              labels: 'area-testing,deployment-e2e',
              per_page: 10
            });

            const todayIssue = existingIssues.data.find(issue =>
              issue.title.includes(date) && issue.title.includes('[Deployment E2E]')
            );

            if (todayIssue) {
              console.log(`Issue already exists for today: ${todayIssue.html_url}`);
              // Add a comment instead
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: todayIssue.number,
                body: `Another failure occurred. See: ${runUrl}`
              });
            } else {
              // Create new issue
              const issue = await github.rest.issues.create({
                owner: context.repo.owner,
                repo: context.repo.repo,
                title: issueTitle,
                body: issueBody,
                labels: ['area-testing', 'deployment-e2e']
              });
              console.log(`Created issue: ${issue.data.html_url}`);
            }

  # Post completion comment back to PR when triggered via /deployment-test command
  post_pr_comment:
    name: Post PR Comment
    needs: [deploy-test]
    runs-on: ubuntu-latest
    if: ${{ always() && inputs.pr_number != '' }}
    permissions:
      pull-requests: write
      actions: read
    steps:
      - name: Get job results and download recording artifacts
        id: get_results
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
        with:
          script: |
            const fs = require('fs');
            const path = require('path');

            // Get all jobs for this workflow run to determine per-test results
            const jobs = await github.paginate(
              github.rest.actions.listJobsForWorkflowRun,
              {
                owner: context.repo.owner,
                repo: context.repo.repo,
                run_id: context.runId,
                per_page: 100
              }
            );

            console.log(`Total jobs found: ${jobs.length}`);

            // Filter for deploy-test matrix jobs (format: "Deploy (TestClassName)")
            const deployJobs = jobs.filter(job => job.name.startsWith('Deploy ('));

            const passedTests = [];
            const failedTests = [];
            const cancelledTests = [];

            for (const job of deployJobs) {
              // Extract test name from job name "Deploy (TestClassName)"
              const match = job.name.match(/^Deploy \((.+)\)$/);
              const testName = match ? match[1] : job.name;

              console.log(`Job "${job.name}" - conclusion: ${job.conclusion}, status: ${job.status}`);

              if (job.conclusion === 'success') {
                passedTests.push(testName);
              } else if (job.conclusion === 'failure') {
                failedTests.push(testName);
              } else if (job.conclusion === 'cancelled') {
                cancelledTests.push(testName);
              }
            }

            console.log(`Passed: ${passedTests.length}, Failed: ${failedTests.length}, Cancelled: ${cancelledTests.length}`);

            // Output results for later steps
            core.setOutput('passed_tests', JSON.stringify(passedTests));
            core.setOutput('failed_tests', JSON.stringify(failedTests));
            core.setOutput('cancelled_tests', JSON.stringify(cancelledTests));
            core.setOutput('total_tests', passedTests.length + failedTests.length + cancelledTests.length);

            // List all artifacts for the current workflow run
            const allArtifacts = await github.paginate(
              github.rest.actions.listWorkflowRunArtifacts,
              {
                owner: context.repo.owner,
                repo: context.repo.repo,
                run_id: context.runId,
                per_page: 100
              }
            );

            console.log(`Total artifacts found: ${allArtifacts.length}`);

            // Filter for deployment test recording artifacts
            const recordingArtifacts = allArtifacts.filter(a =>
              a.name.startsWith('deployment-test-recordings-')
            );

            console.log(`Found ${recordingArtifacts.length} recording artifacts`);

            // Create recordings directory
            const recordingsDir = 'recordings';
            fs.mkdirSync(recordingsDir, { recursive: true });

            // Download each artifact
            for (const artifact of recordingArtifacts) {
              console.log(`Downloading ${artifact.name}...`);

              const download = await github.rest.actions.downloadArtifact({
                owner: context.repo.owner,
                repo: context.repo.repo,
                artifact_id: artifact.id,
                archive_format: 'zip'
              });

              const artifactPath = path.join(recordingsDir, `${artifact.name}.zip`);
              fs.writeFileSync(artifactPath, Buffer.from(download.data));
              console.log(`Saved to ${artifactPath}`);
            }

            core.setOutput('artifact_count', recordingArtifacts.length);

      - name: Extract recordings from artifacts
        shell: bash
        run: |
          mkdir -p cast_files

          for zipfile in recordings/*.zip; do
            if [ -f "$zipfile" ]; then
              echo "Extracting $zipfile..."
              unzip -o "$zipfile" -d "recordings/extracted_$(basename "$zipfile" .zip)" || true
            fi
          done

          # Find and copy all .cast files
          find recordings -name "*.cast" -exec cp {} cast_files/ \; 2>/dev/null || true

          echo "Found recordings:"
          ls -la cast_files/ || echo "No .cast files found"

      - name: Upload recordings to asciinema and post comment
        env:
          GH_TOKEN: ${{ github.token }}
          PASSED_TESTS: ${{ steps.get_results.outputs.passed_tests }}
          FAILED_TESTS: ${{ steps.get_results.outputs.failed_tests }}
          CANCELLED_TESTS: ${{ steps.get_results.outputs.cancelled_tests }}
          TOTAL_TESTS: ${{ steps.get_results.outputs.total_tests }}
        shell: bash
        run: |
          PR_NUMBER="${{ inputs.pr_number }}"
          RUN_ID="${{ github.run_id }}"
          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${RUN_ID}"
          TEST_RESULT="${{ needs.deploy-test.result }}"

          # Parse the test results from JSON
          PASSED_COUNT=$(echo "$PASSED_TESTS" | jq 'length')
          FAILED_COUNT=$(echo "$FAILED_TESTS" | jq 'length')
          CANCELLED_COUNT=$(echo "$CANCELLED_TESTS" | jq 'length')

          # Determine overall status
          if [ "$FAILED_COUNT" -gt 0 ]; then
            EMOJI="❌"
            STATUS="failed"
          elif [ "$CANCELLED_COUNT" -gt 0 ] && [ "$PASSED_COUNT" -eq 0 ]; then
            EMOJI="⚠️"
            STATUS="cancelled"
          elif [ "$PASSED_COUNT" -gt 0 ]; then
            EMOJI="✅"
            STATUS="passed"
          else
            EMOJI="❓"
            STATUS="unknown"
          fi

          # Build the comment header
          COMMENT_BODY="${EMOJI} **Deployment E2E Tests ${STATUS}**

          **Summary:** ${PASSED_COUNT} passed, ${FAILED_COUNT} failed, ${CANCELLED_COUNT} cancelled

          [View workflow run](${RUN_URL})"

          # Add passed tests section if any
          if [ "$PASSED_COUNT" -gt 0 ]; then
            PASSED_LIST=$(echo "$PASSED_TESTS" | jq -r '.[]' | while read test; do echo "- ✅ ${test}"; done)
            COMMENT_BODY="${COMMENT_BODY}

          ### Passed Tests
          ${PASSED_LIST}"
          fi

          # Add failed tests section if any
          if [ "$FAILED_COUNT" -gt 0 ]; then
            FAILED_LIST=$(echo "$FAILED_TESTS" | jq -r '.[]' | while read test; do echo "- ❌ ${test}"; done)
            COMMENT_BODY="${COMMENT_BODY}

          ### Failed Tests
          ${FAILED_LIST}"
          fi

          # Add cancelled tests section if any
          if [ "$CANCELLED_COUNT" -gt 0 ]; then
            CANCELLED_LIST=$(echo "$CANCELLED_TESTS" | jq -r '.[]' | while read test; do echo "- ⚠️ ${test}"; done)
            COMMENT_BODY="${COMMENT_BODY}

          ### Cancelled Tests
          ${CANCELLED_LIST}"
          fi

          # Check for recordings and upload them
          RECORDINGS_DIR="cast_files"

          if [ -d "$RECORDINGS_DIR" ] && compgen -G "$RECORDINGS_DIR"/*.cast > /dev/null; then
            # Install asciinema
            pip install --quiet asciinema

            RECORDING_TABLE="

          ### 🎬 Terminal Recordings

          | Test | Recording |
          |------|-----------|"

            UPLOAD_COUNT=0

            for castfile in "$RECORDINGS_DIR"/*.cast; do
              if [ -f "$castfile" ]; then
                filename=$(basename "$castfile" .cast)
                echo "Uploading $castfile..."

                # Upload to asciinema and capture URL
                UPLOAD_OUTPUT=$(asciinema upload "$castfile" 2>&1) || true
                ASCIINEMA_URL=$(echo "$UPLOAD_OUTPUT" | grep -oP 'https://asciinema\.org/a/[a-zA-Z0-9_-]+' | head -1) || true

                if [ -n "$ASCIINEMA_URL" ]; then
                  RECORDING_TABLE="${RECORDING_TABLE}
          | ${filename} | [▶️ View Recording](${ASCIINEMA_URL}) |"
                  echo "Uploaded: $ASCIINEMA_URL"
                  UPLOAD_COUNT=$((UPLOAD_COUNT + 1))
                else
                  RECORDING_TABLE="${RECORDING_TABLE}
          | ${filename} | ❌ Upload failed |"
                  echo "Failed to upload $castfile"
                fi
              fi
            done

            if [ $UPLOAD_COUNT -gt 0 ]; then
              COMMENT_BODY="${COMMENT_BODY}${RECORDING_TABLE}"
            fi

            echo "Uploaded $UPLOAD_COUNT recordings"
          else
            echo "No recordings found in $RECORDINGS_DIR"
          fi

          # Post the comment
          gh pr comment "${PR_NUMBER}" --repo "${{ github.repository }}" --body "$COMMENT_BODY"
          echo "Posted comment to PR #${PR_NUMBER}"