From 4ae03bcbd0888ef331bbcc038b26231b5beb89a0 Mon Sep 17 00:00:00 2001 From: bagel897 Date: Wed, 22 Jan 2025 09:50:29 -0800 Subject: [PATCH 1/8] move changes --- .circleci/config.yml | 91 ++++++++++++++++++++-- .github/actions/report/action.yml | 21 ----- .github/actions/setup-oss-repos/action.yml | 27 ------- .github/codecov.yml | 6 ++ .github/workflows/cache-warm-up.yml | 63 --------------- .github/workflows/unit-tests.yml | 64 --------------- 6 files changed, 92 insertions(+), 180 deletions(-) delete mode 100644 .github/actions/report/action.yml delete mode 100644 .github/actions/setup-oss-repos/action.yml delete mode 100644 .github/workflows/cache-warm-up.yml delete mode 100644 .github/workflows/unit-tests.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index c9316ec91..23075a1cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,18 +46,22 @@ commands: command: | curl -LsSf https://astral.sh/uv/install.sh | sh source $HOME/.local/bin/env - cache-extra-repos: + clone-repos: + parameters: + extra_repos: + type: boolean + default: true steps: - restore_cache: keys: - - extra-repos + - repos-{{ parameters.extra_repos }} - run: command: | - uv run --frozen gs codemod clone-repos --extra-repos --token ${CODEGEN_BOT_GHE_TOKEN} --clean-cache + uv run --frozen gs codemod clone-repos --extra-repos=<> --token ${CODEGEN_BOT_GHE_TOKEN} --clean-cache - save_cache: paths: - $GITHUB_WORKSPACE - key: extra-repos + key: repos-{{ parameters.extra_repos }} fetch-verified-codemods: steps: - run: @@ -242,6 +246,33 @@ jobs: branch_pattern: "develop" channel: "alerts-codemod-tests" template: basic_fail_1 + oss-codemod-tests: + parameters: + sync_graph: + type: boolean + default: true + size: + type: string + default: "small" + parallelism: 2 + executor: default_vm + resource_class: xlarge + steps: + - install-uv + - setup-uv + - clone-repos: + extra_repos: false + - run_ats: + default_tests: "tests/codemod/test_codemods.py" + codecov_flags: "smart-tests-codemod-oss" + collect_args: --size=${parameters.size} --sync-graph=${parameters.sync_graph} --token ${CODEGEN_BOT_GHE_TOKEN} + ats_collect_args: --size=${parameters.size},--sync-graph=${parameters.sync_graph},--token=${CODEGEN_BOT_GHE_TOKEN}, + split_tests: false + - slack/notify: + event: fail + branch_pattern: "develop" + channel: "alerts-codemod-tests" + template: basic_fail_1 parse-tests: parameters: extra_repos: @@ -257,7 +288,8 @@ jobs: install-pnpm: true install-yarn: true use-nvm-cache: true - - cache-extra-repos + - clone-repos: + extra_repos: <> - run: command: | PYTEST_ARGS='--extra-repos=<> --token ${CODEGEN_BOT_GHE_TOKEN} -o junit_suite_name="${CIRCLE_JOB}" tests/codemod/test_parse.py' @@ -370,6 +402,9 @@ workflows: - GHE requires: - hold-parse-tests + matrix: + parameters: + extra_repos: [true, false] - parse-tests: context: - GHE @@ -377,19 +412,65 @@ workflows: filters: branches: only: develop + extra_repos: true + - parse-tests: + name: oss-parse-tests + context: + - GHE + - slack + filters: + branches: + only: develop + extra_repos: false + - oss-codemod-tests: + context: + - Codecov + - GHE + - slack + matrix: + parameters: + sync_graph: [true, false] + size: [large] pr_checks: jobs: - unit-tests: context: - Codecov + - hold-codemod-tests: + type: approval + - hold-large-oss-codemod-tests: + type: approval - codemod-tests: context: - Codecov - GHE - Codemod - slack + requires: + - hold-codemod-tests - pre-commit - doctests + - oss-codemod-tests: + context: + - Codecov + - GHE + - slack + matrix: + parameters: + sync_graph: [true, false] + size: [small] + - oss-codemod-tests: + name: large-oss-codemod-tests + context: + - Codecov + - GHE + - slack + matrix: + parameters: + sync_graph: [true, false] + size: [large] + requires: + - hold-large-oss-codemod-tests publish-packages: jobs: - linux-wheels: diff --git a/.github/actions/report/action.yml b/.github/actions/report/action.yml deleted file mode 100644 index 7a9978c55..000000000 --- a/.github/actions/report/action.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: report -description: "Publish Junit Report" - -runs: - using: "composite" - steps: - - name: Upload Test Report - uses: actions/upload-artifact@v4 - with: - name: junit-test-results-${{github.run_id}}-${{github.job}}-${{ join(matrix.*, ' - ') }} - path: '**/build/test-results/test/TEST.xml' - retention-days: 1 - - name: Download Test Report - uses: actions/download-artifact@v4 - with: - pattern: junit-test-results-${{github.run_id}}* - - name: Publish Test Report - uses: EnricoMi/publish-unit-test-result-action@v2 - with: - files: "**/build/test-results/test/TEST.xml" - deduplicate_classes_by_file_name: true \ No newline at end of file diff --git a/.github/actions/setup-oss-repos/action.yml b/.github/actions/setup-oss-repos/action.yml deleted file mode 100644 index 30dca0a87..000000000 --- a/.github/actions/setup-oss-repos/action.yml +++ /dev/null @@ -1,27 +0,0 @@ -# yaml-language-server: $schema=https://json.schemastore.org/github-action.json -name: "Setup OSS repos" -description: "Setup OSS repos" -inputs: - CODEGEN_BOT_GHE_TOKEN: - required: true - description: "Codegen bot PAT for readonly GHE access" -runs: - using: "composite" - steps: - - name: Cache oss-repos - id: cache-oss-repos - uses: actions/cache@v4 - with: - path: oss_repos - key: ${{ runner.os }}-repo-cache-2-${{hashFiles('codegen-backend/codegen_tests/graph_sitter/codemod/repos/open_source/*.json')}} - - name: Populate oss-repos if the cache is empty - if: steps.cache-oss-repos.outputs.cache-hit != 'true' - shell: bash - run: | - uv run gs codemod clone-repos --clean-cache --token $CODEGEN_BOT_GHE_TOKEN - env: - GITHUB_WORKSPACE: $GITHUB_WORKSPACE - CODEGEN_BOT_GHE_TOKEN: ${{ inputs.CODEGEN_BOT_GHE_TOKEN }} - - name: Verify cache contents - shell: bash - run: ls -la $GITHUB_WORKSPACE/oss_repos/ diff --git a/.github/codecov.yml b/.github/codecov.yml index 634b84ff9..183deb54e 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -54,6 +54,12 @@ flag_management: statuses: - type: 'project' - type: 'patch' + - name: smart-tests-codemod-oss + carryforward: true + carryforward_mode: 'labels' + statuses: + - type: 'project' + - type: 'patch' comment: layout: "condensed_header, condensed_files" hide_project_coverage: true diff --git a/.github/workflows/cache-warm-up.yml b/.github/workflows/cache-warm-up.yml deleted file mode 100644 index d7b8f0aaf..000000000 --- a/.github/workflows/cache-warm-up.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Cache Warm-Up - -on: - workflow_dispatch: - push: - branches: - - "develop" - -jobs: - - warm-up-cache-uv: - # changing the following value will significantly affect github's billing. Be careful and consult with the team before changing it. - strategy: - matrix: - os: [ ubuntu-latest, macos-latest ] - runs-on: ${{ matrix.os }} - - steps: - - name: Check if on develop branch - run: | - if [ "${{ github.ref }}" != "refs/heads/develop" ]; then - echo "Not on develop branch. Exiting." - exit 1 - fi - - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: develop # Ensure we're operating on the 'develop' branch - - - name: Setup backend - uses: ./.github/actions/setup-backend - warm-up-cache: - # changing the following value will significantly affect github's billing. Be careful and consult with the team before changing it. - runs-on: ubuntu-latest - needs: [warm-up-cache-uv] - steps: - - name: Check if on develop branch - run: | - if [ "${{ github.ref }}" != "refs/heads/develop" ]; then - echo "Not on develop branch. Exiting." - exit 1 - fi - - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: develop # Ensure we're operating on the 'develop' branch - - - name: Setup backend - uses: ./.github/actions/setup-backend - - name: Cache oss-repos - uses: ./.github/actions/setup-oss-repos - with: - CODEGEN_BOT_GHE_TOKEN: ${{ secrets.CODEGEN_BOT_GHE_TOKEN }} - - name: Setup-pre-commit - run: uv tool install pre-commit --with pre-commit-uv --force-reinstall - - uses: actions/cache@v4 - with: - path: ~/.cache/pre-commit - key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} - - run: pre-commit install-hooks - shell: bash diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index 04f10774e..000000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Unit Tests - -on: - push: - branches: - - "develop" - pull_request: - types: [ opened, synchronize, reopened, labeled ] - branches: - - "develop" - workflow_dispatch: - -jobs: - test_codemods: - # changing the following value will significantly affect github's billing. Be careful and consult with the team before changing it. - runs-on: ubuntu-latest-32 - strategy: - matrix: - sync_graph: [ true, false ] - size: [ small, large ] - exclude: - # Exclude large codemod tests when not needed - - size: ${{(contains(github.event.pull_request.labels.*.name, 'big-codemod-tests') || github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'kevin' || 'large'}} - - size: large - sync_graph: true - concurrency: - group: ${{ github.workflow }}-${{github.ref}}-${{matrix.sync_graph}}-${{matrix.size}}-${{github.event_name == 'push'&& github.sha}} - cancel-in-progress: true - name: "Codemod tests ${{matrix.size}}: Sync Graph=${{matrix.sync_graph}}" - steps: - - uses: actions/checkout@v4 - with: - ssh-key: ${{secrets.CODEGEN_BOT_SSH_KEY}} - - run: | - git lfs install - git lfs pull - - name: Setup backend - uses: ./.github/actions/setup-backend - - name: Cache oss-repos - uses: ./.github/actions/setup-oss-repos - with: - CODEGEN_BOT_GHE_TOKEN: ${{ secrets.CODEGEN_BOT_GHE_TOKEN }} - - name: Test with pytest - timeout-minutes: 12 - run: | - uv run pytest \ - -n auto \ - --size=${{matrix.size}} \ - --sync-graph=${{matrix.sync_graph}} \ - --token $CODEGEN_BOT_GHE_TOKEN \ - -o junit_suite_name="${{github.job}}" \ - tests/codemod/test_codemods.py::test_codemods_cloned_repos - env: - CODEGEN_BOT_GHE_TOKEN: ${{ secrets.CODEGEN_BOT_GHE_TOKEN }} - GITHUB_WORKSPACE: $GITHUB_WORKSPACE - - name: Install pycoverage - run: | - pip install coverage - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: build/test-results/test/TEST.xml From f444b35b624f2853c3382a9fa8b34bfe0599a281 Mon Sep 17 00:00:00 2001 From: bagel897 Date: Wed, 22 Jan 2025 10:45:59 -0800 Subject: [PATCH 2/8] Update lock --- uv.lock | 2 -- 1 file changed, 2 deletions(-) diff --git a/uv.lock b/uv.lock index 2119ca4a2..ec89f17ec 100644 --- a/uv.lock +++ b/uv.lock @@ -716,7 +716,6 @@ dependencies = [ { name = "tree-sitter-javascript" }, { name = "tree-sitter-python" }, { name = "tree-sitter-typescript" }, - { name = "typer" }, { name = "typing-extensions" }, { name = "watchfiles" }, { name = "wrapt" }, @@ -802,7 +801,6 @@ requires-dist = [ { name = "tree-sitter-javascript", specifier = ">=0.23.1" }, { name = "tree-sitter-python", specifier = ">=0.23.4" }, { name = "tree-sitter-typescript", specifier = ">=0.23.2" }, - { name = "typer", specifier = ">=0.12.5,<1.0.0" }, { name = "types-networkx", marker = "extra == 'types'", specifier = ">=3.2.1.20240918" }, { name = "types-tabulate", marker = "extra == 'types'", specifier = ">=0.9.0.20240106" }, { name = "typing-extensions", specifier = ">=4.12.2" }, From 5d9741a8848a533ad828210bc64670466010764d Mon Sep 17 00:00:00 2001 From: bagel897 Date: Wed, 22 Jan 2025 10:59:21 -0800 Subject: [PATCH 3/8] Fix pipeline --- .circleci/config.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 23075a1cd..2455ffe49 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -292,7 +292,11 @@ jobs: extra_repos: <> - run: command: | - PYTEST_ARGS='--extra-repos=<> --token ${CODEGEN_BOT_GHE_TOKEN} -o junit_suite_name="${CIRCLE_JOB}" tests/codemod/test_parse.py' + EXTRA_REPOS_ARG="" + if [ "<>" = "true" ]; then + EXTRA_REPOS_ARG="--extra-repos=true" + fi + PYTEST_ARGS="${EXTRA_REPOS_ARG} --token ${CODEGEN_BOT_GHE_TOKEN} -o junit_suite_name=\"${CIRCLE_JOB}\" tests/codemod/test_parse.py" echo "Running tests with args: $PYTEST_ARGS" TESTS_TO_RUN=$(PYTEST_ARGS=${PYTEST_ARGS} ./.circleci/collect.sh) echo $TESTS_TO_RUN | circleci tests run --command "ulimit -s unlimited; xargs uv run --frozen pytest -n auto ${PYTEST_ARGS}" From b95cdbbf82c1391355bf7cc920b3df121c8255b2 Mon Sep 17 00:00:00 2001 From: bagel897 Date: Wed, 22 Jan 2025 11:02:07 -0800 Subject: [PATCH 4/8] Remove codecov context --- .circleci/config.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2455ffe49..8e84cfaee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -428,7 +428,6 @@ workflows: extra_repos: false - oss-codemod-tests: context: - - Codecov - GHE - slack matrix: @@ -437,16 +436,13 @@ workflows: size: [large] pr_checks: jobs: - - unit-tests: - context: - - Codecov + - unit-tests - hold-codemod-tests: type: approval - hold-large-oss-codemod-tests: type: approval - codemod-tests: context: - - Codecov - GHE - Codemod - slack @@ -456,7 +452,6 @@ workflows: - doctests - oss-codemod-tests: context: - - Codecov - GHE - slack matrix: @@ -466,7 +461,6 @@ workflows: - oss-codemod-tests: name: large-oss-codemod-tests context: - - Codecov - GHE - slack matrix: From e8229fa64176108b3787612832a5b530e9c47fde Mon Sep 17 00:00:00 2001 From: bagel897 Date: Wed, 22 Jan 2025 11:04:32 -0800 Subject: [PATCH 5/8] Fix extra-repos param --- .circleci/config.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8e84cfaee..283a611cb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,7 +57,11 @@ commands: - repos-{{ parameters.extra_repos }} - run: command: | - uv run --frozen gs codemod clone-repos --extra-repos=<> --token ${CODEGEN_BOT_GHE_TOKEN} --clean-cache + EXTRA_REPOS_ARG="" + if [ "<>" = "true" ]; then + EXTRA_REPOS_ARG="--extra-repos" + fi + uv run --frozen gs codemod clone-repos ${EXTRA_REPOS_ARG} --token ${CODEGEN_BOT_GHE_TOKEN} --clean-cache - save_cache: paths: - $GITHUB_WORKSPACE @@ -294,7 +298,7 @@ jobs: command: | EXTRA_REPOS_ARG="" if [ "<>" = "true" ]; then - EXTRA_REPOS_ARG="--extra-repos=true" + EXTRA_REPOS_ARG="--extra-repos" fi PYTEST_ARGS="${EXTRA_REPOS_ARG} --token ${CODEGEN_BOT_GHE_TOKEN} -o junit_suite_name=\"${CIRCLE_JOB}\" tests/codemod/test_parse.py" echo "Running tests with args: $PYTEST_ARGS" From 4305f78dedb4fd8eb5c501f198c2cf8a3cb4bc7b Mon Sep 17 00:00:00 2001 From: bagel897 Date: Wed, 22 Jan 2025 11:07:07 -0800 Subject: [PATCH 6/8] Update filters --- .circleci/config.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 283a611cb..478c245ba 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -431,13 +431,17 @@ workflows: only: develop extra_repos: false - oss-codemod-tests: + name: oss-codemod-tests context: - GHE - slack + filters: + branches: + only: develop matrix: parameters: sync_graph: [true, false] - size: [large] + size: [small, large] pr_checks: jobs: - unit-tests @@ -455,6 +459,9 @@ workflows: - pre-commit - doctests - oss-codemod-tests: + filters: + branches: + ignore: develop context: - GHE - slack From fe352ba075c9ffd21d19c515bdcf190fc6e4d28c Mon Sep 17 00:00:00 2001 From: bagel897 Date: Wed, 22 Jan 2025 11:11:28 -0800 Subject: [PATCH 7/8] Fix cache key --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 478c245ba..e061049a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,7 +54,7 @@ commands: steps: - restore_cache: keys: - - repos-{{ parameters.extra_repos }} + - repos-<> - run: command: | EXTRA_REPOS_ARG="" @@ -65,7 +65,7 @@ commands: - save_cache: paths: - $GITHUB_WORKSPACE - key: repos-{{ parameters.extra_repos }} + key: repos-<> fetch-verified-codemods: steps: - run: From d1bb0f8904c142f78c076459a65c9b1932426e65 Mon Sep 17 00:00:00 2001 From: bagel897 Date: Wed, 22 Jan 2025 11:12:01 -0800 Subject: [PATCH 8/8] Fix formatting --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e061049a8..af5bfacf6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -269,8 +269,8 @@ jobs: - run_ats: default_tests: "tests/codemod/test_codemods.py" codecov_flags: "smart-tests-codemod-oss" - collect_args: --size=${parameters.size} --sync-graph=${parameters.sync_graph} --token ${CODEGEN_BOT_GHE_TOKEN} - ats_collect_args: --size=${parameters.size},--sync-graph=${parameters.sync_graph},--token=${CODEGEN_BOT_GHE_TOKEN}, + collect_args: --size=<> --sync-graph=<> --token ${CODEGEN_BOT_GHE_TOKEN} + ats_collect_args: --size=<>,--sync-graph=<>,--token=${CODEGEN_BOT_GHE_TOKEN}, split_tests: false - slack/notify: event: fail