[9.1] Restore CI

.buildkite/it/run_serverless.sh

@@ -14,6 +14,7 @@ echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf
 
 PYTHON_VERSION="$1"
 TEST_NAME="$2"
+IFS=',' read -ra RUN_FULL_CI_WHEN_CHANGED <<< "$3"
 
 echo "--- System dependencies"
 
@@ -29,7 +30,29 @@ echo "--- Python modules"
 source .venv/bin/activate
 python -m pip install .[develop]
 
-echo "--- Run IT serverless test \"$TEST_NAME\" :pytest:"
+echo "--- Track filter modification"
 
-hatch -v -e it_serverless run $TEST_NAME
+CHANGED_FILES=$(git diff --name-only origin/master...HEAD)
+readarray -t changed_files_arr <<< "$CHANGED_FILES"
 
+CHANGED_TOP_LEVEL_DIRS=$(echo "$CHANGED_FILES" | grep '/' | awk -F/ '{print $1}' | sort -u | paste -sd, -)
+CHANGED_TOP_LEVEL_DIRS=${CHANGED_TOP_LEVEL_DIRS%,}
+IFS=',' read -ra changed_dirs_arr <<< "$CHANGED_TOP_LEVEL_DIRS"
+
+all_changed_arr=("${changed_files_arr[@]}" "${changed_dirs_arr[@]}")
+
+TRACK_FILTER_ARG="--track-filter=${CHANGED_TOP_LEVEL_DIRS}"
+
+# If any changes match one of the RUN_FULL_CI_WHEN_CHANGED paths, run full CI
+for static_path in "${RUN_FULL_CI_WHEN_CHANGED[@]}"; do
+    for changed in "${all_changed_arr[@]}"; do
+        if [[ "$static_path" == "$changed" ]]; then
+            echo "Matched '$static_path' in changed files/dirs. Running full CI."
+            TRACK_FILTER_ARG=""
+            break 2
+        fi
+    done
+done
+echo "--- Run IT serverless test \"$TEST_NAME\" $TRACK_FILTER_ARG :pytest:"
+
+hatch -v -e it_serverless run $TEST_NAME $TRACK_FILTER_ARG

.buildkite/it/serverless-pipeline.yml

@@ -1,3 +1,6 @@
+env:
+    RUN_FULL_CI_WHEN_CHANGED: pyproject.toml,.buildkite,it_tracks_serverless
+
 common:
   plugins:
     - elastic/vault-secrets#v0.0.2: &vault-base_url
@@ -19,14 +22,16 @@ agents:
 
 steps:
   - label: "Run IT serverless tests with user privileges"
+    if: build.pull_request.base_branch == "master" || build.source == "schedule"
     plugins:
       - elastic/vault-secrets#v0.0.2: *vault-base_url
       - elastic/vault-secrets#v0.0.2: *vault-get_credentials_endpoint
       - elastic/vault-secrets#v0.0.2: *vault-api_key
-    command: bash .buildkite/it/run_serverless.sh 3.11 test_user
+    command: bash .buildkite/it/run_serverless.sh 3.13 test_user $RUN_FULL_CI_WHEN_CHANGED
   - label: "Run IT Serverless tests with operator privileges"
+    if: build.pull_request.base_branch == "master" || build.source == "schedule"
     plugins:
       - elastic/vault-secrets#v0.0.2: *vault-base_url
       - elastic/vault-secrets#v0.0.2: *vault-get_credentials_endpoint
       - elastic/vault-secrets#v0.0.2: *vault-api_key
-    command: bash .buildkite/it/run_serverless.sh 3.11 test_operator
+    command: bash .buildkite/it/run_serverless.sh 3.13 test_operator $RUN_FULL_CI_WHEN_CHANGED

.github/workflows/backport.action.yml

@@ -0,0 +1,29 @@
+name: Automatic backport action
+
+on:
+  pull_request_target:
+    branches: ['master']
+    types: ["labeled", "closed"]
+
+jobs:
+  backport:
+    name: Backport PR
+    runs-on: ubuntu-latest
+    if: |
+        github.event.pull_request.merged == true
+        && !contains(github.event.pull_request.labels.*.name, 'backport')
+
+    steps:
+      - name: Backport Action
+        uses: sorenlouv/backport-github-action@ad888e978060bc1b2798690dd9d03c4036560947  # v9.5.1
+        continue-on-error: true
+        with:
+          github_token: ${{ secrets.BACKPORT_TOKEN }}
+
+      - name: Info log
+        if: ${{ success() }}
+        run: cat ~/.backport/backport.info.log
+
+      - name: Debug log
+        if: ${{ failure() }}
+        run: cat ~/.backport/backport.debug.log

.github/workflows/backport.reminder.yml

@@ -0,0 +1,42 @@
+name: Backport reminder
+
+on:
+  pull_request_target:
+    branches: [master]
+    types: [closed]
+  schedule:
+    - cron: '0 6 * * *'  # Every day at 06:00 UTC
+  workflow_dispatch:
+    inputs:
+      lookback_days:
+        description: 'How many days back to search merged PRs'
+        required: false
+        default: '7'
+      pending_label_age_days:
+        description: 'Minimum age in days before reminding'
+        required: false
+        default: '14'
+
+env:
+    BACKPORT_TOKEN: ${{ secrets.BACKPORT_TOKEN }}
+
+jobs:
+  reminder:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      issues: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Add Backport pending label (single PR)
+        if: github.event_name == 'pull_request_target'
+        run: |
+          python github_ci_tools/scripts/backport.py label --pr-mode
+          python github_ci_tools/scripts/backport.py remind --pr-mode --pending-reminder-age-days ${{ github.event.inputs.pending_label_age_days }}
+      - name: Add Backport pending label (bulk)
+        if: github.event_name != 'pull_request_target'
+        run: |
+          python github_ci_tools/scripts/backport.py label --lookback-days ${{ github.event.inputs.lookback_days }}
+          python github_ci_tools/scripts/backport.py remind --lookback-days ${{ github.event.inputs.lookback_days }} --pending-reminder-age-days ${{ github.event.inputs.pending_label_age_days }}

.github/workflows/ci.yml

@@ -2,6 +2,7 @@ name: CI
 
 on:
   push:
+    branches: [ master, '[0-9]*' ]
   pull_request:
   schedule:
     - cron: '0 14 * * *'
@@ -10,6 +11,8 @@ on:
 
 env:
   DEFAULT_BRANCH: master
+  # comma-separated list of paths that if changed will trigger a full CI run (Note: don't use trailing '/' at the end)
+  RUN_FULL_CI_WHEN_CHANGED: 'pyproject.toml,.github,it_tracks'
 
 permissions: "read-all"
 
@@ -21,7 +24,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.13"
           cache: pip
           cache-dependency-path: pyproject.toml
       - name: "Install dependencies"
@@ -34,17 +37,80 @@ jobs:
           slack_channel: ${{ secrets.SLACK_CHANNEL }}
           status: FAILED
 
+  determine-es-build:
+    runs-on: ubuntu-22.04
+    outputs:
+      revision: ${{ steps.revision-argument.outputs.revision }}
+      release_build: ${{ steps.release-build-argument.outputs.release_build }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: "Determine ES version"
+        id: es-version
+        run: |
+          ES_VERSION=$(cat es-version)
+          echo "Determined es-version: $ES_VERSION"
+          echo "version=$ES_VERSION" >> $GITHUB_OUTPUT
+      - name: "Determine --revision argument"
+        id: revision-argument
+        run: |
+          echo "revision= --revision=${{ steps.es-version.outputs.version }}" >> $GITHUB_OUTPUT
+      - name: "Determine ES release or snapshot"
+        id: release-build-argument
+        run: |
+          if [[ "${{ steps.es-version.outputs.version }}" == "current" || "${{ steps.es-version.outputs.version }}" == "latest" ]]; then
+            echo "release_build=" >> $GITHUB_OUTPUT
+          else
+            echo "release_build= --source-build-release" >> $GITHUB_OUTPUT
+          fi
+      - name: "Show revision argument"
+        if: ${{ steps.revision-argument.outputs.revision != '' }}
+        run: echo "Using${{ steps.revision-argument.outputs.revision }}"
+      - name: "Show release build argument"
+        if: ${{ steps.release-build-argument.outputs.release_build != '' }}
+        run: echo "Using${{ steps.release-build-argument.outputs.release_build }}"
+
+  filter-pr-changes:
+    runs-on: ubuntu-22.04
+    outputs:
+      track_filter: ${{ steps.track-filter.outputs.track_filter }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Parse repo and create filters.yml
+        run: python3 github_ci_tools/scripts/track-filter.py
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 #v3.0.2
+        id: changes
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          filters: github_ci_tools/filters.yml
+      - name: Collect changed tracks and calculate --track-filter argument
+        id: track-filter
+        run: |
+            TRACKS=$(echo '${{ toJSON(steps.changes.outputs) }}' | jq -r '
+            to_entries
+            | map(select(.value == "true"))
+            | map(.key)
+            | join(",")
+            ')
+            if echo "$TRACKS" | grep -qw "full_ci"; then
+              echo 'track_filter=' >> $GITHUB_OUTPUT
+            else
+              echo "track_filter= --track-filter=$TRACKS" >> $GITHUB_OUTPUT
+            fi
+      - name: show track filter argument
+        if: ${{ steps.track-filter.outputs.track_filter != '' }}
+        run: echo "Using${{ steps.track-filter.outputs.track_filter }}"
+
   test:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
         os:
-          - macos-13
+          - macos-latest
           - ubuntu-latest
 
     runs-on: ${{ matrix.os }}
-    name: unit ${{ fromJson('{"macos-13":"macOS","ubuntu-latest":"Ubuntu"}')[matrix.os] }} ${{ matrix.python-version }}
+    name: unit ${{ fromJson('{"macos-latest":"macOS","ubuntu-latest":"Ubuntu"}')[matrix.os] }} ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -64,15 +130,22 @@ jobs:
           status: FAILED
 
   rally-tracks-compat:
+    needs: 
+      - filter-pr-changes
+      - determine-es-build
+
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.12"]
+        python-version: ["3.10", "3.13"]
 
     runs-on: ubuntu-22.04
 
     name: rally-tracks-compat ${{ matrix.python-version }}
     steps:
+      - name: Check public IP address
+        run: curl -4s ifconfig.me
+        continue-on-error: true
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
@@ -85,15 +158,30 @@ jobs:
           java-version: "21"
       - run: echo "JAVA21_HOME=$JAVA_HOME_21_X64" >> $GITHUB_ENV
       - run: echo "JAVA11_HOME=$JAVA_HOME_11_X64" >> $GITHUB_ENV
+      - name: Free Disk Space
+        continue-on-error: true
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
+        with:
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          docker-images: false
+          swap-storage: false
+          tool-cache: false
+      - name: Check disk space before
+        run: df -h
       - name: "Install dependencies"
         run: python -m pip install .[develop]
-      - name: "Run tests"
-        run: hatch -v -e it run test
-        timeout-minutes: 120
+      - name: "Run tests${{ needs.filter-pr-changes.outputs.track_filter }}${{ needs.determine-es-build.outputs.revision }}${{ needs.determine-es-build.outputs.release_build }}"
+        run: hatch -v -e it run test${{ needs.filter-pr-changes.outputs.track_filter }}${{ needs.determine-es-build.outputs.revision }}${{ needs.determine-es-build.outputs.release_build }}
+        timeout-minutes: 160
         env:
           # elastic/endpoint fetches assets from GitHub, authenticate to avoid
           # being rate limited
           ASSETS_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check disk space after
+        run: df -h
       - uses: elastic/es-perf-github-status@v2
         if: ${{ failure() && ( github.event_name == 'schedule' || ( github.event_name == 'push' && github.ref_name == env.DEFAULT_BRANCH ) ) }}
         with:

.gitignore

@@ -99,6 +99,9 @@ target/
 #Pickles
 *.pk
 
+# direnv
+.envrc
+
 # pyenv
 .python-version
 

Makefile

@@ -21,7 +21,7 @@ PYENV_REGEX = .pyenv/shims
 PY_BIN = python3
 # https://github.com/pypa/pip/issues/5599
 PIP_WRAPPER = $(PY_BIN) -m pip
-export PY38 = "3.8.13"
+export PY313 = "3.13.7"
 VIRTUAL_ENV ?= .venv
 VENV_ACTIVATE_FILE = $(VIRTUAL_ENV)/bin/activate
 VENV_ACTIVATE = . $(VENV_ACTIVATE_FILE)
@@ -31,8 +31,8 @@ PYENV_PREREQ_HELP = "\033[0;31mIMPORTANT\033[0m: please type \033[0;31mpyenv ini
 VE_MISSING_HELP = "\033[0;31mIMPORTANT\033[0m: Couldn't find $(PWD)/$(VIRTUAL_ENV); have you executed make venv-create?\033[0m\n"
 
 prereq:
-	pyenv install --skip-existing $(PY38)
-	pyenv local $(PY38)
+	pyenv install --skip-existing $(PY313)
+	pyenv local $(PY313)
 
 venv-create:
 	@if [[ ! -x $$(command -v pyenv) ]]; then \

elastic/Makefile

@@ -59,7 +59,7 @@ install: venv-create
 	# install pytest for tests
 	. $(VENV_ACTIVATE_FILE); pip3 install pytest==6.2.5 pytest-benchmark==3.2.2
 	# install dependencies for tests
-	. $(VENV_ACTIVATE_FILE); pip3 install geneve==0.0.3 pytest-asyncio==0.18.1 git+https://github.com/elastic/package-assets.git
+	. $(VENV_ACTIVATE_FILE); pip3 install geneve==0.3.0 pytest-asyncio==0.18.1 git+https://github.com/elastic/package-assets.git
 	# install (latest) Rally for smoke tests
 	. $(VENV_ACTIVATE_FILE); pip3 install git+ssh://git@github.com/elastic/rally.git --use-feature=2020-resolver
 

elastic/security/README.md

@@ -148,7 +148,7 @@ This challenge executes indexing and querying sequentially. Queries will be issu
 
 ### Generate source events for detection rules (generate-alerts-source-events)
 
-This challenge is a demo usage of [Geneve](https://github.com/elastic/geneve) via the `events-emitter-source` [parameter source](https://github.com/elastic/rally-tracks/blob/master/elastic/security/parameter_sources/events_emitter.py), it generates source events but does not interact with anything else. It's executed as part of the [it/test_security.py](https://github.com/elastic/rally-tracks/blob/master/it/test_security.py) integration tests. Currently, Geneve is pinned to version [v0.2.0](https://github.com/elastic/rally-tracks/blob/master/elastic/security/track.json#L410). This is the only challenge that depends on Geneve and pyyaml (Geneve requires pyyaml).
+This challenge is a demo usage of [Geneve](https://github.com/elastic/geneve) via the `events-emitter-source` [parameter source](https://github.com/elastic/rally-tracks/blob/master/elastic/security/parameter_sources/events_emitter.py), it generates source events but does not interact with anything else. It's executed as part of the [it/test_security.py](https://github.com/elastic/rally-tracks/blob/master/it/test_security.py) integration tests. Currently, Geneve is pinned to version [v0.3.0](https://github.com/elastic/rally-tracks/blob/master/elastic/security/track.json#L410). This is the only challenge that depends on Geneve and pyyaml (Geneve requires pyyaml).
 
 ## Ratios
 

elastic/security/track.json

@@ -352,7 +352,7 @@
 {% endfor %}
   ],
   "dependencies": [
-    "geneve==0.2.0",
+    "geneve==0.3.0",
     "pyyaml",
     "elastic-transport==8.4.1",
     "elasticsearch==8.6.1"

elastic/shared/parameter_sources/__init__.py

@@ -6,7 +6,7 @@
 DEFAULT_MAX_DATE = "2020-01-01"
 
 # this provides a universal start date for `now` if we are using it as the current time
-now = datetime.utcnow().replace(tzinfo=timezone.utc)
+now = datetime.now(tz=timezone.utc)
 
 
 def utc_now():

elastic/shared/parameter_sources/processed.py

@@ -79,7 +79,7 @@ def __init__(self, track, params, **kwargs):
         self._volume_per_day_gb = convert_to_gib(raw_volume_per_day)
         self.start_time = int(time.perf_counter())
         self._profile = params.get("profile", "fixed_interval")
-        now = datetime.utcnow().replace(tzinfo=timezone.utc)
+        now = datetime.now(tz=timezone.utc)
 
         def utc_now():
             return now