Create integration test for rally and many tracks.

.cursor/rules/run-tests.mdc

@@ -0,0 +1,15 @@
+---
+description: Run tests via make and always outside the sandbox
+alwaysApply: true
+---
+
+# Running Tests
+
+When running tests in this project:
+
+1. **Use the Make command** — Do not invoke pytest (or uv run pytest) directly. Use the project's Make targets instead:
+   - **Unit tests:** `make test` (or `make test-3.10`, etc. for a specific Python version)
+   - **Integration tests:** `make it`
+   - **With options:** Pass pytest options via `OPTS`, e.g. `make it OPTS="-x --ff"` for fail-fast and run failed tests first.
+
+2. **Run outside the sandbox** — Test commands must be executed with full permissions (e.g. request `all` or disable sandbox) so that Docker, network, and file access work correctly. Do not run test commands inside a restricted sandbox.

.github/workflows/ci.yml

@@ -154,3 +154,58 @@ jobs:
           slack_channel: ${{ secrets.SLACK_CHANNEL }}
           status: FAILED
           persist-credentials: false
+
+
+  test-tracks:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Check public IP address
+        run: curl -4s ifconfig.me
+        continue-on-error: true
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.13"
+      - name: Install UV
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.8.22"
+      - name: Free Disk Space
+        continue-on-error: true
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
+        with:
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          docker-images: false
+          swap-storage: false
+          tool-cache: false
+      - name: Check disk space before
+        run: df -h
+      - name: Run tests
+        run: make it_tracks
+        timeout-minutes: 160
+        env:
+          # elastic/endpoint fetches assets from GitHub, authenticate to avoid
+          # being rate limited
+          ASSETS_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check disk space after
+        run: df -h
+      - uses: elastic/es-perf-github-status@v2
+        if: ${{ failure() && ( github.event_name == 'schedule' || ( github.event_name == 'push' && github.ref_name == env.DEFAULT_BRANCH ) ) }}
+        with:
+          slack_bot_token: ${{ secrets.SLACK_BOT_TOKEN }}
+          slack_channel: ${{ secrets.SLACK_CHANNEL }}
+          status: FAILED
+      # Artifact will show up under "Artifacts" in the "Summary" page of runs
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: rally-tracks-logs
+          path: |
+            /home/runner/.rally/logs/
+            /home/runner/.rally/benchmarks/races/**/*.log
+          if-no-files-found: error
+          persist-credentials: false

Dockerfile

@@ -0,0 +1,15 @@
+FROM elastic/rally:2.12.0 AS rally
+
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+
+COPY pyproject.toml /src/pyproject.toml
+COPY esrally/_version.py /src/esrally/_version.py
+COPY esrally/__init__.py /src/esrally/__init__.py
+COPY README.md /src/README.md
+RUN uv pip install --upgrade /src/
+
+COPY . /src
+RUN uv pip install --upgrade /src/
+
+ENTRYPOINT ["esrally"]
+CMD ["--help"]

Makefile

@@ -17,18 +17,27 @@
 
 SHELL := /bin/bash
 
+DEFAULT_PY_VERSION = $(shell jq -r '.python_versions.DEFAULT_PY_VER' .ci/variables.json)
+
+
 # We assume an active virtualenv for development
-VIRTUAL_ENV := $(or $(VIRTUAL_ENV),.venv$(if $(PY_VERSION),-$(PY_VERSION)))
-VENV_ACTIVATE_FILE := $(VIRTUAL_ENV)/bin/activate
+
+VENV_DIR = $(if $(PY_VERSION),.venv-$(PY_VERSION),.venv)
+
+VENV_ACTIVATE_FILE := $(VENV_DIR)/bin/activate
 VENV_ACTIVATE := source $(VENV_ACTIVATE_FILE)
 
-PY_VERSION := $(shell jq -r '.python_versions.DEFAULT_PY_VER' .ci/variables.json)
-export UV_PYTHON := $(PY_VERSION)
-export UV_PROJECT_ENVIRONMENT := $(VIRTUAL_ENV)
+export UV_PYTHON = $(or $(PY_VERSION),$(DEFAULT_PY_VERSION))
+
+export UV_PROJECT_ENVIRONMENT = $(VENV_DIR)
 
 PRE_COMMIT_HOOK_PATH := .git/hooks/pre-commit
 
-LOG_CI_LEVEL := INFO
+TESTS = tests/
+LOG_LEVEL =
+OPTS = $(if $(LOG_LEVEL), --log-cli-level=$(LOG_LEVEL),)
+RUN_TESTS ?= uv run -- pytest $(OPTS) $(TESTS)
+
 
 # --- Global goals ---
 
@@ -85,9 +94,7 @@ uv:
 
 # It adds a list of packages to the project.
 uv-add:
-ifndef ARGS
-	$(error Missing arguments. Use make uv-add ARGS="...")
-endif
+	$(if args,,$(error Missing arguments. Use make uv-add ARGS="..."))
 	uv add $$ARGS
 
 # It updates the uv lock file.
@@ -106,11 +113,11 @@ $(VENV_ACTIVATE_FILE):
 
 # It delete the project virtual environment from disk.
 clean-venv:
-	rm -fR '$(VIRTUAL_ENV)'
+	rm -fR '$(VENV_DIR)'
 
 # It installs the Rally PyTest plugin.
 install_pytest_rally_plugin: venv
-	$(VENV_ACTIVATE); uv pip install 'pytest-rally @ git+https://github.com/elastic/pytest-rally.git'
+	$(VENV_ACTIVATE); pip install 'pytest-rally @ git+https://github.com/elastic/pytest-rally.git'
 
 # Old legacy alias goals
 install: venv
@@ -140,7 +147,7 @@ lint: venv
 	uv run -- pre-commit run --all-files
 
 # It run all linters on changed files using pre-commit.
-precommit pre-commit: venv
+pre-commit: venv
 	uv run -- pre-commit run
 
 # It install a pre-commit hook in the project .git dir so modified files are checked before creating every commit.
@@ -174,45 +181,65 @@ clean-docs: venv
 
 # It runs unit tests using the default python interpreter version.
 test: venv
-	uv run -- pytest -s $(or $(ARGS), tests/)
+	$(RUN_TESTS)
 
 # It runs unit tests using all supported python versions.
 test-all: test-3.10 test-3.11 test-3.12 test-3.13
 
 # It runs unit tests using Python 3.10.
-test-3.10:
-	$(MAKE) test PY_VERSION=3.10
+test-3.10: PY_VERSION = 3.10
+test-3.10: venv
+	$(RUN_TESTS)
 
 # It runs unit tests using Python 3.11.
+test-3.11: PY_VERSION = 3.11
 test-3.11:
-	$(MAKE) test PY_VERSION=3.11
+	$(RUN_TESTS)
 
 # It runs unit tests using Python 3.12.
+test-3.12: PY_VERSION = 3.12
 test-3.12:
-	$(MAKE) test PY_VERSION=3.12
+	$(RUN_TESTS)
 
 # It runs unit tests using Python 3.13.
+test-3.13: PY_VERSION = 3.13
 test-3.13:
-	$(MAKE) test PY_VERSION=3.13
+	$(RUN_TESTS)
+
+# It runs unit tests using Python 3.14.
+test-3.14: PY_VERSION = 3.14
+test-3.14:
+	$(RUN_TESTS)
 
 
 # --- Integration tests goals ---
 
 # It runs integration tests.
+it: TESTS = it/
 it: venv
-	$(MAKE) test ARGS=$(or $(ARGS),it/)
+	$(RUN_TESTS)
 
-# It runs serverless integration tests.
+# It runs track_repo_compatibility integration tests for serverless tracks.
+it_serverless: TESTS = it/track_repo_compatibility
+it_serverless: OPTS = --track-repository-test-directory=it_tracks_serverless
 it_serverless: install_pytest_rally_plugin
-	uv run -- pytest -s --log-cli-level=$(LOG_CI_LEVEL) --track-repository-test-directory=it_tracks_serverless it/track_repo_compatibility $(ARGS)
+	$(RUN_TESTS)
 
-# It runs rally_tracks_compat integration tests.
+# It runs track_repo_compatibility integration tests.
+it_tracks_compat: TESTS = it/track_repo_compatibility
 it_tracks_compat: install_pytest_rally_plugin
-	uv run -- pytest -s --log-cli-level=$(LOG_CI_LEVEL) it/track_repo_compatibility $(ARGS)
+	$(RUN_TESTS)
+
+# It runs rally_tracks integration tests.
+it_tracks: TESTS = it/tracks_test.py
+it_tracks: install_pytest_rally_plugin
+	$(RUN_TESTS)
 
 # It runs benchmark tests.
+benchmark: TESTS = benchmarks/
 benchmark: venv
-	$(MAKE) test ARGS=benchmarks/
+	$(RUN_TESTS)
+
 
 # --- Release goals ---
 

compose.yaml

@@ -0,0 +1,47 @@
+services:
+
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION:-9.3.1}
+    container_name: es01
+    networks:
+      - elastic-net
+    environment:
+      - node.name=es01
+      - discovery.type=single-node
+      - xpack.security.enabled=false # Disabled for simpler benchmarking setup
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - es_data:/usr/share/elasticsearch-${ES_VERSION:-latest}/data
+    healthcheck:
+      test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"\\|\"status\":\"yellow\"'"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  rally:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: rally-driver
+    depends_on:
+      es01:
+        condition: service_healthy
+    volumes:
+      - rally_data:/rally/.rally
+    # The command below lists tracks by default so the container doesn't exit immediately.
+    # You will likely override this via CLI to run actual benchmarks.
+    # command: "esrally list tracks"
+    networks:
+      - elastic-net
+
+networks:
+  elastic-net:
+    driver: bridge
+
+volumes:
+  es_data:
+  rally_data:

esrally/client/factory.py

@@ -320,7 +320,10 @@ def wait_for_rest_layer(es, max_attempts=40):
         except TlsError as e:
             raise exceptions.SystemSetupError("Could not connect to cluster via HTTPS. Are you sure this is an HTTPS endpoint?", e)
         except ConnectionError as e:
-            if "ProtocolError" in str(e):
+            err_str = str(e)
+            # Only treat as permanent scheme error when the message clearly indicates HTTP/HTTPS mismatch.
+            # Connection reset / connection aborted (e.g. REST not ready yet) are transient and should be retried.
+            if "ProtocolError" in err_str and ("HTTPS" in err_str or "HTTP request to an HTTPS" in err_str):
                 raise exceptions.SystemSetupError(
                     "Received a protocol error. Are you sure you're using the correct scheme (HTTP or HTTPS)?", e
                 )

esrally/log.py

@@ -170,7 +170,6 @@ def install_default_log_config():
     io.ensure_dir(paths.logs())
 
 
-# pylint: disable=unused-argument
 def configure_file_handler(*, filename: str, encoding: str = "UTF-8", delay: bool = False, **kwargs: Any) -> logging.Handler:
     """
     Configures the WatchedFileHandler supporting expansion of `~` and `${LOG_PATH}` to the user's home and the log path respectively.