Switch from docker-compose to docker compose command.

CONTRIBUTING.md

@@ -60,7 +60,7 @@ Once your changes and tests are ready to submit for review:
 
     Ensure that all tests pass by running `make check-all`. This runs sequentially lint checks, unit tests and integration tests. These can be executed in isolation using `make lint`, `make test` and `make it` respectively, in case you need to iterate over a subset of tests.
 
-    Note: Integration tests are much slower than unit tests and require `docker-compose`.
+    Note: Integration tests are much slower than unit tests.
 
 3. Sign the Contributor License Agreement
 

docker/docker-compose-tests.yml

@@ -1,33 +1,51 @@
-version: '2.2'
 services:
+
+  # With both `image` and `build`, Compose uses the image name/tag from `image` and builds from `build` when the
+  # image is absent locally. IT builds the dev image before `compose run`; release-docker-test.sh builds first and sets
+  # RALLY_DOCKER_IMAGE / RALLY_VERSION_TAG so `compose run` uses that image (Compose pulls or finds it locally; it does
+  # not rebuild over a correctly tagged image). Keeping `build` is a convenience for ad-hoc `docker compose run`
+  # without a separate `docker compose build`.
   rally:
-    image: ${RALLY_DOCKER_IMAGE}:${RALLY_VERSION_TAG}
+    image: ${RALLY_DOCKER_IMAGE:-elastic/rally}:${RALLY_VERSION_TAG:-dev}
+    build:
+      context: ../
+      dockerfile: ${RALLY_DOCKER_FILE:-docker/Dockerfiles/dev/Dockerfile}
     container_name: rally
-    command: ${TEST_COMMAND}
+    command: ${TEST_COMMAND:---help}
     volumes:
       - rally:/rally/.rally
     networks:
       - esnet
     depends_on:
       es01:
         condition: service_healthy
+
   es01:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
     container_name: es01
     environment:
       - discovery.type=single-node
       - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+      # Elasticsearch 8+ Docker images enable TLS/enrollment by default; 7.x does not. When you move this service to 8+,
+      # uncomment the two lines below for unsecured local/integration use, or keep security on and trust the HTTP CA
+      # that the image generates (copy from the container, e.g. `docker cp es01:/usr/share/elasticsearch/config/certs/http_ca.crt .`
+      # — path varies by version; see Elasticsearch "Install Elasticsearch with Docker" docs for the current layout).
+      # - xpack.security.enabled=false
+      # - xpack.security.enrollment.enabled=false
     volumes:
       - esdata1:/usr/share/elasticsearch/data
-    ports:
-      - 19200:9200
+    # Uncomment `ports` when you need HTTP access from the host (debugging, curl from the machine running Compose).
+    # Pick a host port that does not conflict with another local Elasticsearch instance.
+    # ports:
+    #   - 19200:9200
     networks:
       - esnet
     healthcheck:
-          test: curl -f http://localhost:9200
+          test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -vq '\"status\":\"red\"'"]
           interval: 5s
           timeout: 2s
-          retries: 10
+          retries: 30
+
 networks:
   esnet:
     name: rally-tests

docs/developing.rst

@@ -8,7 +8,7 @@ Install the following software packages:
 
 * `uv <https://docs.astral.sh/uv/getting-started/installation/>`_ 
 * JDK version required to build Elasticsearch. Please refer to the `build setup requirements <https://github.com/elastic/elasticsearch/blob/main/CONTRIBUTING.md#contributing-to-the-elasticsearch-codebase>`_.
-* `Docker <https://docs.docker.com/install/>`_ and on Linux additionally `docker-compose <https://docs.docker.com/compose/install/>`_.
+* `Docker <https://docs.docker.com/install/>`_ and on Linux additionally `docker compose <https://docs.docker.com/compose/install/>`_.
 * `jq <https://stedolan.github.io/jq/download/>`_
 * git
 

esrally/mechanic/launcher.py

@@ -65,7 +65,7 @@ def _start_process(self, binary_path):
         self._wait_for_healthy_running_container(container_id, DockerLauncher.PROCESS_WAIT_TIMEOUT_SECONDS)
 
     def _docker_compose(self, compose_config, cmd):
-        return "docker-compose -f {} {}".format(os.path.join(compose_config, "docker-compose.yml"), cmd)
+        return "docker compose -f {} {}".format(os.path.join(compose_config, "docker-compose.yml"), cmd)
 
     def _get_container_id(self, compose_config):
         compose_ps_cmd = self._docker_compose(compose_config, "ps -q")

it/conftest.py

@@ -30,7 +30,7 @@ def check_prerequisites():
     print("Checking prerequisites...")
     if process.run_subprocess_with_logging("docker ps") != 0:
         raise AssertionError("Docker must be installed and the daemon must be up and running to run integration tests.")
-    if process.run_subprocess_with_logging("docker-compose --help") != 0:
+    if process.run_subprocess_with_logging("docker compose version") != 0:
         raise AssertionError("Docker Compose is required to run integration tests.")
 
 

it/docker_dev_image_test.py

@@ -14,61 +14,206 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
+import dataclasses
+import logging
 import os
+import re
+import subprocess
+from collections.abc import Callable, Generator, Mapping
+
+import pytest
 
 import it
 from esrally import version
-from esrally.utils import process
+from esrally.utils import cases
 
+LOG = logging.getLogger(__name__)
 
-def test_docker_geonames():
-    test_command = (
-        "race --pipeline=benchmark-only --test-mode --track=geonames --challenge=append-no-conflicts-index-only --target-hosts=es01:9200"
-    )
-    run_docker_compose_test(test_command)
+# Compose file used by both this test module and release-docker-test.sh.
+COMPOSE_FILE = os.path.join(it.ROOT_DIR, "docker", "docker-compose-tests.yml")
 
+# Start of Rally's tabular output; everything before this (e.g. ASCII banner) is ignored when parsing.
+_AVAILABLE_TRACKS_MARKER = "Available tracks:"
 
-def test_docker_list_tracks():
-    test_command = "list tracks"
-    run_docker_compose_test(test_command)
+# Shared race invocation against the ES service defined in docker-compose-tests.yml (es01:9200).
+_RACE_FLAGS = (
+    "race --pipeline=benchmark-only --test-mode --track=geonames --challenge=append-no-conflicts-index-only --target-hosts=es01:9200"
+)
 
 
-def test_docker_help():
-    test_command = "--help"
-    run_docker_compose_test(test_command)
+def _docker_compose_process_env(compose_env: Mapping[str, str]) -> dict[str, str]:
+    """Merge parent env with compose_env and set defaults for every var interpolated in docker-compose-tests.yml.
 
+    Avoids Docker Compose warnings (e.g. \"variable is not set\") on ``down`` / ``logs`` when ``TEST_COMMAND`` was
+    only defined for a prior ``run``.
+    """
+    env = os.environ.copy()
+    env.update(compose_env)
+    env.setdefault("TEST_COMMAND", "--help")
+    env.setdefault("RALLY_DOCKER_FILE", "docker/Dockerfiles/dev/Dockerfile")
+    return env
 
-def test_docker_override_cmd():
-    test_command = (
-        "esrally race --pipeline=benchmark-only --test-mode --track=geonames "
-        "--challenge=append-no-conflicts-index-only --target-hosts=es01:9200"
-    )
-    run_docker_compose_test(test_command)
 
+def tear_down_stack(compose_env: Mapping[str, str]) -> None:
+    """Stop and remove the compose project so the next run starts from a clean stack."""
+    LOG.info("Tearing down docker stack... (compose_env=%s)", compose_env)
+    env = _docker_compose_process_env(compose_env)
 
-def run_docker_compose_test(test_command):
     try:
-        if run_docker_compose_up(test_command) != 0:
-            raise AssertionError(f"The docker-compose test failed with test command: {test_command}")
+        subprocess.run(
+            f"docker compose -f '{COMPOSE_FILE}' down -v",
+            env=env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            check=True,
+            shell=True,
+        )
+    except subprocess.CalledProcessError as err:
+        msg = (
+            "Docker compose down failed:\n"
+            f"  - command: '{err.cmd}'\n"
+            f"  - args: {err.args}\n"
+            f"  - return code: {err.returncode}\n"
+            f"  - output:\n{err.stdout}\n"
+        )
+        LOG.error(msg)
+        pytest.fail(msg)
+    else:
+        LOG.debug("Compose stack is down (compose_env=%s).", compose_env)
+
+
+@pytest.fixture(scope="module")
+def compose_env() -> Generator[Mapping[str, str]]:
+    """Env vars passed to ``docker compose`` so the rally image tag matches the workspace Rally version."""
+    env = {
+        "RALLY_VERSION": version.__version__,
+        "RALLY_VERSION_TAG": version.__version__,
+        "RALLY_DOCKER_IMAGE": os.environ.get("RALLY_DOCKER_IMAGE", "elastic/rally"),
+        # Isolate project name per test module to avoid clashing with other compose runs on the same host.
+        "COMPOSE_PROJECT_NAME": __name__.replace(".", "_"),
+    }
+    tear_down_stack(env)
+    try:
+        yield env
     finally:
-        # Always ensure proper cleanup regardless of results
-        run_docker_compose_down()
-
-
-def run_docker_compose_up(test_command):
-    env_variables = os.environ.copy()
-    env_variables["TEST_COMMAND"] = test_command
-    env_variables["RALLY_DOCKER_IMAGE"] = "elastic/rally"
-    env_variables["RALLY_VERSION"] = version.__version__
-    env_variables["RALLY_VERSION_TAG"] = version.__version__
-
-    return process.run_subprocess_with_logging(
-        f"docker-compose -f {it.ROOT_DIR}/docker/docker-compose-tests.yml up --abort-on-container-exit",
-        env=env_variables,
-    )
-
-
-def run_docker_compose_down():
-    if process.run_subprocess_with_logging(f"docker-compose -f {it.ROOT_DIR}/docker/docker-compose-tests.yml down -v") != 0:
-        raise AssertionError("Failed to stop running containers from docker-compose-tests.yml")
+        if LOG.isEnabledFor(logging.DEBUG):
+            logs = subprocess.run(
+                f"docker compose -f '{COMPOSE_FILE}' logs -t",
+                capture_output=True,
+                shell=True,
+                check=False,
+                env=_docker_compose_process_env(env),
+            )
+            LOG.debug("Containers logs:\n%s", logs.stdout.decode("utf-8"))
+        tear_down_stack(env)
+
+
+def assert_list_tracks_contains(expected_track_names: list[str]) -> Callable[[str], None]:
+    """Return a ``want_stdout`` callable that parses ``list tracks`` stdout and requires every expected track name."""
+    expected = frozenset(expected_track_names)
+
+    def check_stdout(stdout: str) -> None:
+        actual = set(track_names_from_list_tracks_stdout(stdout))
+        missing = expected - actual
+        if missing:
+            sample = sorted(actual)[:15]
+            pytest.fail(
+                "list tracks output missing expected track name(s) "
+                f"{sorted(missing)!r}; parsed had {len(actual)} name(s); sample: {sample!r}"
+            )
+
+    return check_stdout
+
+
+def track_names_from_list_tracks_stdout(stdout: str) -> list[str]:
+    """Parse ``list tracks`` tabular stdout and return track names sorted for stable comparison."""
+    text = stdout.replace("\r\n", "\n")
+    idx = text.find(_AVAILABLE_TRACKS_MARKER)
+    if idx == -1:
+        pytest.fail("stdout did not contain the list-tracks marker " f"{_AVAILABLE_TRACKS_MARKER!r}; head:\n{text[:800]!r}")
+    body = text[idx:]
+    success_idx = body.find("\n[INFO] SUCCESS")
+    if success_idx != -1:
+        body = body[:success_idx]
+    lines = body.splitlines()
+    header_i = 0
+    while header_i < len(lines):
+        s = lines[header_i].strip()
+        if s.startswith("Name") and "Description" in s:
+            break
+        header_i += 1
+    else:
+        pytest.fail("list tracks stdout did not contain the expected table header after " f"{_AVAILABLE_TRACKS_MARKER!r}")
+    sep_i = header_i + 1
+    if sep_i >= len(lines) or "-" not in lines[sep_i]:
+        pytest.fail("list tracks stdout did not contain a table separator after the header")
+    names: list[str] = []
+    for line in lines[sep_i + 1 :]:
+        stripped = line.rstrip()
+        if not stripped.strip():
+            continue
+        if stripped.strip().replace("-", "") == "":
+            break
+        parts = re.split(r"\s{2,}", stripped.lstrip(), maxsplit=1)
+        if parts[0].strip():
+            names.append(parts[0].strip())
+    return sorted(names)
+
+
+@dataclasses.dataclass
+class ComposeCase:
+    # Passed as TEST_COMMAND to the rally service (see docker-compose-tests.yml).
+    command: str
+    want_return_code: int = 0
+    # Exact string match, or a callable that validates ``result.stdout``.
+    want_stdout: str | Callable[[str], None] | None = None
+    want_stderr: str | None = None
+
+
+@cases.cases(
+    help=ComposeCase("--help"),
+    list_tracks=ComposeCase(
+        "list tracks",
+        want_stdout=assert_list_tracks_contains(["elastic/logs", "geonames", "http_logs", "nyc_taxis", "so_vector"]),
+    ),
+    race=ComposeCase(_RACE_FLAGS),
+    race_explicit_esrally=ComposeCase(f"esrally {_RACE_FLAGS}"),
+)
+def test_docker_compose(
+    case: ComposeCase,
+    compose_env: Mapping[str, str],
+) -> None:
+    """Run one rally subcommand inside the compose-defined rally container (with ES dependency)."""
+    LOG.info("Running rally with 'docker compose', command='%s', env=%r", case.command, compose_env)
+    env = _docker_compose_process_env(compose_env)
+    env["TEST_COMMAND"] = case.command
+    try:
+        # ``run`` starts dependencies (es01), runs the rally service once, then tears down that one-off container.
+        result = subprocess.run(
+            f"docker compose -f '{COMPOSE_FILE}' run --remove-orphans rally",
+            env=env,
+            capture_output=True,
+            text=True,
+            check=case.want_return_code == 0,
+            shell=True,
+        )
+    except subprocess.CalledProcessError as err:
+        pytest.fail(
+            "Docker compose run failed:\n"
+            f"  - command: {err.cmd}\n"
+            f"  - args: {err.args}\n"
+            f"  - return code: {err.returncode}\n"
+            f"  - stdout:\n{err.stdout}\n"
+            f"  - stderr:\n{err.stderr}\n"
+        )
+
+    LOG.debug("Docker compose up succeeded. STDOUT:\n%s", result.stdout)
+    assert result.returncode == case.want_return_code
+    if case.want_stdout is not None:
+        if callable(case.want_stdout):
+            case.want_stdout(result.stdout)
+        else:
+            assert result.stdout == case.want_stdout
+    if case.want_stderr is not None:
+        assert result.stderr == case.want_stderr

pyproject.toml

@@ -49,7 +49,8 @@ dependencies = [
     # License: Apache 2.0
     "aiohttp==3.13.3",
     "aiosignal==1.4.0",
-    "docker==6.0.0",
+    # License: Apache 2.0 (docker-py)
+    "docker==7.1.0",
     # avoid specific requests version to fix bug in docker-py
     "requests<2.32.0",
     # License: BSD
@@ -92,8 +93,8 @@ dependencies = [
     "hatch==1.3.1",
     "hatchling==1.6.0",
     "wheel==0.46.2",
-    # License: MIT — Rally runs `python -m pip install` to install track dependencies (see esrally.track.loader).
-    "pip>=24.0",
+    # License: MIT
+    "pip==26.0.1",
 ]
 
 [project.optional-dependencies]

recipes/ccr/start.sh

@@ -4,10 +4,10 @@ set -e
 source .elastic-version
 
 # Start metrics store
-docker-compose -f ./metricstore-docker-compose.yml up -d 
+docker compose -f ./metricstore-docker-compose.yml up -d
 
 # Start Elasticsearch
-docker-compose up -d
+docker compose up -d
 
 printf "Waiting for clusters to get ready "
 

recipes/ccr/stop.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 source .elastic-version
 
-docker-compose down -v
-docker-compose -f metricstore-docker-compose.yml down -v
+docker compose down -v
+docker compose -f metricstore-docker-compose.yml down -v