feat: speed benchmark and network creation optimization

.github/tests/speed-3x3.yaml

@@ -0,0 +1,26 @@
+participants:
+  - el_type: geth
+    cl_type: lighthouse
+  - el_type: reth
+    cl_type: teku
+  - el_type: nethermind
+    cl_type: nimbus
+  - el_type: geth
+    cl_type: teku
+  - el_type: reth
+    cl_type: lighthouse
+  - el_type: nethermind
+    cl_type: nimbus
+  - el_type: geth
+    cl_type: nimbus
+  - el_type: reth
+    cl_type: teku
+  - el_type: nethermind
+    cl_type: lighthouse
+network_params:
+  preset: minimal
+  genesis_delay: 5
+  electra_fork_epoch: 0
+  fulu_fork_epoch: 18446744073709551615
+parallel_keystore_generation: true
+additional_services: []

.github/tests/speed.yaml

@@ -0,0 +1,10 @@
+participants:
+  - el_type: geth
+    cl_type: lighthouse
+network_params:
+  preset: minimal
+  genesis_delay: 5
+  electra_fork_epoch: 0
+  fulu_fork_epoch: 18446744073709551615
+parallel_keystore_generation: true
+additional_services: []

.github/workflows/per-pr.yml

@@ -78,6 +78,116 @@ jobs:
           ethereum_package_args: .github/tests/mix-assert.yaml
 
 
+  speed_benchmark:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - uses: ./.github/actions/docker-login
+        with:
+          username: ethpandaops
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Setup Kurtosis
+        uses: ./.github/actions/kurtosis-install
+      - name: Pre-pull Docker images
+        run: |
+          # Pre-pull images in parallel so kurtosis finds them cached (with --image-download missing)
+          docker pull ethereum/client-go:latest &
+          docker pull ethpandaops/lighthouse:unstable &
+          docker pull ethpandaops/ethereum-genesis-generator:5.2.4 &
+          docker pull protolambda/eth2-val-tools:latest &
+          wait
+      - name: Run Speed Benchmark
+        run: |
+          set -euo pipefail
+
+          echo "=== Speed Benchmark ==="
+          echo "Start time: $(date -u +%Y-%m-%dT%H:%M:%S%Z)"
+          START_TIME=$(date +%s)
+
+          kurtosis run ${{ github.workspace }} \
+            --args-file .github/tests/speed.yaml \
+            --verbosity detailed \
+            --image-download missing 2>&1 | tee /tmp/speed-benchmark-output.log
+
+          END_TIME=$(date +%s)
+          TOTAL_DURATION=$((END_TIME - START_TIME))
+
+          echo ""
+          echo "========================================="
+          echo "=== SPEED BENCHMARK RESULTS ==="
+          echo "========================================="
+          echo "Total wall-clock time: ${TOTAL_DURATION}s"
+          echo "Target: 60s"
+          echo ""
+          echo "=== Phase Timing Markers ==="
+          grep "TIMING:" /tmp/speed-benchmark-output.log || echo "No timing markers found"
+          echo ""
+          if [ "$TOTAL_DURATION" -le 60 ]; then
+            echo "PASS: Network created within 60s target"
+          else
+            echo "OVER TARGET: ${TOTAL_DURATION}s exceeds 60s target by $((TOTAL_DURATION - 60))s"
+          fi
+          echo "========================================="
+
+  speed_benchmark_3x3:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - uses: ./.github/actions/docker-login
+        with:
+          username: ethpandaops
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Setup Kurtosis
+        uses: ./.github/actions/kurtosis-install
+      - name: Pre-pull Docker images
+        run: |
+          # Pre-pull images in parallel so kurtosis finds them cached (with --image-download missing)
+          docker pull ethereum/client-go:latest &
+          docker pull ghcr.io/paradigmxyz/reth &
+          docker pull nethermind/nethermind:latest &
+          docker pull ethpandaops/lighthouse:unstable &
+          docker pull ethpandaops/teku:master &
+          docker pull ethpandaops/nimbus-eth2:unstable-minimal &
+          docker pull ethpandaops/ethereum-genesis-generator:5.2.4 &
+          docker pull protolambda/eth2-val-tools:latest &
+          wait
+      - name: Run Speed Benchmark 3x3
+        run: |
+          set -euo pipefail
+
+          echo "=== Speed Benchmark 3x3 (9 participants) ==="
+          echo "Start time: $(date -u +%Y-%m-%dT%H:%M:%S%Z)"
+          START_TIME=$(date +%s)
+
+          kurtosis run ${{ github.workspace }} \
+            --args-file .github/tests/speed-3x3.yaml \
+            --verbosity detailed \
+            --image-download missing 2>&1 | tee /tmp/speed-benchmark-3x3-output.log
+
+          END_TIME=$(date +%s)
+          TOTAL_DURATION=$((END_TIME - START_TIME))
+
+          echo ""
+          echo "========================================="
+          echo "=== SPEED BENCHMARK 3x3 RESULTS ==="
+          echo "========================================="
+          echo "Total wall-clock time: ${TOTAL_DURATION}s"
+          echo "Target: 120s"
+          echo ""
+          echo "=== Phase Timing Markers ==="
+          grep "TIMING:" /tmp/speed-benchmark-3x3-output.log || echo "No timing markers found"
+          echo ""
+          if [ "$TOTAL_DURATION" -le 120 ]; then
+            echo "PASS: Network created within 120s target"
+          else
+            echo "OVER TARGET: ${TOTAL_DURATION}s exceeds 120s target by $((TOTAL_DURATION - 120))s"
+          fi
+          echo "========================================="
+
   #check_optimism_package:
   #  runs-on: ubuntu-latest
   #  steps:

main.star

@@ -85,6 +85,7 @@ def run(plan, args={}):
     """
 
     args_with_right_defaults = input_parser.input_parser(plan, args)
+    plan.print("TIMING:run:start")
 
     num_participants = len(args_with_right_defaults.participants)
     network_params = args_with_right_defaults.network_params
@@ -611,6 +612,7 @@ def run(plan, args={}):
                 all_mevboost_contexts.append(mev_boost_context)
 
     if len(args_with_right_defaults.additional_services) == 0:
+        plan.print("TIMING:run:end")
         output = struct(
             all_participants=all_participants,
             pre_funded_accounts=prefunded_accounts,
@@ -1094,6 +1096,7 @@ def run(plan, args={}):
         password=GRAFANA_PASSWORD,
     )
 
+    plan.print("TIMING:run:end")
     output = struct(
         grafana_info=grafana_info,
         blockscout_sc_verif_url=None

src/cl/cl_launcher.star

@@ -317,6 +317,7 @@ def launch(
                 tempo_otlp_grpc_url,
                 bootnode_enr_override,
                 cl_binary_artifact,
+                skip_ready_conditions=True,
             )
 
             cl_participant_info[cl_service_name] = {
@@ -353,6 +354,7 @@ def launch(
             info["snooper_el_engine_context"],
             info["new_cl_node_validator_keystores"],
             info["node_selectors"],
+            skip_identity=True,
         )
 
         blobber_config = get_blobber_config(
@@ -391,3 +393,71 @@ def launch(
         global_other_index,
         blobber_configs_with_contexts,
     )
+
+
+def collect_identities(plan, all_cl_contexts, participants):
+    """Fill in missing ENRs/multiaddrs/peer_ids for contexts created with skip_identity=True.
+    Uses plan.wait to retry until CLs are healthy (health checks are deferred for non-boot nodes).
+    """
+    enriched = []
+    for index, ctx in enumerate(all_cl_contexts):
+        participant = participants[index] if index < len(participants) else None
+        skip_start = participant.skip_start if participant else False
+        if ctx.enr == "" and ctx.beacon_service_name != "" and not skip_start:
+            # Determine multiaddr jq based on client type
+            multiaddr_jq = ".data.p2p_addresses[0]"
+            headers = {}
+            if ctx.client_name == "lodestar":
+                multiaddr_jq = ".data.p2p_addresses[-1]"
+            if ctx.client_name == "prysm":
+                headers = {"Accept-Encoding": "identity"}
+
+            extract = {
+                "enr": ".data.enr",
+                "multiaddr": multiaddr_jq,
+                "peer_id": ".data.peer_id",
+            }
+            if headers:
+                beacon_node_identity_recipe = GetHttpRequestRecipe(
+                    endpoint="/eth/v1/node/identity",
+                    port_id=constants.HTTP_PORT_ID,
+                    extract=extract,
+                    headers=headers,
+                )
+            else:
+                beacon_node_identity_recipe = GetHttpRequestRecipe(
+                    endpoint="/eth/v1/node/identity",
+                    port_id=constants.HTTP_PORT_ID,
+                    extract=extract,
+                )
+            response = plan.wait(
+                recipe=beacon_node_identity_recipe,
+                service_name=ctx.beacon_service_name,
+                field="code",
+                assertion="IN",
+                target_value=[200],
+                interval="1s",
+                timeout="5m",
+            )
+            enriched.append(
+                cl_context_l.new_cl_context(
+                    client_name=ctx.client_name,
+                    enr=response["extract.enr"],
+                    ip_addr=ctx.ip_addr,
+                    ip_address=ctx.ip_address,
+                    http_port=ctx.http_port,
+                    beacon_http_url=ctx.beacon_http_url,
+                    cl_nodes_metrics_info=ctx.cl_nodes_metrics_info,
+                    beacon_service_name=ctx.beacon_service_name,
+                    beacon_grpc_url=ctx.beacon_grpc_url,
+                    multiaddr=response["extract.multiaddr"],
+                    peer_id=response["extract.peer_id"],
+                    snooper_enabled=ctx.snooper_enabled,
+                    snooper_el_engine_context=ctx.snooper_el_engine_context,
+                    validator_keystore_files_artifact_uuid=ctx.validator_keystore_files_artifact_uuid,
+                    supernode=ctx.supernode,
+                )
+            )
+        else:
+            enriched.append(ctx)
+    return enriched

src/cl/cl_node_ready_conditions.star

@@ -6,6 +6,7 @@ def get_ready_conditions(port_id):
         field="code",
         assertion="IN",
         target_value=[200, 206],
+        interval="0.5s",
         timeout="30m",
     )
 

src/cl/consensoor/consensoor_launcher.star

@@ -115,6 +115,7 @@ def get_beacon_config(
     tempo_otlp_grpc_url,
     bootnode_enr_override=None,
     cl_binary_artifact=None,
+    skip_ready_conditions=False,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant.cl_log_level, global_log_level, VERBOSITY_LEVELS
@@ -291,7 +292,7 @@ def get_beacon_config(
         "node_selectors": node_selectors,
     }
 
-    if not participant.skip_start:
+    if not participant.skip_start and not skip_ready_conditions:
         config_args["ready_conditions"] = cl_node_ready_conditions.get_ready_conditions(
             constants.HTTP_PORT_ID
         )
@@ -315,11 +316,14 @@ def get_cl_context(
     snooper_el_engine_context,
     node_keystore_files,
     node_selectors,
+    skip_identity=False,
 ):
     beacon_http_port = service.ports[constants.HTTP_PORT_ID]
     beacon_http_url = "http://{0}:{1}".format(service.name, beacon_http_port.number)
 
-    if participant.skip_start:
+    # Skip HTTP requests if skip_start is enabled (service won't be running)
+    # or if skip_identity is set (identity will be collected later)
+    if participant.skip_start or skip_identity:
         beacon_node_enr = ""
         beacon_multiaddr = ""
         beacon_peer_id = ""

src/cl/grandine/grandine_launcher.star

@@ -126,6 +126,7 @@ def get_beacon_config(
     tempo_otlp_grpc_url,
     bootnode_enr_override=None,
     cl_binary_artifact=None,
+    skip_ready_conditions=False,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant.cl_log_level, global_log_level, VERBOSITY_LEVELS
@@ -383,8 +384,8 @@ def get_beacon_config(
 
     if len(participant.cl_devices) > 0:
         config_args["devices"] = participant.cl_devices
-    # Only add ready_conditions if not skipping start
-    if not participant.skip_start:
+    # Only add ready_conditions if not skipping start and not deferring health checks
+    if not participant.skip_start and not skip_ready_conditions:
         config_args["ready_conditions"] = cl_node_ready_conditions.get_ready_conditions(
             constants.HTTP_PORT_ID
         )
@@ -408,6 +409,7 @@ def get_cl_context(
     snooper_el_engine_context,
     node_keystore_files,
     node_selectors,
+    skip_identity=False,
 ):
     beacon_http_port = service.ports[constants.HTTP_PORT_ID]
     beacon_http_url = "http://{0}:{1}".format(service.name, beacon_http_port.number)
@@ -416,7 +418,8 @@ def get_cl_context(
     beacon_metrics_url = "{0}:{1}".format(service.name, beacon_metrics_port.number)
 
     # Skip HTTP requests if skip_start is enabled (service won't be running)
-    if participant.skip_start:
+    # or if skip_identity is set (identity will be collected later)
+    if participant.skip_start or skip_identity:
         beacon_node_enr = ""
         beacon_multiaddr = ""
         beacon_peer_id = ""

src/cl/lighthouse/lighthouse_launcher.star

@@ -130,6 +130,7 @@ def get_beacon_config(
     tempo_otlp_grpc_url,
     bootnode_enr_override=None,
     cl_binary_artifact=None,
+    skip_ready_conditions=False,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant.cl_log_level, global_log_level, VERBOSITY_LEVELS
@@ -359,8 +360,8 @@ def get_beacon_config(
 
     if len(participant.cl_devices) > 0:
         config_args["devices"] = participant.cl_devices
-    # Only add ready_conditions if not skipping start
-    if not participant.skip_start:
+    # Only add ready_conditions if not skipping start and not deferring health checks
+    if not participant.skip_start and not skip_ready_conditions:
         config_args["ready_conditions"] = cl_node_ready_conditions.get_ready_conditions(
             constants.HTTP_PORT_ID
         )
@@ -384,12 +385,14 @@ def get_cl_context(
     snooper_el_engine_context,
     node_keystore_files,
     node_selectors,
+    skip_identity=False,
 ):
     beacon_http_port = service.ports[constants.HTTP_PORT_ID]
     beacon_http_url = "http://{0}:{1}".format(service.name, beacon_http_port.number)
 
     # Skip HTTP requests if skip_start is enabled (service won't be running)
-    if participant.skip_start:
+    # or if skip_identity is set (identity will be collected later)
+    if participant.skip_start or skip_identity:
         beacon_node_enr = ""
         beacon_multiaddr = ""
         beacon_peer_id = ""