From a7d2219459172c6d5abaef126b208d410c07be09 Mon Sep 17 00:00:00 2001 From: Paul Reinlein Date: Fri, 13 Mar 2026 14:42:03 -0400 Subject: [PATCH] Adds SMP tests for idle & ascii --- experiments/regression/README.md | 9 +++++++ .../cases/http_ascii_1000mib/README.md | 21 +++++++++++++++ .../cases/http_ascii_1000mib/experiment.yaml | 27 +++++++++++++++++++ .../lading-target/lading.yaml | 18 +++++++++++++ .../http_ascii_1000mib/lading/lading.yaml | 9 +++++++ .../cases/http_ascii_100mib/README.md | 21 +++++++++++++++ .../cases/http_ascii_100mib/experiment.yaml | 27 +++++++++++++++++++ .../lading-target/lading.yaml | 18 +++++++++++++ .../http_ascii_100mib/lading/lading.yaml | 9 +++++++ .../cases/http_ascii_10mib/README.md | 21 +++++++++++++++ .../cases/http_ascii_10mib/experiment.yaml | 27 +++++++++++++++++++ .../lading-target/lading.yaml | 18 +++++++++++++ .../cases/http_ascii_10mib/lading/lading.yaml | 9 +++++++ .../cases/http_ascii_1mib/README.md | 21 +++++++++++++++ .../cases/http_ascii_1mib/experiment.yaml | 27 +++++++++++++++++++ .../http_ascii_1mib/lading-target/lading.yaml | 18 +++++++++++++ .../cases/http_ascii_1mib/lading/lading.yaml | 9 +++++++ experiments/regression/cases/idle/README.md | 16 +++++++++++ .../regression/cases/idle/experiment.yaml | 27 +++++++++++++++++++ .../cases/idle/lading-target/lading.yaml | 8 ++++++ .../regression/cases/idle/lading/lading.yaml | 9 +++++++ experiments/regression/config.yaml | 6 +++++ lading_payload/benches/ascii.rs | 3 +++ 23 files changed, 378 insertions(+) create mode 100644 experiments/regression/README.md create mode 100644 experiments/regression/cases/http_ascii_1000mib/README.md create mode 100644 experiments/regression/cases/http_ascii_1000mib/experiment.yaml create mode 100644 experiments/regression/cases/http_ascii_1000mib/lading-target/lading.yaml create mode 100644 experiments/regression/cases/http_ascii_1000mib/lading/lading.yaml create mode 100644 experiments/regression/cases/http_ascii_100mib/README.md create mode 100644 experiments/regression/cases/http_ascii_100mib/experiment.yaml create mode 100644 experiments/regression/cases/http_ascii_100mib/lading-target/lading.yaml create mode 100644 experiments/regression/cases/http_ascii_100mib/lading/lading.yaml create mode 100644 experiments/regression/cases/http_ascii_10mib/README.md create mode 100644 experiments/regression/cases/http_ascii_10mib/experiment.yaml create mode 100644 experiments/regression/cases/http_ascii_10mib/lading-target/lading.yaml create mode 100644 experiments/regression/cases/http_ascii_10mib/lading/lading.yaml create mode 100644 experiments/regression/cases/http_ascii_1mib/README.md create mode 100644 experiments/regression/cases/http_ascii_1mib/experiment.yaml create mode 100644 experiments/regression/cases/http_ascii_1mib/lading-target/lading.yaml create mode 100644 experiments/regression/cases/http_ascii_1mib/lading/lading.yaml create mode 100644 experiments/regression/cases/idle/README.md create mode 100644 experiments/regression/cases/idle/experiment.yaml create mode 100644 experiments/regression/cases/idle/lading-target/lading.yaml create mode 100644 experiments/regression/cases/idle/lading/lading.yaml create mode 100644 experiments/regression/config.yaml diff --git a/experiments/regression/README.md b/experiments/regression/README.md new file mode 100644 index 000000000..eba119ca1 --- /dev/null +++ b/experiments/regression/README.md @@ -0,0 +1,9 @@ +# Regression Experiments + +This folder defines all the regression detector experiments. + +The regression detector is a tool provided by Single Machine Performance thru the `smp` CLI. + +It allows us to performance test lading under different scenarios/loads. + +IMPORTANT: Any local benchmarks should be supplemented with SMP regression detector experiments that exercise that same local benchmark test. diff --git a/experiments/regression/cases/http_ascii_1000mib/README.md b/experiments/regression/cases/http_ascii_1000mib/README.md new file mode 100644 index 000000000..b692c6ad6 --- /dev/null +++ b/experiments/regression/cases/http_ascii_1000mib/README.md @@ -0,0 +1,21 @@ +# HTTP ASCII 1000 MiB/s + +Resource usage of lading under a steady 1000 MiB/s HTTP load with ASCII payloads. + +## What + +Runs lading as a target with an HTTP generator sending ASCII payloads at 1000 MiB/s to its own HTTP blackhole. This exercises the HTTP generator, ASCII payload construction, and HTTP blackhole within a single lading instance at maximum throughput. + +## Why + +Establishes a baseline for lading's resource consumption under an extreme HTTP workload. Regressions here indicate overhead in the HTTP generator or payload path at high throughput. + +## Paired Benchmark + +This experiment is paired with the local benchmark in `lading_payload/benches/ascii.rs`. +If throughput sizes change in either place, update the other to match. + +## Enforcements + +Memory usage is enforced by bounding `total_pss_bytes`. +CPU usage is enforced by bounding `avg(total_cpu_usage_millicores)`. diff --git a/experiments/regression/cases/http_ascii_1000mib/experiment.yaml b/experiments/regression/cases/http_ascii_1000mib/experiment.yaml new file mode 100644 index 000000000..4e0b3a161 --- /dev/null +++ b/experiments/regression/cases/http_ascii_1000mib/experiment.yaml @@ -0,0 +1,27 @@ +optimization_goal: memory +erratic: false + +target: + name: lading-target + # Setting experiment duration infinite here implies that lading will not go thru the shutdown process + # This is acceptable as we are less concerned about the shutdown performance + command: "/usr/bin/lading --no-target --experiment-duration-infinite --config-path /etc/lading-target/lading.yaml" + cpu_allotment: 2 + memory_allotment: 1250 MiB + + environment: + DD_SERVICE: lading-target + RUST_LOG: info + RUST_BACKTRACE: 1 + +checks: + - name: memory_usage + description: "Memory usage quality gate. Bounds total memory usage." + bounds: + series: total_pss_bytes + upper_bound: "1130 MiB" + - name: cpu_usage + description: "CPU usage quality gate. Bounds total average millicore usage." + bounds: + series: avg(total_cpu_usage_millicores) + upper_bound: 400 diff --git a/experiments/regression/cases/http_ascii_1000mib/lading-target/lading.yaml b/experiments/regression/cases/http_ascii_1000mib/lading-target/lading.yaml new file mode 100644 index 000000000..18e157d3f --- /dev/null +++ b/experiments/regression/cases/http_ascii_1000mib/lading-target/lading.yaml @@ -0,0 +1,18 @@ +generator: + - http: + seed: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32] + headers: {} + target_uri: "http://127.0.0.1:8080/" + bytes_per_second: "1000 MiB" + parallel_connections: 1 + method: + post: + maximum_prebuild_cache_size_bytes: "5 GiB" + variant: "ascii" + +blackhole: + - http: + binding_addr: "0.0.0.0:8080" + +telemetry: + addr: "0.0.0.0:9000" diff --git a/experiments/regression/cases/http_ascii_1000mib/lading/lading.yaml b/experiments/regression/cases/http_ascii_1000mib/lading/lading.yaml new file mode 100644 index 000000000..4c90f4924 --- /dev/null +++ b/experiments/regression/cases/http_ascii_1000mib/lading/lading.yaml @@ -0,0 +1,9 @@ +generator: [] + +blackhole: + - http: + binding_addr: "127.0.0.1:9091" + +target_metrics: + - prometheus: + uri: "http://127.0.0.1:9000/metrics" diff --git a/experiments/regression/cases/http_ascii_100mib/README.md b/experiments/regression/cases/http_ascii_100mib/README.md new file mode 100644 index 000000000..5b166198e --- /dev/null +++ b/experiments/regression/cases/http_ascii_100mib/README.md @@ -0,0 +1,21 @@ +# HTTP ASCII 100 MiB/s + +Resource usage of lading under a steady 100 MiB/s HTTP load with ASCII payloads. + +## What + +Runs lading as a target with an HTTP generator sending ASCII payloads at 100 MiB/s to its own HTTP blackhole. This exercises the HTTP generator, ASCII payload construction, and HTTP blackhole within a single lading instance. + +## Why + +Establishes a baseline for lading's resource consumption under a high HTTP workload. Regressions here indicate overhead in the HTTP generator or payload path independent of payload complexity. + +## Paired Benchmark + +This experiment is paired with the local benchmark in `lading_payload/benches/ascii.rs`. +If throughput sizes change in either place, update the other to match. + +## Enforcements + +Memory usage is enforced by bounding `total_pss_bytes`. +CPU usage is enforced by bounding `avg(total_cpu_usage_millicores)`. diff --git a/experiments/regression/cases/http_ascii_100mib/experiment.yaml b/experiments/regression/cases/http_ascii_100mib/experiment.yaml new file mode 100644 index 000000000..f0815f66b --- /dev/null +++ b/experiments/regression/cases/http_ascii_100mib/experiment.yaml @@ -0,0 +1,27 @@ +optimization_goal: memory +erratic: false + +target: + name: lading-target + # Setting experiment duration infinite here implies that lading will not go thru the shutdown process + # This is acceptable as we are less concerned about the shutdown performance + command: "/usr/bin/lading --no-target --experiment-duration-infinite --config-path /etc/lading-target/lading.yaml" + cpu_allotment: 2 + memory_allotment: 630 MiB + + environment: + DD_SERVICE: lading-target + RUST_LOG: info + RUST_BACKTRACE: 1 + +checks: + - name: memory_usage + description: "Memory usage quality gate. Bounds total memory usage." + bounds: + series: total_pss_bytes + upper_bound: "575 MiB" + - name: cpu_usage + description: "CPU usage quality gate. Bounds total average millicore usage." + bounds: + series: avg(total_cpu_usage_millicores) + upper_bound: 155 diff --git a/experiments/regression/cases/http_ascii_100mib/lading-target/lading.yaml b/experiments/regression/cases/http_ascii_100mib/lading-target/lading.yaml new file mode 100644 index 000000000..5771f6a7e --- /dev/null +++ b/experiments/regression/cases/http_ascii_100mib/lading-target/lading.yaml @@ -0,0 +1,18 @@ +generator: + - http: + seed: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32] + headers: {} + target_uri: "http://127.0.0.1:8080/" + bytes_per_second: "100 MiB" + parallel_connections: 1 + method: + post: + maximum_prebuild_cache_size_bytes: "500 MiB" + variant: "ascii" + +blackhole: + - http: + binding_addr: "0.0.0.0:8080" + +telemetry: + addr: "0.0.0.0:9000" diff --git a/experiments/regression/cases/http_ascii_100mib/lading/lading.yaml b/experiments/regression/cases/http_ascii_100mib/lading/lading.yaml new file mode 100644 index 000000000..4c90f4924 --- /dev/null +++ b/experiments/regression/cases/http_ascii_100mib/lading/lading.yaml @@ -0,0 +1,9 @@ +generator: [] + +blackhole: + - http: + binding_addr: "127.0.0.1:9091" + +target_metrics: + - prometheus: + uri: "http://127.0.0.1:9000/metrics" diff --git a/experiments/regression/cases/http_ascii_10mib/README.md b/experiments/regression/cases/http_ascii_10mib/README.md new file mode 100644 index 000000000..ee330a383 --- /dev/null +++ b/experiments/regression/cases/http_ascii_10mib/README.md @@ -0,0 +1,21 @@ +# HTTP ASCII 10 MiB/s + +Resource usage of lading under a steady 10 MiB/s HTTP load with ASCII payloads. + +## What + +Runs lading as a target with an HTTP generator sending ASCII payloads at 10 MiB/s to its own HTTP blackhole. This exercises the HTTP generator, ASCII payload construction, and HTTP blackhole within a single lading instance. + +## Why + +Establishes a baseline for lading's resource consumption under a moderate HTTP workload. Regressions here indicate overhead in the HTTP generator or payload path independent of payload complexity. + +## Paired Benchmark + +This experiment is paired with the local benchmark in `lading_payload/benches/ascii.rs`. +If throughput sizes change in either place, update the other to match. + +## Enforcements + +Memory usage is enforced by bounding `total_pss_bytes`. +CPU usage is enforced by bounding `avg(total_cpu_usage_millicores)`. diff --git a/experiments/regression/cases/http_ascii_10mib/experiment.yaml b/experiments/regression/cases/http_ascii_10mib/experiment.yaml new file mode 100644 index 000000000..3d7cd9f8b --- /dev/null +++ b/experiments/regression/cases/http_ascii_10mib/experiment.yaml @@ -0,0 +1,27 @@ +optimization_goal: memory +erratic: false + +target: + name: lading-target + # Setting experiment duration infinite here implies that lading will not go thru the shutdown process + # This is acceptable as we are less concerned about the shutdown performance + command: "/usr/bin/lading --no-target --experiment-duration-infinite --config-path /etc/lading-target/lading.yaml" + cpu_allotment: 2 + memory_allotment: 150 MiB + + environment: + DD_SERVICE: lading-target + RUST_LOG: info + RUST_BACKTRACE: 1 + +checks: + - name: memory_usage + description: "Memory usage quality gate. Bounds total memory usage." + bounds: + series: total_pss_bytes + upper_bound: "132 MiB" + - name: cpu_usage + description: "CPU usage quality gate. Bounds total average millicore usage." + bounds: + series: avg(total_cpu_usage_millicores) + upper_bound: 14 diff --git a/experiments/regression/cases/http_ascii_10mib/lading-target/lading.yaml b/experiments/regression/cases/http_ascii_10mib/lading-target/lading.yaml new file mode 100644 index 000000000..a30e73924 --- /dev/null +++ b/experiments/regression/cases/http_ascii_10mib/lading-target/lading.yaml @@ -0,0 +1,18 @@ +generator: + - http: + seed: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32] + headers: {} + target_uri: "http://127.0.0.1:8080/" + bytes_per_second: "10 MiB" + parallel_connections: 1 + method: + post: + maximum_prebuild_cache_size_bytes: "100 MiB" + variant: "ascii" + +blackhole: + - http: + binding_addr: "0.0.0.0:8080" + +telemetry: + addr: "0.0.0.0:9000" diff --git a/experiments/regression/cases/http_ascii_10mib/lading/lading.yaml b/experiments/regression/cases/http_ascii_10mib/lading/lading.yaml new file mode 100644 index 000000000..4c90f4924 --- /dev/null +++ b/experiments/regression/cases/http_ascii_10mib/lading/lading.yaml @@ -0,0 +1,9 @@ +generator: [] + +blackhole: + - http: + binding_addr: "127.0.0.1:9091" + +target_metrics: + - prometheus: + uri: "http://127.0.0.1:9000/metrics" diff --git a/experiments/regression/cases/http_ascii_1mib/README.md b/experiments/regression/cases/http_ascii_1mib/README.md new file mode 100644 index 000000000..2aac61671 --- /dev/null +++ b/experiments/regression/cases/http_ascii_1mib/README.md @@ -0,0 +1,21 @@ +# HTTP ASCII 1 MiB/s + +Resource usage of lading under a steady 1 MiB/s HTTP load with ASCII payloads. + +## What + +Runs lading as a target with an HTTP generator sending ASCII payloads at 1 MiB/s to its own HTTP blackhole. This exercises the HTTP generator, ASCII payload construction, and HTTP blackhole within a single lading instance. + +## Why + +Establishes a baseline for lading's resource consumption under a simple, sustained HTTP workload. Regressions here indicate overhead in the HTTP generator or payload path independent of payload complexity. + +## Paired Benchmark + +This experiment is paired with the local benchmark in `lading_payload/benches/ascii.rs`. +If throughput sizes change in either place, update the other to match. + +## Enforcements + +Memory usage is enforced by bounding `total_pss_bytes`. +CPU usage is enforced by bounding `avg(total_cpu_usage_millicores)`. diff --git a/experiments/regression/cases/http_ascii_1mib/experiment.yaml b/experiments/regression/cases/http_ascii_1mib/experiment.yaml new file mode 100644 index 000000000..176049258 --- /dev/null +++ b/experiments/regression/cases/http_ascii_1mib/experiment.yaml @@ -0,0 +1,27 @@ +optimization_goal: memory +erratic: false + +target: + name: lading-target + # Setting experiment duration infinite here implies that lading will not go thru the shutdown process + # This is acceptable as we are less concerned about the shutdown performance + command: "/usr/bin/lading --no-target --experiment-duration-infinite --config-path /etc/lading-target/lading.yaml" + cpu_allotment: 2 + memory_allotment: 40 MiB + + environment: + DD_SERVICE: lading-target + RUST_LOG: info + RUST_BACKTRACE: 1 + +checks: + - name: memory_usage + description: "Memory usage quality gate. Bounds total memory usage." + bounds: + series: total_pss_bytes + upper_bound: "34 MiB" + - name: cpu_usage + description: "CPU usage quality gate. Bounds total average millicore usage." + bounds: + series: avg(total_cpu_usage_millicores) + upper_bound: 2.4 diff --git a/experiments/regression/cases/http_ascii_1mib/lading-target/lading.yaml b/experiments/regression/cases/http_ascii_1mib/lading-target/lading.yaml new file mode 100644 index 000000000..92218de08 --- /dev/null +++ b/experiments/regression/cases/http_ascii_1mib/lading-target/lading.yaml @@ -0,0 +1,18 @@ +generator: + - http: + seed: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32] + headers: {} + target_uri: "http://127.0.0.1:8080/" + bytes_per_second: "1 MiB" + parallel_connections: 1 + method: + post: + maximum_prebuild_cache_size_bytes: "10 MiB" + variant: "ascii" + +blackhole: + - http: + binding_addr: "0.0.0.0:8080" + +telemetry: + addr: "0.0.0.0:9000" diff --git a/experiments/regression/cases/http_ascii_1mib/lading/lading.yaml b/experiments/regression/cases/http_ascii_1mib/lading/lading.yaml new file mode 100644 index 000000000..4c90f4924 --- /dev/null +++ b/experiments/regression/cases/http_ascii_1mib/lading/lading.yaml @@ -0,0 +1,9 @@ +generator: [] + +blackhole: + - http: + binding_addr: "127.0.0.1:9091" + +target_metrics: + - prometheus: + uri: "http://127.0.0.1:9000/metrics" diff --git a/experiments/regression/cases/idle/README.md b/experiments/regression/cases/idle/README.md new file mode 100644 index 000000000..39149a9be --- /dev/null +++ b/experiments/regression/cases/idle/README.md @@ -0,0 +1,16 @@ +# Idle + +Baseline resource usage of lading with no active workload. + +## What + +Runs lading as a target with an empty generator and a single HTTP blackhole; this represents the most basic way to run lading. + +## Why + +Establishes a floor for lading's memory footprint. Regressions here indicate overhead introduced independent of any workload. + +## Enforcements + +Memory usage is enforced by bounding `total_pss_bytes`. +CPU usage is enforced by bounding `avg(total_cpu_usage_millicores)`. diff --git a/experiments/regression/cases/idle/experiment.yaml b/experiments/regression/cases/idle/experiment.yaml new file mode 100644 index 000000000..d3c05ac90 --- /dev/null +++ b/experiments/regression/cases/idle/experiment.yaml @@ -0,0 +1,27 @@ +optimization_goal: memory +erratic: false + +target: + name: lading-target + # Setting experiment duration infinite here implies that lading will not go thru the shutdown process + # This is acceptable as we are less concerned about the shutdown performance + command: "/usr/bin/lading --no-target --experiment-duration-infinite --config-path /etc/lading-target/lading.yaml" + cpu_allotment: 2 + memory_allotment: 17 MiB + + environment: + DD_SERVICE: lading-target + RUST_LOG: info + RUST_BACKTRACE: 1 + +checks: + - name: memory_usage + description: "Memory usage quality gate. Bounds total memory usage." + bounds: + series: total_pss_bytes + upper_bound: "14 MiB" + - name: cpu_usage + description: "CPU usage quality gate. Bounds total average millicore usage." + bounds: + series: avg(total_cpu_usage_millicores) + upper_bound: 0.6 \ No newline at end of file diff --git a/experiments/regression/cases/idle/lading-target/lading.yaml b/experiments/regression/cases/idle/lading-target/lading.yaml new file mode 100644 index 000000000..044ab085a --- /dev/null +++ b/experiments/regression/cases/idle/lading-target/lading.yaml @@ -0,0 +1,8 @@ +generator: [] + +blackhole: + - http: + binding_addr: "0.0.0.0:8080" + +telemetry: + addr: "0.0.0.0:9000" \ No newline at end of file diff --git a/experiments/regression/cases/idle/lading/lading.yaml b/experiments/regression/cases/idle/lading/lading.yaml new file mode 100644 index 000000000..4c90f4924 --- /dev/null +++ b/experiments/regression/cases/idle/lading/lading.yaml @@ -0,0 +1,9 @@ +generator: [] + +blackhole: + - http: + binding_addr: "127.0.0.1:9091" + +target_metrics: + - prometheus: + uri: "http://127.0.0.1:9000/metrics" diff --git a/experiments/regression/config.yaml b/experiments/regression/config.yaml new file mode 100644 index 000000000..19de5fb45 --- /dev/null +++ b/experiments/regression/config.yaml @@ -0,0 +1,6 @@ +lading: + version: 0.31.2 + +target: + ddprof_replicas: 0 + internal_profiling_replicas: 0 diff --git a/lading_payload/benches/ascii.rs b/lading_payload/benches/ascii.rs index 05838897c..aec42b66f 100644 --- a/lading_payload/benches/ascii.rs +++ b/lading_payload/benches/ascii.rs @@ -1,4 +1,7 @@ //! Benchmarks for ASCII payload generation. +//! +//! Paired with SMP regression experiments in `experiments/regression/cases/http_ascii_*`. +//! If throughput sizes here change, update the corresponding SMP experiments and vice versa. use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; use lading_payload::{Serialize, ascii};