Skip to content

Enabled lifetime window for batch's byte size metric #18911

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andsel merged 3 commits into from
Apr 1, 2026
Merged

Enabled lifetime window for batch's byte size metric #18911

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6aded95
Enabled lifetime window for batch's byte size metric
andsel Mar 27, 2026
3859bbf
[Test] Better wording in assertion
andsel Apr 1, 2026
c42cb40
[Doc] Updated API documentation to reflect current data shape of the …
andsel Apr 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions docs/static/spec/openapi/logstash-api.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -921,13 +921,20 @@ paths:
last_5_minutes: 14012
last_15_minutes: 14567
p50:
lifetime: 14815
last_1_minute: 15240
last_5_minutes: 14030
last_15_minutes: 14600
p90:
lifetime: 14825
last_1_minute: 15300
last_5_minutes: 14050
last_15_minutes: 14630
last_15_minutes: 14630
max:
lifetime: 14830
last_1_minute: 16000
last_5_minutes: 14080
last_15_minutes: 14670
events:
duration_in_millis: 365495
in: 216610
Expand Down Expand Up @@ -1226,13 +1233,20 @@ paths:
last_5_minutes: 14012
last_15_minutes: 14567
p50:
lifetime: 14815
last_1_minute: 15240
last_5_minutes: 14030
last_15_minutes: 14600
p90:
lifetime: 14825
last_1_minute: 15300
last_5_minutes: 14050
last_15_minutes: 14630
last_15_minutes: 14630
max:
lifetime: 14830
last_1_minute: 16000
last_5_minutes: 14080
last_15_minutes: 14670
events:
queue_push_duration_in_millis: 159
in: 45
Expand Down
2 changes: 1 addition & 1 deletion logstash-core/lib/logstash/api/commands/stats.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ def refine_batch_metrics(stats)
# stats[:batch][:byte_size][:batch_structure_metric] is an instance of org.logstash.instrument.metrics.BatchStructureMetric
# so need to call "value" to grab the map of sub-metrics which contains the histogram percentiles
# as org.logstash.instrument.metrics.BatchStructureMetric$HistogramMetricData
[:last_1_minute, :last_5_minutes, :last_15_minutes].each do |window|
[:last_1_minute, :last_5_minutes, :last_15_minutes, :lifetime].each do |window|
byte_size_histogram_value = byte_size_histogram.value
if byte_size_histogram_value[window.to_s]
reshape_histogram_percentiles_for_window(:byte_size, byte_size_histogram_value, window, result)
Expand Down
3 changes: 2 additions & 1 deletion logstash-core/src/main/java/org/logstash/instrument/metrics/BatchStructureMetric.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ public class BatchStructureMetric extends AbstractMetric<Map<String, BatchStruct
private static final List<FlowMetricRetentionPolicy> SUPPORTED_POLICIES = List.of(
BuiltInFlowMetricRetentionPolicies.LAST_1_MINUTE,
BuiltInFlowMetricRetentionPolicies.LAST_5_MINUTES,
BuiltInFlowMetricRetentionPolicies.LAST_15_MINUTES
BuiltInFlowMetricRetentionPolicies.LAST_15_MINUTES,
BuiltInFlowMetricRetentionPolicies.LIFETIME
);
private final ConcurrentMap<FlowMetricRetentionPolicy, HistogramRetentionWindow> histogramsWindows = new ConcurrentHashMap<>();

Expand Down
21 changes: 15 additions & 6 deletions logstash-core/src/test/java/org/logstash/instrument/metrics/BatchStructureMetricTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,13 +70,16 @@ public void givenMostlyStaticValues_whenRecordingValues_thenHistogramsReflectThi
// Retrieve histogram values, getValue takes another capture and computes the diff with the initial empty capture
Map<String, BatchStructureMetric.HistogramMetricData> histogramMap = sut.getValue();

assertThat("contains just the last 1 minute histogram", histogramMap,
allOf(aMapWithSize(1), hasKey("last_1_minute")));
assertThat("contains just the last 1 minute and lifetime histogram", histogramMap,
allOf(aMapWithSize(2), hasKey("last_1_minute"), hasKey("lifetime")));

// Check against the reference histogram
BatchStructureMetric.HistogramMetricData last1MinuteData = histogramMap.get("last_1_minute");
assertEquals(referenceHistogram.getValueAtPercentile(50), last1MinuteData.get50Percentile(), 0.1);
assertEquals(referenceHistogram.getValueAtPercentile(90), last1MinuteData.get90Percentile(), 0.1);
BatchStructureMetric.HistogramMetricData lifetimeData = histogramMap.get("lifetime");
assertEquals(referenceHistogram.getValueAtPercentile(50), lifetimeData.get50Percentile(), 0.1);
assertEquals(referenceHistogram.getValueAtPercentile(90), lifetimeData.get90Percentile(), 0.1);
}

@Test
Expand Down Expand Up @@ -109,8 +112,8 @@ public void givenRunningMetricForMoreMinutesThenHistogramsHasToReflectTimeAndVal
// Retrieve histogram values and verify values for the time windows
Map<String, BatchStructureMetric.HistogramMetricData> histogramMap = sut.getValue();

assertThat("contains just last 1 minute and 5 minutes histograms", histogramMap,
allOf(aMapWithSize(2), hasKey("last_1_minute"), hasKey("last_5_minutes")));
assertThat("contains last 1 minute, 5 minutes, and lifetime histograms", histogramMap,
allOf(aMapWithSize(3), hasKey("last_1_minute"), hasKey("last_5_minutes"), hasKey("lifetime")));

// Since values are uniformly distributed, we can check expected percentiles
BatchStructureMetric.HistogramMetricData last1MinuteData = histogramMap.get("last_1_minute");
Expand All @@ -119,6 +122,9 @@ public void givenRunningMetricForMoreMinutesThenHistogramsHasToReflectTimeAndVal
BatchStructureMetric.HistogramMetricData last5MinutesData = histogramMap.get("last_5_minutes");
assertEquals(100, last5MinutesData.get50Percentile(), 10);
assertEquals(1000, last5MinutesData.get90Percentile(), 10);
BatchStructureMetric.HistogramMetricData lifetimeData = histogramMap.get("lifetime");
assertEquals(100, lifetimeData.get50Percentile(), 10);
assertEquals(1000, lifetimeData.get90Percentile(), 10);
}

@Test
Expand Down Expand Up @@ -149,8 +155,8 @@ public void givenRunningMetricWhenNoDataComesInForLastMinuteThenHistogramReflect
// Retrieve histogram values and verify values for the time windows
Map<String, BatchStructureMetric.HistogramMetricData> histogramMap = sut.getValue();

assertThat("contains just last 1 minute and 5 minutes histograms", histogramMap,
allOf(aMapWithSize(2), hasKey("last_1_minute"), hasKey("last_5_minutes")));
assertThat("contains just last 1 minute, 5 minutes and lifetime histograms", histogramMap,
allOf(aMapWithSize(3), hasKey("last_1_minute"), hasKey("last_5_minutes"), hasKey("lifetime")));

// Since values are uniformly distributed, we can check expected percentiles
BatchStructureMetric.HistogramMetricData last1MinuteData = histogramMap.get("last_1_minute");
Expand All @@ -159,5 +165,8 @@ public void givenRunningMetricWhenNoDataComesInForLastMinuteThenHistogramReflect
BatchStructureMetric.HistogramMetricData last5MinutesData = histogramMap.get("last_5_minutes");
assertEquals(100, last5MinutesData.get50Percentile(), 10);
assertEquals(200, last5MinutesData.get90Percentile(), 10);
BatchStructureMetric.HistogramMetricData lifetimeData = histogramMap.get("lifetime");
assertEquals(100, lifetimeData.get50Percentile(), 10);
assertEquals(200, lifetimeData.get90Percentile(), 10);
}
}
Loading