From c749beb8738c298931dbfe66ac42f49d8588100a Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Thu, 29 Jan 2026 10:23:57 -0800 Subject: [PATCH 01/26] intra segment support for query_string Signed-off-by: Prudhvi Godithi --- .../search/query/QueryStringQueryIT.java | 69 +++++++++++++++++++ .../index/query/QueryStringQueryBuilder.java | 5 ++ 2 files changed, 74 insertions(+) create mode 100644 server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringQueryIT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringQueryIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringQueryIT.java new file mode 100644 index 0000000000000..f80ac47068a12 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringQueryIT.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; + +import static org.opensearch.index.query.QueryBuilders.queryStringQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; + +/** + * Integration tests for query_string queries with concurrent segment search and partition strategies. + */ +public class QueryStringQueryIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { + + public QueryStringQueryIT(Settings staticSettings) { + super(staticSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), "all").build() }, + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "segment").build() }, + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "force").build() }, + new Object[] { + Settings.builder() + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "balanced") + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE.getKey(), 1000) + .build() } + ); + } + + public void testQueryStringQuery() throws Exception { + createIndex("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + int totalDocs = 2500; + for (int i = 0; i < totalDocs; i++) { + String content = (i % 100 == 0) ? "monkey jackal bear" : (i % 50 == 0) ? "monkey bear" : "other words " + i; + client().prepareIndex("test").setId(String.valueOf(i)).setSource("message", content).get(); + } + refresh(); + forceMerge(1); + indexRandomForConcurrentSearch("test"); + SearchResponse response = client().prepareSearch("test").setQuery(queryStringQuery("message:monkey AND message:jackal")).get(); + assertHitCount(response, 25L); + response = client().prepareSearch("test").setQuery(queryStringQuery("message:monkey")).get(); + assertHitCount(response, 50L); + response = client().prepareSearch("test").setQuery(queryStringQuery("message:bear")).get(); + assertHitCount(response, 50L); + } +} diff --git a/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java index bf84740ca4003..df79159fcb561 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java @@ -987,4 +987,9 @@ protected Query doToQuery(QueryShardContext context) throws IOException { return query; } + + @Override + public boolean supportsIntraSegmentSearch() { + return true; + } } From 0f62870e078275acd02d318ec862ec79b6c9fbbc Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Thu, 29 Jan 2026 10:27:43 -0800 Subject: [PATCH 02/26] update changelog Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index df9eb6ab93bf3..8d008d4367135 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,27 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add range validations in query builder and field mapper ([#20497](https://github.com/opensearch-project/OpenSearch/issues/20497)) - [Workload Management] Enhance Scroll API support for autotagging ([#20151](https://github.com/opensearch-project/OpenSearch/pull/20151)) - Add indices to search request slowlog ([#20588](https://github.com/opensearch-project/OpenSearch/pull/20588)) +- Add support for fields containing dots in their name as literals ([#19958](https://github.com/opensearch-project/OpenSearch/pull/19958)) +- Add support for forward translog reading ([#20163](https://github.com/opensearch-project/OpenSearch/pull/20163)) +- Added public getter method in `SourceFieldMapper` to return excluded field ([#20205](https://github.com/opensearch-project/OpenSearch/pull/20205)) +- Add integ test for simulating node join left event when data node cluster state publication lag because the cluster applier thread being busy ([#19907](https://github.com/opensearch-project/OpenSearch/pull/19907)). +- Relax jar hell check when extended plugins share transitive dependencies ([#20103](https://github.com/opensearch-project/OpenSearch/pull/20103)) +- Added public getter method in `SourceFieldMapper` to return included field ([#20290](https://github.com/opensearch-project/OpenSearch/pull/20290)) +- Support for HTTP/3 (server side) ([#20017](https://github.com/opensearch-project/OpenSearch/pull/20017)) +- Add circuit breaker support for gRPC transport to prevent out-of-memory errors ([#20203](https://github.com/opensearch-project/OpenSearch/pull/20203)) +- Add index-level-encryption support for snapshots and remote-store ([#20095](https://github.com/opensearch-project/OpenSearch/pull/20095)) +- Adding BackWardCompatibility test for remote publication enabled cluster ([#20221](https://github.com/opensearch-project/OpenSearch/pull/20221)) +- Support for hll field mapper to support cardinality rollups ([#20129](https://github.com/opensearch-project/OpenSearch/pull/20129)) +- Add tracing support for StreamingRestChannel ([#20361](https://github.com/opensearch-project/OpenSearch/pull/20361)) +- Introduce new libs/netty4 module to share common implementation between netty-based plugins and modules (transport-netty4, transport-reactor-netty4) ([#20447](https://github.com/opensearch-project/OpenSearch/pull/20447)) +- Add validation to make crypto store settings immutable ([#20123](https://github.com/opensearch-project/OpenSearch/pull/20123)) +- Introduce concurrent translog recovery to accelerate segment replication primary promotion ([#20251](https://github.com/opensearch-project/OpenSearch/pull/20251)) +- Update to `almalinux:10` ([#20482](https://github.com/opensearch-project/OpenSearch/pull/20482)) +- Add X-Request-Id to uniquely identify a search request ([#19798](https://github.com/opensearch-project/OpenSearch/pull/19798)) +- Added TopN selection logic for streaming terms aggregations ([#20481](https://github.com/opensearch-project/OpenSearch/pull/20481)) +- Added support for Intra Segment Search ([#19704](https://github.com/opensearch-project/OpenSearch/pull/19704)) +- Introduce AdditionalCodecs and EnginePlugin::getAdditionalCodecs hook to allow additional Codec registration ([#20411](https://github.com/opensearch-project/OpenSearch/pull/20411)) +- Added intra segment support for `query_string` ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) ### Changed - Move Randomness from server to libs/common ([#20570](https://github.com/opensearch-project/OpenSearch/pull/20570)) From 0d5eb3c5c2d4007b763bfec32dc02270d71f6445 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Wed, 4 Feb 2026 17:19:09 -0800 Subject: [PATCH 03/26] Intra segment support for single-value metric aggregations Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 2 +- .../search/query/QueryStringQueryIT.java | 69 ------------------- .../index/query/QueryStringQueryBuilder.java | 5 -- .../metrics/AvgAggregatorFactory.java | 5 ++ .../metrics/CardinalityAggregatorFactory.java | 5 ++ .../metrics/MaxAggregatorFactory.java | 5 ++ .../metrics/MinAggregatorFactory.java | 5 ++ .../metrics/StatsAggregatorFactory.java | 5 ++ .../metrics/SumAggregatorFactory.java | 5 ++ .../metrics/ValueCountAggregatorFactory.java | 5 ++ 10 files changed, 36 insertions(+), 75 deletions(-) delete mode 100644 server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringQueryIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d008d4367135..6f3d1923465b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Added TopN selection logic for streaming terms aggregations ([#20481](https://github.com/opensearch-project/OpenSearch/pull/20481)) - Added support for Intra Segment Search ([#19704](https://github.com/opensearch-project/OpenSearch/pull/19704)) - Introduce AdditionalCodecs and EnginePlugin::getAdditionalCodecs hook to allow additional Codec registration ([#20411](https://github.com/opensearch-project/OpenSearch/pull/20411)) -- Added intra segment support for `query_string` ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) +- Added intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) ### Changed - Move Randomness from server to libs/common ([#20570](https://github.com/opensearch-project/OpenSearch/pull/20570)) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringQueryIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringQueryIT.java deleted file mode 100644 index f80ac47068a12..0000000000000 --- a/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringQueryIT.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.search.query; - -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.settings.Settings; -import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; - -import java.util.Arrays; -import java.util.Collection; - -import static org.opensearch.index.query.QueryBuilders.queryStringQuery; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; -import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE; -import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; - -/** - * Integration tests for query_string queries with concurrent segment search and partition strategies. - */ -public class QueryStringQueryIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { - - public QueryStringQueryIT(Settings staticSettings) { - super(staticSettings); - } - - @ParametersFactory - public static Collection parameters() { - return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), "all").build() }, - new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "segment").build() }, - new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "force").build() }, - new Object[] { - Settings.builder() - .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "balanced") - .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE.getKey(), 1000) - .build() } - ); - } - - public void testQueryStringQuery() throws Exception { - createIndex("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); - int totalDocs = 2500; - for (int i = 0; i < totalDocs; i++) { - String content = (i % 100 == 0) ? "monkey jackal bear" : (i % 50 == 0) ? "monkey bear" : "other words " + i; - client().prepareIndex("test").setId(String.valueOf(i)).setSource("message", content).get(); - } - refresh(); - forceMerge(1); - indexRandomForConcurrentSearch("test"); - SearchResponse response = client().prepareSearch("test").setQuery(queryStringQuery("message:monkey AND message:jackal")).get(); - assertHitCount(response, 25L); - response = client().prepareSearch("test").setQuery(queryStringQuery("message:monkey")).get(); - assertHitCount(response, 50L); - response = client().prepareSearch("test").setQuery(queryStringQuery("message:bear")).get(); - assertHitCount(response, 50L); - } -} diff --git a/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java index df79159fcb561..bf84740ca4003 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java @@ -987,9 +987,4 @@ protected Query doToQuery(QueryShardContext context) throws IOException { return query; } - - @Override - public boolean supportsIntraSegmentSearch() { - return true; - } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregatorFactory.java index 57389f19b4577..212090d35a5ed 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregatorFactory.java @@ -100,4 +100,9 @@ protected Aggregator doCreateInternal( protected boolean supportsConcurrentSegmentSearch() { return true; } + + @Override + protected boolean supportsIntraSegmentSearch() { + return true; + } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorFactory.java index 977d5202fa569..dbccaa2c1b7d0 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorFactory.java @@ -162,6 +162,11 @@ protected boolean supportsConcurrentSegmentSearch() { return true; } + @Override + protected boolean supportsIntraSegmentSearch() { + return true; + } + private int precision() { return precisionThreshold == null ? HyperLogLogPlusPlus.DEFAULT_PRECISION diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregatorFactory.java index 90b61fb0d0866..1aa5b88426957 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregatorFactory.java @@ -103,6 +103,11 @@ protected boolean supportsConcurrentSegmentSearch() { return true; } + @Override + protected boolean supportsIntraSegmentSearch() { + return true; + } + @Override public StreamingCostMetrics estimateStreamingCost(SearchContext searchContext) { return StreamingCostMetrics.neutral(); diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregatorFactory.java index 7f553d94ccd38..3659ece739166 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregatorFactory.java @@ -103,6 +103,11 @@ protected boolean supportsConcurrentSegmentSearch() { return true; } + @Override + protected boolean supportsIntraSegmentSearch() { + return true; + } + @Override public StreamingCostMetrics estimateStreamingCost(SearchContext searchContext) { return StreamingCostMetrics.neutral(); diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/StatsAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/StatsAggregatorFactory.java index 0e96e631044dd..f3f6c29584ab9 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/StatsAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/StatsAggregatorFactory.java @@ -95,4 +95,9 @@ protected Aggregator doCreateInternal( protected boolean supportsConcurrentSegmentSearch() { return true; } + + @Override + protected boolean supportsIntraSegmentSearch() { + return true; + } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregatorFactory.java index eb1e35687e0f4..cf9179e65deb6 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregatorFactory.java @@ -103,6 +103,11 @@ protected boolean supportsConcurrentSegmentSearch() { return true; } + @Override + protected boolean supportsIntraSegmentSearch() { + return true; + } + @Override public StreamingCostMetrics estimateStreamingCost(SearchContext searchContext) { return StreamingCostMetrics.neutral(); diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregatorFactory.java index 0c82279484461..d865a038ea881 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregatorFactory.java @@ -94,4 +94,9 @@ protected Aggregator doCreateInternal( protected boolean supportsConcurrentSegmentSearch() { return true; } + + @Override + protected boolean supportsIntraSegmentSearch() { + return true; + } } From c8a7992c13ca26b633ba697356a2962f7133f391 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Wed, 4 Feb 2026 17:20:22 -0800 Subject: [PATCH 04/26] upstream fetch Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f3d1923465b9..3a1a67b44c273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Added TopN selection logic for streaming terms aggregations ([#20481](https://github.com/opensearch-project/OpenSearch/pull/20481)) - Added support for Intra Segment Search ([#19704](https://github.com/opensearch-project/OpenSearch/pull/19704)) - Introduce AdditionalCodecs and EnginePlugin::getAdditionalCodecs hook to allow additional Codec registration ([#20411](https://github.com/opensearch-project/OpenSearch/pull/20411)) +- Support expected cluster name with validation in CCS Sniff mode ([#20532](https://github.com/opensearch-project/OpenSearch/pull/20532)) - Added intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) ### Changed From 42be1ed9cee36e48f0536ac769c3d101e2c5c398 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Wed, 4 Feb 2026 17:24:09 -0800 Subject: [PATCH 05/26] update CHANGELOG.md Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a1a67b44c273..4843a2803b93c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Added support for Intra Segment Search ([#19704](https://github.com/opensearch-project/OpenSearch/pull/19704)) - Introduce AdditionalCodecs and EnginePlugin::getAdditionalCodecs hook to allow additional Codec registration ([#20411](https://github.com/opensearch-project/OpenSearch/pull/20411)) - Support expected cluster name with validation in CCS Sniff mode ([#20532](https://github.com/opensearch-project/OpenSearch/pull/20532)) -- Added intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) +- Add intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) ### Changed - Move Randomness from server to libs/common ([#20570](https://github.com/opensearch-project/OpenSearch/pull/20570)) From d654cc2bef9c3c7785358a7265c1a3e80970671c Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Tue, 10 Feb 2026 13:24:01 -0800 Subject: [PATCH 06/26] Upstream fetch Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4843a2803b93c..40782f2657822 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Added support for Intra Segment Search ([#19704](https://github.com/opensearch-project/OpenSearch/pull/19704)) - Introduce AdditionalCodecs and EnginePlugin::getAdditionalCodecs hook to allow additional Codec registration ([#20411](https://github.com/opensearch-project/OpenSearch/pull/20411)) - Support expected cluster name with validation in CCS Sniff mode ([#20532](https://github.com/opensearch-project/OpenSearch/pull/20532)) +- Add security policy to allow `accessUnixDomainSocket` in `transport-grpc` module ([#20463](https://github.com/opensearch-project/OpenSearch/pull/20463)) +- [Workload Management] Enhance Scroll API support for autotagging ([#20151](https://github.com/opensearch-project/OpenSearch/pull/20151)) - Add intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) ### Changed From f0db14edd111b391bd57a205276f1fa7cf8bc9bd Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Wed, 11 Feb 2026 13:15:21 -0800 Subject: [PATCH 07/26] Add tests Signed-off-by: Prudhvi Godithi --- .../search/aggregations/metrics/AvgIT.java | 86 +++++++++++++++++++ .../aggregations/metrics/CardinalityIT.java | 10 ++- .../search/aggregations/metrics/MaxIT.java | 86 +++++++++++++++++++ .../search/aggregations/metrics/MinIT.java | 86 +++++++++++++++++++ .../search/aggregations/metrics/StatsIT.java | 20 +++++ .../search/aggregations/metrics/SumIT.java | 20 +++++ .../aggregations/metrics/ValueCountIT.java | 10 ++- .../opensearch/test/OpenSearchTestCase.java | 6 +- 8 files changed, 318 insertions(+), 6 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java new file mode 100644 index 0000000000000..d29484987390c --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.metrics; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; + +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY; +import static org.opensearch.search.aggregations.AggregationBuilders.avg; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +/** + * Integration tests for avg aggregation with concurrent segment search partition strategies. + */ +public class AvgIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { + + public AvgIT(Settings staticSettings) { + super(staticSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "segment").build() }, + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "force").build() }, + new Object[] { + Settings.builder() + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "balanced") + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE.getKey(), 1000) + .build() } + ); + } + + public void testAvgAggregation() throws Exception { + createIndex("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + int totalDocs = 5000; + for (int i = 0; i < totalDocs; i++) { + client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) { + refresh(); + } + } + refresh(); + indexRandomForConcurrentSearch("test"); + SearchResponse response = client().prepareSearch("test").addAggregation(avg("avg_agg").field("value")).get(); + assertSearchResponse(response); + Avg avgAgg = response.getAggregations().get("avg_agg"); + assertThat(avgAgg, notNullValue()); + assertThat(avgAgg.getValue(), equalTo(2500.5)); + } + + public void testAvgAggregationMultipleShards() throws Exception { + createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0).build()); + int totalDocs = 5000; + for (int i = 0; i < totalDocs; i++) { + client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) { + refresh(); + } + } + refresh(); + indexRandomForConcurrentSearch("test"); + SearchResponse response = client().prepareSearch("test").addAggregation(avg("avg_agg").field("value")).get(); + assertSearchResponse(response); + Avg avgAgg = response.getAggregations().get("avg_agg"); + assertThat(avgAgg, notNullValue()); + assertThat(avgAgg.getValue(), equalTo(2500.5)); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java index b2ed689622e7d..12091a86448df 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java @@ -62,6 +62,8 @@ import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.search.SearchService.CARDINALITY_AGGREGATION_PRUNING_THRESHOLD; import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY; import static org.opensearch.search.aggregations.AggregationBuilders.cardinality; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -82,7 +84,13 @@ public CardinalityIT(Settings staticSettings) { public static Collection parameters() { return Arrays.asList( new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "segment").build() }, + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "force").build() }, + new Object[] { + Settings.builder() + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "balanced") + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE.getKey(), 1000) + .build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java new file mode 100644 index 0000000000000..789544411cce3 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.metrics; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; + +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY; +import static org.opensearch.search.aggregations.AggregationBuilders.max; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +/** + * Integration tests for max aggregation with concurrent segment search partition strategies. + */ +public class MaxIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { + + public MaxIT(Settings staticSettings) { + super(staticSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "segment").build() }, + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "force").build() }, + new Object[] { + Settings.builder() + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "balanced") + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE.getKey(), 1000) + .build() } + ); + } + + public void testMaxAggregation() throws Exception { + createIndex("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + int totalDocs = 5000; + for (int i = 0; i < totalDocs; i++) { + client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) { + refresh(); + } + } + refresh(); + indexRandomForConcurrentSearch("test"); + SearchResponse response = client().prepareSearch("test").addAggregation(max("max_agg").field("value")).get(); + assertSearchResponse(response); + Max maxAgg = response.getAggregations().get("max_agg"); + assertThat(maxAgg, notNullValue()); + assertThat(maxAgg.getValue(), equalTo(5000.0)); + } + + public void testMaxAggregationMultipleShards() throws Exception { + createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0).build()); + int totalDocs = 5000; + for (int i = 0; i < totalDocs; i++) { + client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) { + refresh(); + } + } + refresh(); + indexRandomForConcurrentSearch("test"); + SearchResponse response = client().prepareSearch("test").addAggregation(max("max_agg").field("value")).get(); + assertSearchResponse(response); + Max maxAgg = response.getAggregations().get("max_agg"); + assertThat(maxAgg, notNullValue()); + assertThat(maxAgg.getValue(), equalTo(5000.0)); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java new file mode 100644 index 0000000000000..5ded6166cab5e --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.metrics; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; + +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY; +import static org.opensearch.search.aggregations.AggregationBuilders.min; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +/** + * Integration tests for min aggregation with concurrent segment search partition strategies. + */ +public class MinIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { + + public MinIT(Settings staticSettings) { + super(staticSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "segment").build() }, + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "force").build() }, + new Object[] { + Settings.builder() + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "balanced") + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE.getKey(), 1000) + .build() } + ); + } + + public void testMinAggregation() throws Exception { + createIndex("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + int totalDocs = 5000; + for (int i = 0; i < totalDocs; i++) { + client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) { + refresh(); + } + } + refresh(); + indexRandomForConcurrentSearch("test"); + SearchResponse response = client().prepareSearch("test").addAggregation(min("min_agg").field("value")).get(); + assertSearchResponse(response); + Min minAgg = response.getAggregations().get("min_agg"); + assertThat(minAgg, notNullValue()); + assertThat(minAgg.getValue(), equalTo(1.0)); + } + + public void testMinAggregationMultipleShards() throws Exception { + createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0).build()); + int totalDocs = 5000; + for (int i = 0; i < totalDocs; i++) { + client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) { + refresh(); + } + } + refresh(); + indexRandomForConcurrentSearch("test"); + SearchResponse response = client().prepareSearch("test").addAggregation(min("min_agg").field("value")).get(); + assertSearchResponse(response); + Min minAgg = response.getAggregations().get("min_agg"); + assertThat(minAgg, notNullValue()); + assertThat(minAgg.getValue(), equalTo(1.0)); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java index c7df3efd6ac1e..a8deb6f609b8f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java @@ -31,6 +31,8 @@ package org.opensearch.search.aggregations.metrics; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.ShardSearchFailure; @@ -46,12 +48,16 @@ import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.terms.Terms; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY; import static org.opensearch.search.aggregations.AggregationBuilders.filter; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -70,6 +76,20 @@ public StatsIT(Settings staticSettings) { super(staticSettings); } + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "segment").build() }, + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "force").build() }, + new Object[] { + Settings.builder() + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "balanced") + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE.getKey(), 1000) + .build() } + ); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(AggregationTestScriptsPlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java index 8886713caa42b..3568bc7dee66b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java @@ -31,6 +31,8 @@ package org.opensearch.search.aggregations.metrics; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; @@ -46,12 +48,16 @@ import org.hamcrest.core.IsNull; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY; import static org.opensearch.search.aggregations.AggregationBuilders.filter; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -72,6 +78,20 @@ public SumIT(Settings staticSettings) { super(staticSettings); } + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "segment").build() }, + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "force").build() }, + new Object[] { + Settings.builder() + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "balanced") + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE.getKey(), 1000) + .build() } + ); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(MetricAggScriptPlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java index c17b2108d4077..c18e51246e9ed 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java @@ -56,6 +56,8 @@ import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY; import static org.opensearch.search.aggregations.AggregationBuilders.count; import static org.opensearch.search.aggregations.AggregationBuilders.filter; import static org.opensearch.search.aggregations.AggregationBuilders.global; @@ -82,7 +84,13 @@ public ValueCountIT(Settings staticSettings) { public static Collection parameters() { return Arrays.asList( new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "segment").build() }, + new Object[] { Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "force").build() }, + new Object[] { + Settings.builder() + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY.getKey(), "balanced") + .put(CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE.getKey(), 1000) + .build() } ); } diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index e78e9504a3ef7..ea8608c13f5be 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -63,7 +63,6 @@ import org.apache.lucene.tests.util.TestUtil; import org.apache.lucene.tests.util.TimeUnits; import org.opensearch.Version; -import org.opensearch.bootstrap.BootstrapForTesting; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.PersistedStateRegistry; @@ -137,7 +136,6 @@ import org.opensearch.test.junit.listeners.LoggingListener; import org.opensearch.test.junit.listeners.ReproduceInfoPrinter; import org.opensearch.threadpool.ThreadPool; -import org.opensearch.transport.TransportService; import org.opensearch.transport.client.Client; import org.opensearch.transport.client.Requests; import org.opensearch.transport.nio.MockNioTransportPlugin; @@ -329,8 +327,8 @@ public void append(LogEvent event) { Configurator.shutdown(context); })); - BootstrapForTesting.ensureInitialized(); - TransportService.ensureClassloaded(); // ensure server streamables are registered + // BootstrapForTesting.ensureInitialized(); + // TransportService.ensureClassloaded(); // ensure server streamables are registered // filter out joda timezones that are deprecated for the java time migration List jodaTZIds = DateTimeZone.getAvailableIDs() From aafbc281effac7b1a572f7569e71c1577e7a07cb Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Wed, 11 Feb 2026 13:17:04 -0800 Subject: [PATCH 08/26] Add tests Signed-off-by: Prudhvi Godithi --- .../main/java/org/opensearch/test/OpenSearchTestCase.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index ea8608c13f5be..e78e9504a3ef7 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -63,6 +63,7 @@ import org.apache.lucene.tests.util.TestUtil; import org.apache.lucene.tests.util.TimeUnits; import org.opensearch.Version; +import org.opensearch.bootstrap.BootstrapForTesting; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.PersistedStateRegistry; @@ -136,6 +137,7 @@ import org.opensearch.test.junit.listeners.LoggingListener; import org.opensearch.test.junit.listeners.ReproduceInfoPrinter; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; import org.opensearch.transport.client.Client; import org.opensearch.transport.client.Requests; import org.opensearch.transport.nio.MockNioTransportPlugin; @@ -327,8 +329,8 @@ public void append(LogEvent event) { Configurator.shutdown(context); })); - // BootstrapForTesting.ensureInitialized(); - // TransportService.ensureClassloaded(); // ensure server streamables are registered + BootstrapForTesting.ensureInitialized(); + TransportService.ensureClassloaded(); // ensure server streamables are registered // filter out joda timezones that are deprecated for the java time migration List jodaTZIds = DateTimeZone.getAvailableIDs() From 57af42e1dde16d9f52feeeaeb2b88a9500977a5a Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Tue, 17 Feb 2026 14:57:34 -0800 Subject: [PATCH 09/26] Fix tests Signed-off-by: Prudhvi Godithi --- .../aggregations/metrics/CardinalityIT.java | 15 +++++++++++++ .../search/aggregations/metrics/StatsIT.java | 21 +++++++++++++++++++ .../search/aggregations/metrics/SumIT.java | 17 +++++++++++++++ .../aggregations/metrics/ValueCountIT.java | 15 +++++++++++++ 4 files changed, 68 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java index 12091a86448df..384a3868a42bd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java @@ -674,4 +674,19 @@ public void testScriptCaching() throws Exception { ); internalCluster().wipeIndices("cache_test_idx"); } + + public void testCardinalityWithIntraSegmentPartitioning() throws Exception { + createIndex("intra_test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + for (int i = 0; i < 5000; i++) { + client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("category", i % 100).get(); + if (i % 2500 == 2499) refresh(); + } + refresh(); + indexRandomForConcurrentSearch("intra_test"); + SearchResponse response = client().prepareSearch("intra_test").addAggregation(cardinality("cardinality").field("category")).get(); + Cardinality cardinalityAgg = response.getAggregations().get("cardinality"); + assertThat(cardinalityAgg, notNullValue()); + assertThat(cardinalityAgg.getValue(), equalTo(100L)); + client().admin().indices().prepareDelete("intra_test").get(); + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java index a8deb6f609b8f..b3f26f4402bd2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java @@ -410,4 +410,25 @@ public void testScriptCaching() throws Exception { ); internalCluster().wipeIndices("cache_test_idx"); } + + public void testStatsWithIntraSegmentPartitioning() throws Exception { + createIndex("intra_test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + long expectedSum = 0; + for (int i = 0; i < 5000; i++) { + expectedSum += (i + 1); + client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) refresh(); + } + refresh(); + indexRandomForConcurrentSearch("intra_test"); + SearchResponse response = client().prepareSearch("intra_test").addAggregation(stats("stats").field("value")).get(); + Stats statsAgg = response.getAggregations().get("stats"); + assertThat(statsAgg, notNullValue()); + assertThat(statsAgg.getCount(), equalTo(5000L)); + assertThat(statsAgg.getMin(), equalTo(1.0)); + assertThat(statsAgg.getMax(), equalTo(5000.0)); + assertThat(statsAgg.getSum(), equalTo((double) expectedSum)); + assertThat(statsAgg.getAvg(), equalTo((double) expectedSum / 5000)); + client().admin().indices().prepareDelete("intra_test").get(); + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java index 3568bc7dee66b..38b330c857feb 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java @@ -431,4 +431,21 @@ public void testFieldAliasInSubAggregation() { assertThat(sum, notNullValue()); assertThat(sum.getValue(), equalTo(50.5)); } + + public void testSumWithIntraSegmentPartitioning() throws Exception { + createIndex("intra_test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + long expectedSum = 0; + for (int i = 0; i < 5000; i++) { + expectedSum += (i + 1); + client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) refresh(); + } + refresh(); + indexRandomForConcurrentSearch("intra_test"); + SearchResponse response = client().prepareSearch("intra_test").addAggregation(sum("sum").field("value")).get(); + Sum sumAgg = response.getAggregations().get("sum"); + assertThat(sumAgg, notNullValue()); + assertThat(sumAgg.getValue(), equalTo((double) expectedSum)); + client().admin().indices().prepareDelete("intra_test").get(); + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java index c18e51246e9ed..ac76525dbcf88 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java @@ -424,4 +424,19 @@ public void testOrderByEmptyAggregation() throws Exception { } } + + public void testValueCountWithIntraSegmentPartitioning() throws Exception { + createIndex("intra_test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + for (int i = 0; i < 5000; i++) { + client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) refresh(); + } + refresh(); + indexRandomForConcurrentSearch("intra_test"); + SearchResponse response = client().prepareSearch("intra_test").addAggregation(count("count").field("value")).get(); + ValueCount countAgg = response.getAggregations().get("count"); + assertThat(countAgg, notNullValue()); + assertThat(countAgg.getValue(), equalTo(5000L)); + client().admin().indices().prepareDelete("intra_test").get(); + } } From 1ea8af56ea5842fecdcad6606fbbc47e36e01310 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Wed, 18 Feb 2026 18:24:01 -0800 Subject: [PATCH 10/26] Temp revert intra for cardinality agg Signed-off-by: Prudhvi Godithi --- .../aggregations/metrics/CardinalityAggregatorFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorFactory.java index dbccaa2c1b7d0..ce7ffbb432e06 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorFactory.java @@ -162,10 +162,10 @@ protected boolean supportsConcurrentSegmentSearch() { return true; } - @Override + /*@Override protected boolean supportsIntraSegmentSearch() { return true; - } + }*/ private int precision() { return precisionThreshold == null From 7187fc31e08ebe3b2a3b94ec95522caa0e8fe616 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Wed, 18 Feb 2026 20:47:30 -0800 Subject: [PATCH 11/26] Enable intra for cardinality agg Signed-off-by: Prudhvi Godithi --- .../aggregations/metrics/CardinalityAggregatorFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorFactory.java index ce7ffbb432e06..dbccaa2c1b7d0 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorFactory.java @@ -162,10 +162,10 @@ protected boolean supportsConcurrentSegmentSearch() { return true; } - /*@Override + @Override protected boolean supportsIntraSegmentSearch() { return true; - }*/ + } private int precision() { return precisionThreshold == null From 0e823af3c563ee95e2a7d2d21748a15dd025b5d0 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Fri, 20 Feb 2026 14:00:12 -0800 Subject: [PATCH 12/26] test: Add unit tests for intra-segment search support Signed-off-by: Prudhvi Godithi --- .../metrics/AvgAggregatorTests.java | 19 ++++++++++++++++ .../metrics/CardinalityAggregatorTests.java | 19 ++++++++++++++++ .../metrics/MaxAggregatorTests.java | 19 ++++++++++++++++ .../metrics/MinAggregatorTests.java | 19 ++++++++++++++++ .../metrics/StatsAggregatorTests.java | 19 ++++++++++++++++ .../metrics/SumAggregatorTests.java | 18 +++++++++++++++ .../metrics/ValueCountAggregatorTests.java | 22 +++++++++++++++++++ 7 files changed, 135 insertions(+) diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/AvgAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/AvgAggregatorTests.java index bfe93286a564a..9d6e4458d32ff 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/AvgAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/AvgAggregatorTests.java @@ -58,6 +58,7 @@ import org.opensearch.script.ScriptType; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorTestCase; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.bucket.filter.Filter; @@ -716,4 +717,22 @@ protected List getSupportedValuesSourceTypes() { protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { return new AvgAggregationBuilder("foo").field(fieldName); } + + public void testSupportsIntraSegmentSearch() throws IOException { + MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType("value", NumberFieldMapper.NumberType.LONG); + try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + indexWriter.addDocument(singleton(new NumericDocValuesField("value", 1))); + try (IndexReader reader = indexWriter.getReader()) { + IndexSearcher searcher = newIndexSearcher(reader); + AggregatorFactories factories = AggregatorFactories.builder() + .addAggregator(new AvgAggregationBuilder("test").field("value")) + .build( + createSearchContext(searcher, createIndexSettings(), new MatchAllDocsQuery(), null, fieldType) + .getQueryShardContext(), + null + ); + assertTrue(factories.allFactoriesSupportIntraSegmentSearch()); + } + } + } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorTests.java index 4b364883d8d5a..eab6184000d0a 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/CardinalityAggregatorTests.java @@ -65,6 +65,7 @@ import org.opensearch.index.mapper.RangeFieldMapper; import org.opensearch.index.mapper.RangeType; import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorTestCase; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.LeafBucketCollector; @@ -860,4 +861,22 @@ public void testMemoryLimitExceptionSingleton() { // Test that it has no stack trace for performance assertEquals(0, ex1.getStackTrace().length); } + + public void testSupportsIntraSegmentSearch() throws IOException { + MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType("value", NumberFieldMapper.NumberType.LONG); + try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + indexWriter.addDocument(singleton(new NumericDocValuesField("value", 1))); + try (IndexReader reader = indexWriter.getReader()) { + IndexSearcher searcher = newIndexSearcher(reader); + AggregatorFactories factories = AggregatorFactories.builder() + .addAggregator(new CardinalityAggregationBuilder("test").field("value")) + .build( + createSearchContext(searcher, createIndexSettings(), new MatchAllDocsQuery(), null, fieldType) + .getQueryShardContext(), + null + ); + assertTrue(factories.allFactoriesSupportIntraSegmentSearch()); + } + } + } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/MaxAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/MaxAggregatorTests.java index ddd43866f55d4..d10395be1c5c3 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/MaxAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/MaxAggregatorTests.java @@ -71,6 +71,7 @@ import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregationBuilders; import org.opensearch.search.aggregations.Aggregator; +import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorTestCase; import org.opensearch.search.aggregations.BucketCollector; import org.opensearch.search.aggregations.BucketOrder; @@ -1027,4 +1028,22 @@ public void testDoReset() throws IOException { indexReader.close(); directory.close(); } + + public void testSupportsIntraSegmentSearch() throws IOException { + MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType("value", NumberFieldMapper.NumberType.LONG); + try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + indexWriter.addDocument(singleton(new NumericDocValuesField("value", 1))); + try (IndexReader reader = indexWriter.getReader()) { + IndexSearcher searcher = newIndexSearcher(reader); + AggregatorFactories factories = AggregatorFactories.builder() + .addAggregator(new MaxAggregationBuilder("test").field("value")) + .build( + createSearchContext(searcher, createIndexSettings(), new MatchAllDocsQuery(), null, fieldType) + .getQueryShardContext(), + null + ); + assertTrue(factories.allFactoriesSupportIntraSegmentSearch()); + } + } + } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/MinAggregatorTests.java index 225831ada671a..3a1231b5a6e5d 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/MinAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/MinAggregatorTests.java @@ -77,6 +77,7 @@ import org.opensearch.script.ScriptService; import org.opensearch.script.ScriptType; import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorTestCase; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.bucket.filter.Filter; @@ -808,4 +809,22 @@ public void testDoReset() throws IOException { indexReader.close(); directory.close(); } + + public void testSupportsIntraSegmentSearch() throws IOException { + MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType("value", NumberFieldMapper.NumberType.LONG); + try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + indexWriter.addDocument(singleton(new NumericDocValuesField("value", 1))); + try (IndexReader reader = indexWriter.getReader()) { + IndexSearcher searcher = newIndexSearcher(reader); + AggregatorFactories factories = AggregatorFactories.builder() + .addAggregator(new MinAggregationBuilder("test").field("value")) + .build( + createSearchContext(searcher, createIndexSettings(), new MatchAllDocsQuery(), null, fieldType) + .getQueryShardContext(), + null + ); + assertTrue(factories.allFactoriesSupportIntraSegmentSearch()); + } + } + } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/StatsAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/StatsAggregatorTests.java index f8a6cf1ef07af..13ace3efe5785 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/StatsAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/StatsAggregatorTests.java @@ -55,6 +55,7 @@ import org.opensearch.script.ScriptService; import org.opensearch.script.ScriptType; import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorTestCase; import org.opensearch.search.aggregations.LeafBucketCollector; import org.opensearch.search.aggregations.support.AggregationInspectionHelper; @@ -518,4 +519,22 @@ public void testCollectRange() throws IOException { } } } + + public void testSupportsIntraSegmentSearch() throws IOException { + MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType("value", NumberFieldMapper.NumberType.LONG); + try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + indexWriter.addDocument(singleton(new NumericDocValuesField("value", 1))); + try (IndexReader reader = indexWriter.getReader()) { + IndexSearcher searcher = newIndexSearcher(reader); + AggregatorFactories factories = AggregatorFactories.builder() + .addAggregator(new StatsAggregationBuilder("test").field("value")) + .build( + createSearchContext(searcher, createIndexSettings(), new MatchAllDocsQuery(), null, fieldType) + .getQueryShardContext(), + null + ); + assertTrue(factories.allFactoriesSupportIntraSegmentSearch()); + } + } + } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/SumAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/SumAggregatorTests.java index 1eecd43aaee2f..db07c422174a1 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/SumAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/SumAggregatorTests.java @@ -63,6 +63,7 @@ import org.opensearch.script.ScriptService; import org.opensearch.script.ScriptType; import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorTestCase; import org.opensearch.search.aggregations.support.AggregationInspectionHelper; import org.opensearch.search.aggregations.support.CoreValuesSourceType; @@ -440,4 +441,21 @@ private static MappedFieldType defaultFieldType() { private static MappedFieldType defaultFieldType(NumberType numberType) { return new NumberFieldMapper.NumberFieldType(FIELD_NAME, numberType); } + + public void testSupportsIntraSegmentSearch() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + indexWriter.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 1))); + try (IndexReader reader = indexWriter.getReader()) { + IndexSearcher searcher = newIndexSearcher(reader); + AggregatorFactories factories = AggregatorFactories.builder() + .addAggregator(new SumAggregationBuilder("test").field(FIELD_NAME)) + .build( + createSearchContext(searcher, createIndexSettings(), new MatchAllDocsQuery(), null, defaultFieldType()) + .getQueryShardContext(), + null + ); + assertTrue(factories.allFactoriesSupportIntraSegmentSearch()); + } + } + } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/ValueCountAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/ValueCountAggregatorTests.java index 7fbd1430a0c2a..7bc06d2f01fa8 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/ValueCountAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/ValueCountAggregatorTests.java @@ -41,9 +41,12 @@ import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.FieldExistsQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; import org.opensearch.common.CheckedConsumer; @@ -66,6 +69,7 @@ import org.opensearch.script.ScriptService; import org.opensearch.script.ScriptType; import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorTestCase; import org.opensearch.search.aggregations.support.AggregationInspectionHelper; import org.opensearch.search.aggregations.support.CoreValuesSourceType; @@ -448,4 +452,22 @@ private static MappedFieldType createMappedFieldType(String name, ValueType valu throw new IllegalArgumentException("Test does not support value type [" + valueType + "]"); } } + + public void testSupportsIntraSegmentSearch() throws IOException { + MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType("value", NumberFieldMapper.NumberType.LONG); + try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + indexWriter.addDocument(singleton(new NumericDocValuesField("value", 1))); + try (IndexReader reader = indexWriter.getReader()) { + IndexSearcher searcher = newIndexSearcher(reader); + AggregatorFactories factories = AggregatorFactories.builder() + .addAggregator(new ValueCountAggregationBuilder("test").field("value")) + .build( + createSearchContext(searcher, createIndexSettings(), new MatchAllDocsQuery(), null, fieldType) + .getQueryShardContext(), + null + ); + assertTrue(factories.allFactoriesSupportIntraSegmentSearch()); + } + } + } } From b6954da616ddfee34b26956965204ba12d570503 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Fri, 20 Feb 2026 14:05:51 -0800 Subject: [PATCH 13/26] Fix commit issue Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40782f2657822..7e4be7b20319d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,29 +12,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add range validations in query builder and field mapper ([#20497](https://github.com/opensearch-project/OpenSearch/issues/20497)) - [Workload Management] Enhance Scroll API support for autotagging ([#20151](https://github.com/opensearch-project/OpenSearch/pull/20151)) - Add indices to search request slowlog ([#20588](https://github.com/opensearch-project/OpenSearch/pull/20588)) -- Add support for fields containing dots in their name as literals ([#19958](https://github.com/opensearch-project/OpenSearch/pull/19958)) -- Add support for forward translog reading ([#20163](https://github.com/opensearch-project/OpenSearch/pull/20163)) -- Added public getter method in `SourceFieldMapper` to return excluded field ([#20205](https://github.com/opensearch-project/OpenSearch/pull/20205)) -- Add integ test for simulating node join left event when data node cluster state publication lag because the cluster applier thread being busy ([#19907](https://github.com/opensearch-project/OpenSearch/pull/19907)). -- Relax jar hell check when extended plugins share transitive dependencies ([#20103](https://github.com/opensearch-project/OpenSearch/pull/20103)) -- Added public getter method in `SourceFieldMapper` to return included field ([#20290](https://github.com/opensearch-project/OpenSearch/pull/20290)) -- Support for HTTP/3 (server side) ([#20017](https://github.com/opensearch-project/OpenSearch/pull/20017)) -- Add circuit breaker support for gRPC transport to prevent out-of-memory errors ([#20203](https://github.com/opensearch-project/OpenSearch/pull/20203)) -- Add index-level-encryption support for snapshots and remote-store ([#20095](https://github.com/opensearch-project/OpenSearch/pull/20095)) -- Adding BackWardCompatibility test for remote publication enabled cluster ([#20221](https://github.com/opensearch-project/OpenSearch/pull/20221)) -- Support for hll field mapper to support cardinality rollups ([#20129](https://github.com/opensearch-project/OpenSearch/pull/20129)) -- Add tracing support for StreamingRestChannel ([#20361](https://github.com/opensearch-project/OpenSearch/pull/20361)) -- Introduce new libs/netty4 module to share common implementation between netty-based plugins and modules (transport-netty4, transport-reactor-netty4) ([#20447](https://github.com/opensearch-project/OpenSearch/pull/20447)) -- Add validation to make crypto store settings immutable ([#20123](https://github.com/opensearch-project/OpenSearch/pull/20123)) -- Introduce concurrent translog recovery to accelerate segment replication primary promotion ([#20251](https://github.com/opensearch-project/OpenSearch/pull/20251)) -- Update to `almalinux:10` ([#20482](https://github.com/opensearch-project/OpenSearch/pull/20482)) -- Add X-Request-Id to uniquely identify a search request ([#19798](https://github.com/opensearch-project/OpenSearch/pull/19798)) -- Added TopN selection logic for streaming terms aggregations ([#20481](https://github.com/opensearch-project/OpenSearch/pull/20481)) -- Added support for Intra Segment Search ([#19704](https://github.com/opensearch-project/OpenSearch/pull/19704)) -- Introduce AdditionalCodecs and EnginePlugin::getAdditionalCodecs hook to allow additional Codec registration ([#20411](https://github.com/opensearch-project/OpenSearch/pull/20411)) -- Support expected cluster name with validation in CCS Sniff mode ([#20532](https://github.com/opensearch-project/OpenSearch/pull/20532)) -- Add security policy to allow `accessUnixDomainSocket` in `transport-grpc` module ([#20463](https://github.com/opensearch-project/OpenSearch/pull/20463)) -- [Workload Management] Enhance Scroll API support for autotagging ([#20151](https://github.com/opensearch-project/OpenSearch/pull/20151)) - Add intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) ### Changed From 398a5a33f8da33e1eff551832e51b159e2329059 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Fri, 20 Feb 2026 14:14:29 -0800 Subject: [PATCH 14/26] Address github comments Signed-off-by: Prudhvi Godithi --- .../opensearch/search/aggregations/metrics/CardinalityIT.java | 2 +- .../org/opensearch/search/aggregations/metrics/StatsIT.java | 2 +- .../java/org/opensearch/search/aggregations/metrics/SumIT.java | 3 ++- .../opensearch/search/aggregations/metrics/ValueCountIT.java | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java index 384a3868a42bd..46067ee2627d9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java @@ -687,6 +687,6 @@ public void testCardinalityWithIntraSegmentPartitioning() throws Exception { Cardinality cardinalityAgg = response.getAggregations().get("cardinality"); assertThat(cardinalityAgg, notNullValue()); assertThat(cardinalityAgg.getValue(), equalTo(100L)); - client().admin().indices().prepareDelete("intra_test").get(); + internalCluster().wipeIndices("intra_test"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java index b3f26f4402bd2..6d526e87f8db6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java @@ -429,6 +429,6 @@ public void testStatsWithIntraSegmentPartitioning() throws Exception { assertThat(statsAgg.getMax(), equalTo(5000.0)); assertThat(statsAgg.getSum(), equalTo((double) expectedSum)); assertThat(statsAgg.getAvg(), equalTo((double) expectedSum / 5000)); - client().admin().indices().prepareDelete("intra_test").get(); + internalCluster().wipeIndices("intra_test"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java index 38b330c857feb..18f62c238b369 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java @@ -446,6 +446,7 @@ public void testSumWithIntraSegmentPartitioning() throws Exception { Sum sumAgg = response.getAggregations().get("sum"); assertThat(sumAgg, notNullValue()); assertThat(sumAgg.getValue(), equalTo((double) expectedSum)); - client().admin().indices().prepareDelete("intra_test").get(); + internalCluster().wipeIndices("intra_test"); + } } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java index ac76525dbcf88..f6aae0f882925 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java @@ -437,6 +437,7 @@ public void testValueCountWithIntraSegmentPartitioning() throws Exception { ValueCount countAgg = response.getAggregations().get("count"); assertThat(countAgg, notNullValue()); assertThat(countAgg.getValue(), equalTo(5000L)); - client().admin().indices().prepareDelete("intra_test").get(); + internalCluster().wipeIndices("intra_test"); + } } } From 32c81a6de20dc36c369a5cd3b52adf8f17df9f4f Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Fri, 20 Feb 2026 14:17:07 -0800 Subject: [PATCH 15/26] Address github comments Signed-off-by: Prudhvi Godithi --- .../aggregations/metrics/CardinalityIT.java | 25 +++++++------ .../search/aggregations/metrics/StatsIT.java | 35 ++++++++++--------- .../search/aggregations/metrics/SumIT.java | 28 ++++++++------- .../aggregations/metrics/ValueCountIT.java | 24 +++++++------ 4 files changed, 62 insertions(+), 50 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java index 46067ee2627d9..2db6aae7760af 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java @@ -677,16 +677,21 @@ public void testScriptCaching() throws Exception { public void testCardinalityWithIntraSegmentPartitioning() throws Exception { createIndex("intra_test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); - for (int i = 0; i < 5000; i++) { - client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("category", i % 100).get(); - if (i % 2500 == 2499) refresh(); + try { + for (int i = 0; i < 5000; i++) { + client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("category", i % 100).get(); + if (i % 2500 == 2499) refresh(); + } + refresh(); + indexRandomForConcurrentSearch("intra_test"); + SearchResponse response = client().prepareSearch("intra_test") + .addAggregation(cardinality("cardinality").field("category")) + .get(); + Cardinality cardinalityAgg = response.getAggregations().get("cardinality"); + assertThat(cardinalityAgg, notNullValue()); + assertThat(cardinalityAgg.getValue(), equalTo(100L)); + } finally { + internalCluster().wipeIndices("intra_test"); } - refresh(); - indexRandomForConcurrentSearch("intra_test"); - SearchResponse response = client().prepareSearch("intra_test").addAggregation(cardinality("cardinality").field("category")).get(); - Cardinality cardinalityAgg = response.getAggregations().get("cardinality"); - assertThat(cardinalityAgg, notNullValue()); - assertThat(cardinalityAgg.getValue(), equalTo(100L)); - internalCluster().wipeIndices("intra_test"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java index 6d526e87f8db6..a89fc5366f43a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java @@ -413,22 +413,25 @@ public void testScriptCaching() throws Exception { public void testStatsWithIntraSegmentPartitioning() throws Exception { createIndex("intra_test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); - long expectedSum = 0; - for (int i = 0; i < 5000; i++) { - expectedSum += (i + 1); - client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) refresh(); + try { + long expectedSum = 0; + for (int i = 0; i < 5000; i++) { + expectedSum += (i + 1); + client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) refresh(); + } + refresh(); + indexRandomForConcurrentSearch("intra_test"); + SearchResponse response = client().prepareSearch("intra_test").addAggregation(stats("stats").field("value")).get(); + Stats statsAgg = response.getAggregations().get("stats"); + assertThat(statsAgg, notNullValue()); + assertThat(statsAgg.getCount(), equalTo(5000L)); + assertThat(statsAgg.getMin(), equalTo(1.0)); + assertThat(statsAgg.getMax(), equalTo(5000.0)); + assertThat(statsAgg.getSum(), equalTo((double) expectedSum)); + assertThat(statsAgg.getAvg(), equalTo((double) expectedSum / 5000)); + } finally { + internalCluster().wipeIndices("intra_test"); } - refresh(); - indexRandomForConcurrentSearch("intra_test"); - SearchResponse response = client().prepareSearch("intra_test").addAggregation(stats("stats").field("value")).get(); - Stats statsAgg = response.getAggregations().get("stats"); - assertThat(statsAgg, notNullValue()); - assertThat(statsAgg.getCount(), equalTo(5000L)); - assertThat(statsAgg.getMin(), equalTo(1.0)); - assertThat(statsAgg.getMax(), equalTo(5000.0)); - assertThat(statsAgg.getSum(), equalTo((double) expectedSum)); - assertThat(statsAgg.getAvg(), equalTo((double) expectedSum / 5000)); - internalCluster().wipeIndices("intra_test"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java index 18f62c238b369..3bb6cbcb6581f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java @@ -434,19 +434,21 @@ public void testFieldAliasInSubAggregation() { public void testSumWithIntraSegmentPartitioning() throws Exception { createIndex("intra_test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); - long expectedSum = 0; - for (int i = 0; i < 5000; i++) { - expectedSum += (i + 1); - client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) refresh(); - } - refresh(); - indexRandomForConcurrentSearch("intra_test"); - SearchResponse response = client().prepareSearch("intra_test").addAggregation(sum("sum").field("value")).get(); - Sum sumAgg = response.getAggregations().get("sum"); - assertThat(sumAgg, notNullValue()); - assertThat(sumAgg.getValue(), equalTo((double) expectedSum)); - internalCluster().wipeIndices("intra_test"); + try { + long expectedSum = 0; + for (int i = 0; i < 5000; i++) { + expectedSum += (i + 1); + client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) refresh(); + } + refresh(); + indexRandomForConcurrentSearch("intra_test"); + SearchResponse response = client().prepareSearch("intra_test").addAggregation(sum("sum").field("value")).get(); + Sum sumAgg = response.getAggregations().get("sum"); + assertThat(sumAgg, notNullValue()); + assertThat(sumAgg.getValue(), equalTo((double) expectedSum)); + } finally { + internalCluster().wipeIndices("intra_test"); } } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java index f6aae0f882925..d2f43b4a25ab2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java @@ -427,17 +427,19 @@ public void testOrderByEmptyAggregation() throws Exception { public void testValueCountWithIntraSegmentPartitioning() throws Exception { createIndex("intra_test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); - for (int i = 0; i < 5000; i++) { - client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) refresh(); - } - refresh(); - indexRandomForConcurrentSearch("intra_test"); - SearchResponse response = client().prepareSearch("intra_test").addAggregation(count("count").field("value")).get(); - ValueCount countAgg = response.getAggregations().get("count"); - assertThat(countAgg, notNullValue()); - assertThat(countAgg.getValue(), equalTo(5000L)); - internalCluster().wipeIndices("intra_test"); + try { + for (int i = 0; i < 5000; i++) { + client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) refresh(); + } + refresh(); + indexRandomForConcurrentSearch("intra_test"); + SearchResponse response = client().prepareSearch("intra_test").addAggregation(count("count").field("value")).get(); + ValueCount countAgg = response.getAggregations().get("count"); + assertThat(countAgg, notNullValue()); + assertThat(countAgg.getValue(), equalTo(5000L)); + } finally { + internalCluster().wipeIndices("intra_test"); } } } From bd366f3af4ceb21b5a82e20375d71676b756d6a1 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Fri, 20 Feb 2026 14:19:44 -0800 Subject: [PATCH 16/26] Add wipeIndices to tests Signed-off-by: Prudhvi Godithi --- .../search/aggregations/metrics/AvgIT.java | 56 +++++++++++-------- .../search/aggregations/metrics/MaxIT.java | 56 +++++++++++-------- .../search/aggregations/metrics/MinIT.java | 56 +++++++++++-------- 3 files changed, 96 insertions(+), 72 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java index d29484987390c..a8ba77fb6cc28 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java @@ -50,37 +50,45 @@ public static Collection parameters() { public void testAvgAggregation() throws Exception { createIndex("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); - int totalDocs = 5000; - for (int i = 0; i < totalDocs; i++) { - client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) { - refresh(); + try { + int totalDocs = 5000; + for (int i = 0; i < totalDocs; i++) { + client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) { + refresh(); + } } + refresh(); + indexRandomForConcurrentSearch("test"); + SearchResponse response = client().prepareSearch("test").addAggregation(avg("avg_agg").field("value")).get(); + assertSearchResponse(response); + Avg avgAgg = response.getAggregations().get("avg_agg"); + assertThat(avgAgg, notNullValue()); + assertThat(avgAgg.getValue(), equalTo(2500.5)); + } finally { + internalCluster().wipeIndices("test"); } - refresh(); - indexRandomForConcurrentSearch("test"); - SearchResponse response = client().prepareSearch("test").addAggregation(avg("avg_agg").field("value")).get(); - assertSearchResponse(response); - Avg avgAgg = response.getAggregations().get("avg_agg"); - assertThat(avgAgg, notNullValue()); - assertThat(avgAgg.getValue(), equalTo(2500.5)); } public void testAvgAggregationMultipleShards() throws Exception { createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0).build()); - int totalDocs = 5000; - for (int i = 0; i < totalDocs; i++) { - client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) { - refresh(); + try { + int totalDocs = 5000; + for (int i = 0; i < totalDocs; i++) { + client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) { + refresh(); + } } + refresh(); + indexRandomForConcurrentSearch("test"); + SearchResponse response = client().prepareSearch("test").addAggregation(avg("avg_agg").field("value")).get(); + assertSearchResponse(response); + Avg avgAgg = response.getAggregations().get("avg_agg"); + assertThat(avgAgg, notNullValue()); + assertThat(avgAgg.getValue(), equalTo(2500.5)); + } finally { + internalCluster().wipeIndices("test"); } - refresh(); - indexRandomForConcurrentSearch("test"); - SearchResponse response = client().prepareSearch("test").addAggregation(avg("avg_agg").field("value")).get(); - assertSearchResponse(response); - Avg avgAgg = response.getAggregations().get("avg_agg"); - assertThat(avgAgg, notNullValue()); - assertThat(avgAgg.getValue(), equalTo(2500.5)); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java index 789544411cce3..7e016e3d48bef 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java @@ -50,37 +50,45 @@ public static Collection parameters() { public void testMaxAggregation() throws Exception { createIndex("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); - int totalDocs = 5000; - for (int i = 0; i < totalDocs; i++) { - client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) { - refresh(); + try { + int totalDocs = 5000; + for (int i = 0; i < totalDocs; i++) { + client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) { + refresh(); + } } + refresh(); + indexRandomForConcurrentSearch("test"); + SearchResponse response = client().prepareSearch("test").addAggregation(max("max_agg").field("value")).get(); + assertSearchResponse(response); + Max maxAgg = response.getAggregations().get("max_agg"); + assertThat(maxAgg, notNullValue()); + assertThat(maxAgg.getValue(), equalTo(5000.0)); + } finally { + internalCluster().wipeIndices("test"); } - refresh(); - indexRandomForConcurrentSearch("test"); - SearchResponse response = client().prepareSearch("test").addAggregation(max("max_agg").field("value")).get(); - assertSearchResponse(response); - Max maxAgg = response.getAggregations().get("max_agg"); - assertThat(maxAgg, notNullValue()); - assertThat(maxAgg.getValue(), equalTo(5000.0)); } public void testMaxAggregationMultipleShards() throws Exception { createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0).build()); - int totalDocs = 5000; - for (int i = 0; i < totalDocs; i++) { - client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) { - refresh(); + try { + int totalDocs = 5000; + for (int i = 0; i < totalDocs; i++) { + client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) { + refresh(); + } } + refresh(); + indexRandomForConcurrentSearch("test"); + SearchResponse response = client().prepareSearch("test").addAggregation(max("max_agg").field("value")).get(); + assertSearchResponse(response); + Max maxAgg = response.getAggregations().get("max_agg"); + assertThat(maxAgg, notNullValue()); + assertThat(maxAgg.getValue(), equalTo(5000.0)); + } finally { + internalCluster().wipeIndices("test"); } - refresh(); - indexRandomForConcurrentSearch("test"); - SearchResponse response = client().prepareSearch("test").addAggregation(max("max_agg").field("value")).get(); - assertSearchResponse(response); - Max maxAgg = response.getAggregations().get("max_agg"); - assertThat(maxAgg, notNullValue()); - assertThat(maxAgg.getValue(), equalTo(5000.0)); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java index 5ded6166cab5e..2000796897929 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java @@ -50,37 +50,45 @@ public static Collection parameters() { public void testMinAggregation() throws Exception { createIndex("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); - int totalDocs = 5000; - for (int i = 0; i < totalDocs; i++) { - client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) { - refresh(); + try { + int totalDocs = 5000; + for (int i = 0; i < totalDocs; i++) { + client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) { + refresh(); + } } + refresh(); + indexRandomForConcurrentSearch("test"); + SearchResponse response = client().prepareSearch("test").addAggregation(min("min_agg").field("value")).get(); + assertSearchResponse(response); + Min minAgg = response.getAggregations().get("min_agg"); + assertThat(minAgg, notNullValue()); + assertThat(minAgg.getValue(), equalTo(1.0)); + } finally { + internalCluster().wipeIndices("test"); } - refresh(); - indexRandomForConcurrentSearch("test"); - SearchResponse response = client().prepareSearch("test").addAggregation(min("min_agg").field("value")).get(); - assertSearchResponse(response); - Min minAgg = response.getAggregations().get("min_agg"); - assertThat(minAgg, notNullValue()); - assertThat(minAgg.getValue(), equalTo(1.0)); } public void testMinAggregationMultipleShards() throws Exception { createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0).build()); - int totalDocs = 5000; - for (int i = 0; i < totalDocs; i++) { - client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) { - refresh(); + try { + int totalDocs = 5000; + for (int i = 0; i < totalDocs; i++) { + client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); + if (i % 2500 == 2499) { + refresh(); + } } + refresh(); + indexRandomForConcurrentSearch("test"); + SearchResponse response = client().prepareSearch("test").addAggregation(min("min_agg").field("value")).get(); + assertSearchResponse(response); + Min minAgg = response.getAggregations().get("min_agg"); + assertThat(minAgg, notNullValue()); + assertThat(minAgg.getValue(), equalTo(1.0)); + } finally { + internalCluster().wipeIndices("test"); } - refresh(); - indexRandomForConcurrentSearch("test"); - SearchResponse response = client().prepareSearch("test").addAggregation(min("min_agg").field("value")).get(); - assertSearchResponse(response); - Min minAgg = response.getAggregations().get("min_agg"); - assertThat(minAgg, notNullValue()); - assertThat(minAgg.getValue(), equalTo(1.0)); } } From 5b63598fe73af00f3b5cc924463b0d6dd298507e Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Sun, 8 Mar 2026 20:38:50 -0700 Subject: [PATCH 17/26] Update the IT tests Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 2 +- .../search/aggregations/metrics/AvgIT.java | 42 +++++-------------- .../aggregations/metrics/CardinalityIT.java | 10 +++-- .../search/aggregations/metrics/MaxIT.java | 42 +++++-------------- .../search/aggregations/metrics/MinIT.java | 42 +++++-------------- .../search/aggregations/metrics/StatsIT.java | 19 +++++---- .../search/aggregations/metrics/SumIT.java | 11 ++--- .../aggregations/metrics/ValueCountIT.java | 10 +++-- .../test/OpenSearchIntegTestCase.java | 17 ++++++++ 9 files changed, 77 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06de711f8724a..b9410011a3929 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,10 +34,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Update to `almalinux:10` ([#20482](https://github.com/opensearch-project/OpenSearch/pull/20482)) - Add X-Request-Id to uniquely identify a search request ([#19798](https://github.com/opensearch-project/OpenSearch/pull/19798)) - Added TopN selection logic for streaming terms aggregations ([#20481](https://github.com/opensearch-project/OpenSearch/pull/20481)) -- Added support for Intra Segment Search ([#19704](https://github.com/opensearch-project/OpenSearch/pull/19704)) - Introduce AdditionalCodecs and EnginePlugin::getAdditionalCodecs hook to allow additional Codec registration ([#20411](https://github.com/opensearch-project/OpenSearch/pull/20411)) - Introduced strategy planner interfaces for indexing and deletion ([#20585](https://github.com/opensearch-project/OpenSearch/pull/20585)) - Implement FieldMappingIngestionMessageMapper for pull-based ingestion ([#20729](https://github.com/opensearch-project/OpenSearch/pull/20729)) +- Added support for Intra Segment Search ([#19704](https://github.com/opensearch-project/OpenSearch/pull/19704)) ### Changed - Move Randomness from server to libs/common ([#20570](https://github.com/opensearch-project/OpenSearch/pull/20570)) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java index a8ba77fb6cc28..6b76b81c268b8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java @@ -10,19 +10,22 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.List; import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE; import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.notNullValue; /** @@ -49,44 +52,19 @@ public static Collection parameters() { } public void testAvgAggregation() throws Exception { - createIndex("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); try { - int totalDocs = 5000; - for (int i = 0; i < totalDocs; i++) { - client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) { - refresh(); - } + List builders = new ArrayList<>(5000); + for (int i = 0; i < 5000; i++) { + builders.add(client().prepareIndex("test").setSource("value", i + 1)); } - refresh(); + indexBulkWithSegments(builders, 2); indexRandomForConcurrentSearch("test"); SearchResponse response = client().prepareSearch("test").addAggregation(avg("avg_agg").field("value")).get(); assertSearchResponse(response); Avg avgAgg = response.getAggregations().get("avg_agg"); assertThat(avgAgg, notNullValue()); - assertThat(avgAgg.getValue(), equalTo(2500.5)); - } finally { - internalCluster().wipeIndices("test"); - } - } - - public void testAvgAggregationMultipleShards() throws Exception { - createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0).build()); - try { - int totalDocs = 5000; - for (int i = 0; i < totalDocs; i++) { - client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) { - refresh(); - } - } - refresh(); - indexRandomForConcurrentSearch("test"); - SearchResponse response = client().prepareSearch("test").addAggregation(avg("avg_agg").field("value")).get(); - assertSearchResponse(response); - Avg avgAgg = response.getAggregations().get("avg_agg"); - assertThat(avgAgg, notNullValue()); - assertThat(avgAgg.getValue(), equalTo(2500.5)); + assertThat(avgAgg.getValue(), closeTo(2500.5, 0.1)); } finally { internalCluster().wipeIndices("test"); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java index 2db6aae7760af..6d823ec2ccccf 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java @@ -50,10 +50,12 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Function; @@ -676,13 +678,13 @@ public void testScriptCaching() throws Exception { } public void testCardinalityWithIntraSegmentPartitioning() throws Exception { - createIndex("intra_test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + createIndex("intra_test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); try { + List builders = new ArrayList<>(5000); for (int i = 0; i < 5000; i++) { - client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("category", i % 100).get(); - if (i % 2500 == 2499) refresh(); + builders.add(client().prepareIndex("intra_test").setSource("category", i % 100)); } - refresh(); + indexBulkWithSegments(builders, 2); indexRandomForConcurrentSearch("intra_test"); SearchResponse response = client().prepareSearch("intra_test") .addAggregation(cardinality("cardinality").field("category")) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java index 7e016e3d48bef..835389ca55e2b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java @@ -10,19 +10,22 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.List; import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE; import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY; import static org.opensearch.search.aggregations.AggregationBuilders.max; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.notNullValue; /** @@ -49,44 +52,19 @@ public static Collection parameters() { } public void testMaxAggregation() throws Exception { - createIndex("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); try { - int totalDocs = 5000; - for (int i = 0; i < totalDocs; i++) { - client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) { - refresh(); - } + List builders = new ArrayList<>(5000); + for (int i = 0; i < 5000; i++) { + builders.add(client().prepareIndex("test").setSource("value", i + 1)); } - refresh(); + indexBulkWithSegments(builders, 2); indexRandomForConcurrentSearch("test"); SearchResponse response = client().prepareSearch("test").addAggregation(max("max_agg").field("value")).get(); assertSearchResponse(response); Max maxAgg = response.getAggregations().get("max_agg"); assertThat(maxAgg, notNullValue()); - assertThat(maxAgg.getValue(), equalTo(5000.0)); - } finally { - internalCluster().wipeIndices("test"); - } - } - - public void testMaxAggregationMultipleShards() throws Exception { - createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0).build()); - try { - int totalDocs = 5000; - for (int i = 0; i < totalDocs; i++) { - client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) { - refresh(); - } - } - refresh(); - indexRandomForConcurrentSearch("test"); - SearchResponse response = client().prepareSearch("test").addAggregation(max("max_agg").field("value")).get(); - assertSearchResponse(response); - Max maxAgg = response.getAggregations().get("max_agg"); - assertThat(maxAgg, notNullValue()); - assertThat(maxAgg.getValue(), equalTo(5000.0)); + assertThat(maxAgg.getValue(), closeTo(5000.0, 0.1)); } finally { internalCluster().wipeIndices("test"); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java index 2000796897929..e1d9454c50c10 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java @@ -10,19 +10,22 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.List; import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_MIN_SEGMENT_SIZE; import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_PARTITION_STRATEGY; import static org.opensearch.search.aggregations.AggregationBuilders.min; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.notNullValue; /** @@ -49,44 +52,19 @@ public static Collection parameters() { } public void testMinAggregation() throws Exception { - createIndex("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); try { - int totalDocs = 5000; - for (int i = 0; i < totalDocs; i++) { - client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) { - refresh(); - } + List builders = new ArrayList<>(5000); + for (int i = 0; i < 5000; i++) { + builders.add(client().prepareIndex("test").setSource("value", i + 1)); } - refresh(); + indexBulkWithSegments(builders, 2); indexRandomForConcurrentSearch("test"); SearchResponse response = client().prepareSearch("test").addAggregation(min("min_agg").field("value")).get(); assertSearchResponse(response); Min minAgg = response.getAggregations().get("min_agg"); assertThat(minAgg, notNullValue()); - assertThat(minAgg.getValue(), equalTo(1.0)); - } finally { - internalCluster().wipeIndices("test"); - } - } - - public void testMinAggregationMultipleShards() throws Exception { - createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0).build()); - try { - int totalDocs = 5000; - for (int i = 0; i < totalDocs; i++) { - client().prepareIndex("test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) { - refresh(); - } - } - refresh(); - indexRandomForConcurrentSearch("test"); - SearchResponse response = client().prepareSearch("test").addAggregation(min("min_agg").field("value")).get(); - assertSearchResponse(response); - Min minAgg = response.getAggregations().get("min_agg"); - assertThat(minAgg, notNullValue()); - assertThat(minAgg.getValue(), equalTo(1.0)); + assertThat(minAgg.getValue(), closeTo(1.0, 0.1)); } finally { internalCluster().wipeIndices("test"); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java index a89fc5366f43a..5cb57496b1270 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java @@ -34,6 +34,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.common.settings.Settings; @@ -48,6 +49,7 @@ import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.terms.Terms; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -66,6 +68,7 @@ import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -412,24 +415,24 @@ public void testScriptCaching() throws Exception { } public void testStatsWithIntraSegmentPartitioning() throws Exception { - createIndex("intra_test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + createIndex("intra_test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); try { long expectedSum = 0; + List builders = new ArrayList<>(5000); for (int i = 0; i < 5000; i++) { expectedSum += (i + 1); - client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) refresh(); + builders.add(client().prepareIndex("intra_test").setSource("value", i + 1)); } - refresh(); + indexBulkWithSegments(builders, 2); indexRandomForConcurrentSearch("intra_test"); SearchResponse response = client().prepareSearch("intra_test").addAggregation(stats("stats").field("value")).get(); Stats statsAgg = response.getAggregations().get("stats"); assertThat(statsAgg, notNullValue()); assertThat(statsAgg.getCount(), equalTo(5000L)); - assertThat(statsAgg.getMin(), equalTo(1.0)); - assertThat(statsAgg.getMax(), equalTo(5000.0)); - assertThat(statsAgg.getSum(), equalTo((double) expectedSum)); - assertThat(statsAgg.getAvg(), equalTo((double) expectedSum / 5000)); + assertThat(statsAgg.getMin(), closeTo(1.0, 0.1)); + assertThat(statsAgg.getMax(), closeTo(5000.0, 0.1)); + assertThat(statsAgg.getSum(), closeTo((double) expectedSum, 0.1)); + assertThat(statsAgg.getAvg(), closeTo((double) expectedSum / 5000, 0.1)); } finally { internalCluster().wipeIndices("intra_test"); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java index 3bb6cbcb6581f..9fd301cdd6b76 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java @@ -69,6 +69,7 @@ import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -433,20 +434,20 @@ public void testFieldAliasInSubAggregation() { } public void testSumWithIntraSegmentPartitioning() throws Exception { - createIndex("intra_test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + createIndex("intra_test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); try { long expectedSum = 0; + List builders = new ArrayList<>(5000); for (int i = 0; i < 5000; i++) { expectedSum += (i + 1); - client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) refresh(); + builders.add(client().prepareIndex("intra_test").setSource("value", i + 1)); } - refresh(); + indexBulkWithSegments(builders, 2); indexRandomForConcurrentSearch("intra_test"); SearchResponse response = client().prepareSearch("intra_test").addAggregation(sum("sum").field("value")).get(); Sum sumAgg = response.getAggregations().get("sum"); assertThat(sumAgg, notNullValue()); - assertThat(sumAgg.getValue(), equalTo((double) expectedSum)); + assertThat(sumAgg.getValue(), closeTo((double) expectedSum, 0.1)); } finally { internalCluster().wipeIndices("intra_test"); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java index d2f43b4a25ab2..f9ab1478d3f2e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java @@ -33,6 +33,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; import org.opensearch.plugins.Plugin; @@ -46,6 +47,7 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -426,13 +428,13 @@ public void testOrderByEmptyAggregation() throws Exception { } public void testValueCountWithIntraSegmentPartitioning() throws Exception { - createIndex("intra_test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + createIndex("intra_test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); try { + List builders = new ArrayList<>(5000); for (int i = 0; i < 5000; i++) { - client().prepareIndex("intra_test").setId(String.valueOf(i)).setSource("value", i + 1).get(); - if (i % 2500 == 2499) refresh(); + builders.add(client().prepareIndex("intra_test").setSource("value", i + 1)); } - refresh(); + indexBulkWithSegments(builders, 2); indexRandomForConcurrentSearch("intra_test"); SearchResponse response = client().prepareSearch("intra_test").addAggregation(count("count").field("value")).get(); ValueCount countAgg = response.getAggregations().get("count"); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index a2bbc39a878cb..7393db9a7b603 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -1646,6 +1646,23 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma } } + /** + * Indexes documents in bulk across the specified number of segments. Documents are evenly distributed + * across segments with a refresh between each batch to create segment boundaries. This is useful for + * tests that need multiple predefined segments for testing. + * + * @param builders the documents to index + * @param numSegments the number of segments to create + */ + public void indexBulkWithSegments(List builders, int numSegments) throws InterruptedException { + int batchSize = builders.size() / numSegments; + for (int seg = 0; seg < numSegments; seg++) { + int from = seg * batchSize; + int to = (seg == numSegments - 1) ? builders.size() : from + batchSize; + indexRandom(true, false, builders.subList(from, to)); + } + } + /* * This method ingests bogus documents for the given indices such that multiple slices * are formed. This is useful for testing with the concurrent search use-case as it creates From 3eb08649db4db14c7fec25b1d46eff41105901ee Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Sun, 8 Mar 2026 20:46:28 -0700 Subject: [PATCH 18/26] Update the IT tests Signed-off-by: Prudhvi Godithi --- .../main/java/org/opensearch/test/OpenSearchIntegTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 7393db9a7b603..516ef41fba206 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -1659,7 +1659,7 @@ public void indexBulkWithSegments(List builders, int numSeg for (int seg = 0; seg < numSegments; seg++) { int from = seg * batchSize; int to = (seg == numSegments - 1) ? builders.size() : from + batchSize; - indexRandom(true, false, builders.subList(from, to)); + indexRandom(true, false, false, builders.subList(from, to)); } } From 8b306ac601bcaa932db484dbee29ec68c958386e Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Mon, 9 Mar 2026 08:56:17 -0700 Subject: [PATCH 19/26] Fetch upstream Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index deebbf7dbc71a..b7c437dc777ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,8 +37,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Introduce AdditionalCodecs and EnginePlugin::getAdditionalCodecs hook to allow additional Codec registration ([#20411](https://github.com/opensearch-project/OpenSearch/pull/20411)) - Introduced strategy planner interfaces for indexing and deletion ([#20585](https://github.com/opensearch-project/OpenSearch/pull/20585)) - Implement FieldMappingIngestionMessageMapper for pull-based ingestion ([#20729](https://github.com/opensearch-project/OpenSearch/pull/20729)) -- Added support for Intra Segment Search ([#19704](https://github.com/opensearch-project/OpenSearch/pull/19704)) - Added support of WarmerRefreshListener in NRTReplicationEngine to trigger warmer after replication on replica shards ([#20650](https://github.com/opensearch-project/OpenSearch/pull/20650)) +- Added support for Intra Segment Search ([#19704](https://github.com/opensearch-project/OpenSearch/pull/19704)) ### Changed - Move Randomness from server to libs/common ([#20570](https://github.com/opensearch-project/OpenSearch/pull/20570)) From 26d8a9889c93dc1e3ba979e2c5eb9a355478417a Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Mon, 9 Mar 2026 09:15:13 -0700 Subject: [PATCH 20/26] Edge case bug with indexBulkWithSegments Signed-off-by: Prudhvi Godithi --- .../main/java/org/opensearch/test/OpenSearchIntegTestCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 516ef41fba206..6c213ed775417 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -1655,6 +1655,7 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma * @param numSegments the number of segments to create */ public void indexBulkWithSegments(List builders, int numSegments) throws InterruptedException { + assert numSegments > 0 && numSegments <= builders.size() : "numSegments must be between 1 and builders.size()"; int batchSize = builders.size() / numSegments; for (int seg = 0; seg < numSegments; seg++) { int from = seg * batchSize; From 0ba19420592c6eb7b7463abd56e1e7494362ce5d Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Mon, 9 Mar 2026 09:50:16 -0700 Subject: [PATCH 21/26] Address comments Signed-off-by: Prudhvi Godithi --- .../search/aggregations/metrics/AvgIT.java | 10 +++++----- .../search/aggregations/metrics/CardinalityIT.java | 12 +++++++----- .../search/aggregations/metrics/MaxIT.java | 10 +++++----- .../search/aggregations/metrics/MinIT.java | 10 +++++----- .../search/aggregations/metrics/StatsIT.java | 10 +++++----- .../search/aggregations/metrics/SumIT.java | 10 +++++----- .../search/aggregations/metrics/ValueCountIT.java | 10 +++++----- 7 files changed, 37 insertions(+), 35 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java index 6b76b81c268b8..ee0131e30e5ce 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/AvgIT.java @@ -52,21 +52,21 @@ public static Collection parameters() { } public void testAvgAggregation() throws Exception { - createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); + createIndex("test_avg_agg", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); try { List builders = new ArrayList<>(5000); for (int i = 0; i < 5000; i++) { - builders.add(client().prepareIndex("test").setSource("value", i + 1)); + builders.add(client().prepareIndex("test_avg_agg").setSource("value", i + 1)); } indexBulkWithSegments(builders, 2); - indexRandomForConcurrentSearch("test"); - SearchResponse response = client().prepareSearch("test").addAggregation(avg("avg_agg").field("value")).get(); + indexRandomForConcurrentSearch("test_avg_agg"); + SearchResponse response = client().prepareSearch("test_avg_agg").addAggregation(avg("avg_agg").field("value")).get(); assertSearchResponse(response); Avg avgAgg = response.getAggregations().get("avg_agg"); assertThat(avgAgg, notNullValue()); assertThat(avgAgg.getValue(), closeTo(2500.5, 0.1)); } finally { - internalCluster().wipeIndices("test"); + internalCluster().wipeIndices("test_avg_agg"); } } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java index 6d823ec2ccccf..201cfbf1ad546 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java @@ -73,6 +73,8 @@ import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase @@ -678,22 +680,22 @@ public void testScriptCaching() throws Exception { } public void testCardinalityWithIntraSegmentPartitioning() throws Exception { - createIndex("intra_test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); + createIndex("test_cardinality_agg", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); try { List builders = new ArrayList<>(5000); for (int i = 0; i < 5000; i++) { - builders.add(client().prepareIndex("intra_test").setSource("category", i % 100)); + builders.add(client().prepareIndex("test_cardinality_agg").setSource("category", i % 100)); } indexBulkWithSegments(builders, 2); - indexRandomForConcurrentSearch("intra_test"); - SearchResponse response = client().prepareSearch("intra_test") + indexRandomForConcurrentSearch("test_cardinality_agg"); + SearchResponse response = client().prepareSearch("test_cardinality_agg") .addAggregation(cardinality("cardinality").field("category")) .get(); Cardinality cardinalityAgg = response.getAggregations().get("cardinality"); assertThat(cardinalityAgg, notNullValue()); assertThat(cardinalityAgg.getValue(), equalTo(100L)); } finally { - internalCluster().wipeIndices("intra_test"); + internalCluster().wipeIndices("test_cardinality_agg"); } } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java index 835389ca55e2b..3f849253fd996 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MaxIT.java @@ -52,21 +52,21 @@ public static Collection parameters() { } public void testMaxAggregation() throws Exception { - createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); + createIndex("test_max_agg", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); try { List builders = new ArrayList<>(5000); for (int i = 0; i < 5000; i++) { - builders.add(client().prepareIndex("test").setSource("value", i + 1)); + builders.add(client().prepareIndex("test_max_agg").setSource("value", i + 1)); } indexBulkWithSegments(builders, 2); - indexRandomForConcurrentSearch("test"); - SearchResponse response = client().prepareSearch("test").addAggregation(max("max_agg").field("value")).get(); + indexRandomForConcurrentSearch("test_max_agg"); + SearchResponse response = client().prepareSearch("test_max_agg").addAggregation(max("max_agg").field("value")).get(); assertSearchResponse(response); Max maxAgg = response.getAggregations().get("max_agg"); assertThat(maxAgg, notNullValue()); assertThat(maxAgg.getValue(), closeTo(5000.0, 0.1)); } finally { - internalCluster().wipeIndices("test"); + internalCluster().wipeIndices("test_max_agg"); } } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java index e1d9454c50c10..3cb3cf8581247 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MinIT.java @@ -52,21 +52,21 @@ public static Collection parameters() { } public void testMinAggregation() throws Exception { - createIndex("test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); + createIndex("test_min_agg", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); try { List builders = new ArrayList<>(5000); for (int i = 0; i < 5000; i++) { - builders.add(client().prepareIndex("test").setSource("value", i + 1)); + builders.add(client().prepareIndex("test_min_agg").setSource("value", i + 1)); } indexBulkWithSegments(builders, 2); - indexRandomForConcurrentSearch("test"); - SearchResponse response = client().prepareSearch("test").addAggregation(min("min_agg").field("value")).get(); + indexRandomForConcurrentSearch("test_min_agg"); + SearchResponse response = client().prepareSearch("test_min_agg").addAggregation(min("min_agg").field("value")).get(); assertSearchResponse(response); Min minAgg = response.getAggregations().get("min_agg"); assertThat(minAgg, notNullValue()); assertThat(minAgg.getValue(), closeTo(1.0, 0.1)); } finally { - internalCluster().wipeIndices("test"); + internalCluster().wipeIndices("test_min_agg"); } } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java index 5cb57496b1270..88b09fe7dffe2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java @@ -415,17 +415,17 @@ public void testScriptCaching() throws Exception { } public void testStatsWithIntraSegmentPartitioning() throws Exception { - createIndex("intra_test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); + createIndex("test_stats_agg", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); try { long expectedSum = 0; List builders = new ArrayList<>(5000); for (int i = 0; i < 5000; i++) { expectedSum += (i + 1); - builders.add(client().prepareIndex("intra_test").setSource("value", i + 1)); + builders.add(client().prepareIndex("test_stats_agg").setSource("value", i + 1)); } indexBulkWithSegments(builders, 2); - indexRandomForConcurrentSearch("intra_test"); - SearchResponse response = client().prepareSearch("intra_test").addAggregation(stats("stats").field("value")).get(); + indexRandomForConcurrentSearch("test_stats_agg"); + SearchResponse response = client().prepareSearch("test_stats_agg").addAggregation(stats("stats").field("value")).get(); Stats statsAgg = response.getAggregations().get("stats"); assertThat(statsAgg, notNullValue()); assertThat(statsAgg.getCount(), equalTo(5000L)); @@ -434,7 +434,7 @@ public void testStatsWithIntraSegmentPartitioning() throws Exception { assertThat(statsAgg.getSum(), closeTo((double) expectedSum, 0.1)); assertThat(statsAgg.getAvg(), closeTo((double) expectedSum / 5000, 0.1)); } finally { - internalCluster().wipeIndices("intra_test"); + internalCluster().wipeIndices("test_stats_agg"); } } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java index 9fd301cdd6b76..abeeeebdc6195 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java @@ -434,22 +434,22 @@ public void testFieldAliasInSubAggregation() { } public void testSumWithIntraSegmentPartitioning() throws Exception { - createIndex("intra_test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); + createIndex("test_sum_agg", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); try { long expectedSum = 0; List builders = new ArrayList<>(5000); for (int i = 0; i < 5000; i++) { expectedSum += (i + 1); - builders.add(client().prepareIndex("intra_test").setSource("value", i + 1)); + builders.add(client().prepareIndex("test_sum_agg").setSource("value", i + 1)); } indexBulkWithSegments(builders, 2); - indexRandomForConcurrentSearch("intra_test"); - SearchResponse response = client().prepareSearch("intra_test").addAggregation(sum("sum").field("value")).get(); + indexRandomForConcurrentSearch("test_sum_agg"); + SearchResponse response = client().prepareSearch("test_sum_agg").addAggregation(sum("sum").field("value")).get(); Sum sumAgg = response.getAggregations().get("sum"); assertThat(sumAgg, notNullValue()); assertThat(sumAgg.getValue(), closeTo((double) expectedSum, 0.1)); } finally { - internalCluster().wipeIndices("intra_test"); + internalCluster().wipeIndices("test_sum_agg"); } } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java index f9ab1478d3f2e..5ab3a9bb1c01e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java @@ -428,20 +428,20 @@ public void testOrderByEmptyAggregation() throws Exception { } public void testValueCountWithIntraSegmentPartitioning() throws Exception { - createIndex("intra_test", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); + createIndex("test_value_count_agg", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 1).build()); try { List builders = new ArrayList<>(5000); for (int i = 0; i < 5000; i++) { - builders.add(client().prepareIndex("intra_test").setSource("value", i + 1)); + builders.add(client().prepareIndex("test_value_count_agg").setSource("value", i + 1)); } indexBulkWithSegments(builders, 2); - indexRandomForConcurrentSearch("intra_test"); - SearchResponse response = client().prepareSearch("intra_test").addAggregation(count("count").field("value")).get(); + indexRandomForConcurrentSearch("test_value_count_agg"); + SearchResponse response = client().prepareSearch("test_value_count_agg").addAggregation(count("count").field("value")).get(); ValueCount countAgg = response.getAggregations().get("count"); assertThat(countAgg, notNullValue()); assertThat(countAgg.getValue(), equalTo(5000L)); } finally { - internalCluster().wipeIndices("intra_test"); + internalCluster().wipeIndices("test_value_count_agg"); } } } From 9d380ad3042dea3b3fd87f9f460a2f870e8aff4f Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Mon, 9 Mar 2026 09:56:28 -0700 Subject: [PATCH 22/26] Address comments Signed-off-by: Prudhvi Godithi --- .../opensearch/search/aggregations/metrics/CardinalityIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java index 201cfbf1ad546..d3d0372a6d098 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java @@ -73,8 +73,6 @@ import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase From e441c2a4c5023f4ef634331f0654947da9337536 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Wed, 11 Mar 2026 20:19:47 -0700 Subject: [PATCH 23/26] Upstream fetch, fix conflicts Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b553113437fc8..a2545be378841 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,8 +38,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Introduced strategy planner interfaces for indexing and deletion ([#20585](https://github.com/opensearch-project/OpenSearch/pull/20585)) - Implement FieldMappingIngestionMessageMapper for pull-based ingestion ([#20729](https://github.com/opensearch-project/OpenSearch/pull/20729)) - Added support of WarmerRefreshListener in NRTReplicationEngine to trigger warmer after replication on replica shards ([#20650](https://github.com/opensearch-project/OpenSearch/pull/20650)) -- Added support for Intra Segment Search ([#19704](https://github.com/opensearch-project/OpenSearch/pull/19704)) - WLM group custom search settings - groundwork and timeout ([#20536](https://github.com/opensearch-project/OpenSearch/issues/20536)) +- Added support for Intra Segment Search ([#19704](https://github.com/opensearch-project/OpenSearch/pull/19704)) ### Changed - Make telemetry `Tags` immutable ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788)) From 31440a7c8484d364e7c4dc8f0d48ee03f7160dda Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Thu, 12 Mar 2026 08:05:01 -0700 Subject: [PATCH 24/26] Upstream fetch, fix conflicts Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2545be378841..5794b395f2be4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Support TLS cert hot-reload for Arrow Flight transport ([#20700](https://github.com/opensearch-project/OpenSearch/pull/20700)) - [Workload Management] Enhance Scroll API support for autotagging ([#20151](https://github.com/opensearch-project/OpenSearch/pull/20151)) - Add indices to search request slowlog ([#20588](https://github.com/opensearch-project/OpenSearch/pull/20588)) -- Add intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) - Add mapper_settings support and field_mapping mapper type for pull-based ingestion([#20722](https://github.com/opensearch-project/OpenSearch/pull/20722)) - Add support for fields containing dots in their name as literals ([#19958](https://github.com/opensearch-project/OpenSearch/pull/19958)) - Add support for forward translog reading ([#20163](https://github.com/opensearch-project/OpenSearch/pull/20163)) @@ -34,12 +33,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Update to `almalinux:10` ([#20482](https://github.com/opensearch-project/OpenSearch/pull/20482)) - Add X-Request-Id to uniquely identify a search request ([#19798](https://github.com/opensearch-project/OpenSearch/pull/19798)) - Added TopN selection logic for streaming terms aggregations ([#20481](https://github.com/opensearch-project/OpenSearch/pull/20481)) +- Added support for Intra Segment Search ([#19704](https://github.com/opensearch-project/OpenSearch/pull/19704)) - Introduce AdditionalCodecs and EnginePlugin::getAdditionalCodecs hook to allow additional Codec registration ([#20411](https://github.com/opensearch-project/OpenSearch/pull/20411)) - Introduced strategy planner interfaces for indexing and deletion ([#20585](https://github.com/opensearch-project/OpenSearch/pull/20585)) - Implement FieldMappingIngestionMessageMapper for pull-based ingestion ([#20729](https://github.com/opensearch-project/OpenSearch/pull/20729)) - Added support of WarmerRefreshListener in NRTReplicationEngine to trigger warmer after replication on replica shards ([#20650](https://github.com/opensearch-project/OpenSearch/pull/20650)) - WLM group custom search settings - groundwork and timeout ([#20536](https://github.com/opensearch-project/OpenSearch/issues/20536)) -- Added support for Intra Segment Search ([#19704](https://github.com/opensearch-project/OpenSearch/pull/19704)) +- Add intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) ### Changed - Make telemetry `Tags` immutable ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788)) @@ -69,6 +69,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix TLS cert hot-reload for Arrow Flight transport ([#20732](https://github.com/opensearch-project/OpenSearch/pull/20732)) - Fix misleading heap usage cancellation message in SearchBackpressureService ([#20779](https://github.com/opensearch-project/OpenSearch/pull/20779)) - - Delegate getMin/getMax methods for ExitableTerms ([#20775](https://github.com/opensearch-project/OpenSearch/pull/20775)) +- Fix terms lookup subquery fetch limit reading from non-existent index setting instead of cluster `max_clause_count` ([#20823](https://github.com/opensearch-project/OpenSearch/pull/20823)) ### Dependencies - Bump shadow-gradle-plugin from 8.3.9 to 9.3.1 ([#20569](https://github.com/opensearch-project/OpenSearch/pull/20569)) From 526f41f95214791347555ccf20d8fbe3cd50dffb Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Thu, 12 Mar 2026 14:06:31 -0700 Subject: [PATCH 25/26] Upstream fetch Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1e2f4f59d50e..dc365d49daefd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Implement FieldMappingIngestionMessageMapper for pull-based ingestion ([#20729](https://github.com/opensearch-project/OpenSearch/pull/20729)) - Added support of WarmerRefreshListener in NRTReplicationEngine to trigger warmer after replication on replica shards ([#20650](https://github.com/opensearch-project/OpenSearch/pull/20650)) - WLM group custom search settings - groundwork and timeout ([#20536](https://github.com/opensearch-project/OpenSearch/issues/20536)) -- Add intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) +- Add intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) ### Changed - Make telemetry `Tags` immutable ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788)) From 5a88b052da20debb3526d4eaedaa2e796bf1cd45 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Thu, 12 Mar 2026 15:42:42 -0700 Subject: [PATCH 26/26] Upstream fetch, fix conflicts Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e8ef6867fac5..d9db0161e0f74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Implement FieldMappingIngestionMessageMapper for pull-based ingestion ([#20729](https://github.com/opensearch-project/OpenSearch/pull/20729)) - Added support of WarmerRefreshListener in NRTReplicationEngine to trigger warmer after replication on replica shards ([#20650](https://github.com/opensearch-project/OpenSearch/pull/20650)) - WLM group custom search settings - groundwork and timeout ([#20536](https://github.com/opensearch-project/OpenSearch/issues/20536)) -- Add intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) - Expose JVM runtime metrics via telemetry framework ([#20844](https://github.com/opensearch-project/OpenSearch/pull/20844)) +- Add intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) ### Changed - Make telemetry `Tags` immutable ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788))