diff --git a/CHANGELOG.md b/CHANGELOG.md index ae440d2b6214b..f653392359e95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Security Manager Replacement] Create initial Java Agent to intercept Socket::connect calls ([#17724](https://github.com/opensearch-project/OpenSearch/pull/17724)) - Add ingestion management APIs for pause, resume and get ingestion state ([#17631](https://github.com/opensearch-project/OpenSearch/pull/17631)) - [Security Manager Replacement] Enhance Java Agent to intercept System::exit ([#17746](https://github.com/opensearch-project/OpenSearch/pull/17746)) +- Support AutoExpand for SearchReplica ([#17741](https://github.com/opensearch-project/OpenSearch/pull/17741)) ### Changed - Migrate BC libs to their FIPS counterparts ([#14912](https://github.com/opensearch-project/OpenSearch/pull/14912)) diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/metadata/AutoExpandSearchReplicasIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/metadata/AutoExpandSearchReplicasIT.java new file mode 100644 index 0000000000000..c177b01fea642 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/metadata/AutoExpandSearchReplicasIT.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.opensearch.cluster.routing.UnassignedInfo; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; +import org.opensearch.test.InternalTestCluster; +import org.opensearch.test.OpenSearchIntegTestCase; + +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class AutoExpandSearchReplicasIT extends RemoteStoreBaseIntegTestCase { + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL, Boolean.TRUE).build(); + } + + public void testAutoExpandSearchReplica() throws Exception { + String indexName = "test"; + internalCluster().startClusterManagerOnlyNode(); + + // Create a cluster with 2 data nodes and 1 search node + internalCluster().startDataOnlyNode(); + internalCluster().startDataOnlyNode(); + String searchNode = internalCluster().startSearchOnlyNode(); + + // Create index with 1 primary, 1 replica and 1 search replica shards + createIndex( + indexName, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) + .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueMillis(0)) + .build() + ); + ensureGreen(); + + assertBusy(() -> assertEquals(1, getNumShards(indexName).numSearchReplicas)); + + // Enable auto expand for search replica + client().admin() + .indices() + .prepareUpdateSettings(indexName) + .setSettings(Settings.builder().put("index.auto_expand_search_replicas", "0-all")) + .get(); + + // Add 1 more search nodes + internalCluster().startSearchOnlyNode(); + + assertBusy(() -> assertEquals(2, getNumShards(indexName).numSearchReplicas)); + + // Stop a node which hosts search replica + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(searchNode)); + assertBusy(() -> assertEquals(1, getNumShards(indexName).numSearchReplicas)); + + // Add 1 more search nodes + internalCluster().startSearchOnlyNode(); + assertBusy(() -> assertEquals(2, getNumShards(indexName).numSearchReplicas)); + } +} diff --git a/server/src/main/java/org/opensearch/cluster/metadata/AutoExpandReplicas.java b/server/src/main/java/org/opensearch/cluster/metadata/AutoExpandReplicas.java index bd31350780d72..d9a3d7bf8eb3f 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/AutoExpandReplicas.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/AutoExpandReplicas.java @@ -31,7 +31,6 @@ package org.opensearch.cluster.metadata; -import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.common.Booleans; @@ -142,13 +141,14 @@ public boolean isEnabled() { private OptionalInt getDesiredNumberOfReplicas(IndexMetadata indexMetadata, RoutingAllocation allocation) { if (enabled) { - int numMatchingDataNodes = 0; - for (final DiscoveryNode cursor : allocation.nodes().getDataNodes().values()) { - Decision decision = allocation.deciders().shouldAutoExpandToNode(indexMetadata, cursor, allocation); - if (decision.type() != Decision.Type.NO) { - numMatchingDataNodes++; - } - } + int numMatchingDataNodes = (int) allocation.nodes() + .getDataNodes() + .values() + .stream() + .filter(node -> node.isSearchNode() == false) + .map(node -> allocation.deciders().shouldAutoExpandToNode(indexMetadata, node, allocation)) + .filter(decision -> decision.type() != Decision.Type.NO) + .count(); final int min = getMinReplicas(); final int max = getMaxReplicas(numMatchingDataNodes); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/AutoExpandSearchReplicas.java b/server/src/main/java/org/opensearch/cluster/metadata/AutoExpandSearchReplicas.java new file mode 100644 index 0000000000000..91608d46aff20 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/metadata/AutoExpandSearchReplicas.java @@ -0,0 +1,180 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.allocation.RoutingAllocation; +import org.opensearch.cluster.routing.allocation.decider.Decision; +import org.opensearch.common.Booleans; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Setting.Property; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.OptionalInt; + +import static org.opensearch.cluster.metadata.MetadataIndexStateService.isIndexVerifiedBeforeClosed; + +/** + * This class acts as a functional wrapper around the {@code index.auto_expand_search_replicas} setting. + * This setting's value expands into a minimum and maximum value, requiring special handling based on the + * number of search nodes in the cluster. This class handles parsing and simplifies access to these values. + * + * @opensearch.internal + */ +public final class AutoExpandSearchReplicas { + // the value we recognize in the "max" position to mean all the search nodes + private static final String ALL_NODES_VALUE = "all"; + + private static final AutoExpandSearchReplicas FALSE_INSTANCE = new AutoExpandSearchReplicas(0, 0, false); + + public static final Setting SETTING = new Setting<>( + IndexMetadata.SETTING_AUTO_EXPAND_SEARCH_REPLICAS, + "false", + AutoExpandSearchReplicas::parse, + Property.Dynamic, + Property.IndexScope + ); + + private static AutoExpandSearchReplicas parse(String value) { + final int min; + final int max; + if (Booleans.isFalse(value)) { + return FALSE_INSTANCE; + } + final int dash = value.indexOf('-'); + if (-1 == dash) { + throw new IllegalArgumentException( + "failed to parse [" + IndexMetadata.SETTING_AUTO_EXPAND_SEARCH_REPLICAS + "] from value: [" + value + "] at index " + dash + ); + } + final String sMin = value.substring(0, dash); + try { + min = Integer.parseInt(sMin); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "failed to parse [" + IndexMetadata.SETTING_AUTO_EXPAND_SEARCH_REPLICAS + "] from value: [" + value + "] at index " + dash, + e + ); + } + String sMax = value.substring(dash + 1); + if (sMax.equals(ALL_NODES_VALUE)) { + max = Integer.MAX_VALUE; + } else { + try { + max = Integer.parseInt(sMax); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "failed to parse [" + + IndexMetadata.SETTING_AUTO_EXPAND_SEARCH_REPLICAS + + "] from value: [" + + value + + "] at index " + + dash, + e + ); + } + } + return new AutoExpandSearchReplicas(min, max, true); + } + + private final int minSearchReplicas; + private final int maxSearchReplicas; + private final boolean enabled; + + private AutoExpandSearchReplicas(int minReplicas, int maxReplicas, boolean enabled) { + if (minReplicas > maxReplicas) { + throw new IllegalArgumentException( + "[" + + IndexMetadata.SETTING_AUTO_EXPAND_SEARCH_REPLICAS + + "] minSearchReplicas must be =< maxSearchReplicas but wasn't " + + minReplicas + + " > " + + maxReplicas + ); + } + this.minSearchReplicas = minReplicas; + this.maxSearchReplicas = maxReplicas; + this.enabled = enabled; + } + + int getMinSearchReplicas() { + return minSearchReplicas; + } + + public int getMaxSearchReplicas() { + return maxSearchReplicas; + } + + public boolean isEnabled() { + return enabled; + } + + private OptionalInt getDesiredNumberOfSearchReplicas(IndexMetadata indexMetadata, RoutingAllocation allocation) { + int numMatchingSearchNodes = (int) allocation.nodes() + .getDataNodes() + .values() + .stream() + .filter(DiscoveryNode::isSearchNode) + .map(node -> allocation.deciders().shouldAutoExpandToNode(indexMetadata, node, allocation)) + .filter(decision -> decision.type() != Decision.Type.NO) + .count(); + + return calculateNumberOfSearchReplicas(numMatchingSearchNodes); + } + + // package private for testing + OptionalInt calculateNumberOfSearchReplicas(int numMatchingSearchNodes) { + // Calculate the maximum possible number of search replicas + int maxPossibleReplicas = Math.min(numMatchingSearchNodes, maxSearchReplicas); + + // Determine the number of search replicas + int numberOfSearchReplicas = Math.max(minSearchReplicas, maxPossibleReplicas); + + // Additional check to ensure we don't exceed max possible search replicas + if (numberOfSearchReplicas <= maxPossibleReplicas) { + return OptionalInt.of(numberOfSearchReplicas); + } + + return OptionalInt.empty(); + } + + @Override + public String toString() { + return enabled ? minSearchReplicas + "-" + maxSearchReplicas : "false"; + } + + /** + * Checks if there are search replicas with the auto-expand feature that need to be adapted. + * Returns a map of updates, which maps the indices to be updated to the desired number of search replicas. + * The map has the desired number of search replicas as key and the indices to update as value, as this allows the result + * of this method to be directly applied to RoutingTable.Builder#updateNumberOfSearchReplicas. + */ + public static Map> getAutoExpandSearchReplicaChanges(Metadata metadata, RoutingAllocation allocation) { + Map> updatedSearchReplicas = new HashMap<>(); + + for (final IndexMetadata indexMetadata : metadata) { + if (indexMetadata.getState() == IndexMetadata.State.OPEN || isIndexVerifiedBeforeClosed(indexMetadata)) { + AutoExpandSearchReplicas autoExpandSearchReplicas = SETTING.get(indexMetadata.getSettings()); + if (autoExpandSearchReplicas.isEnabled()) { + autoExpandSearchReplicas.getDesiredNumberOfSearchReplicas(indexMetadata, allocation) + .ifPresent(numberOfSearchReplicas -> { + if (numberOfSearchReplicas != indexMetadata.getNumberOfSearchOnlyReplicas()) { + updatedSearchReplicas.computeIfAbsent(numberOfSearchReplicas, ArrayList::new) + .add(indexMetadata.getIndex().getName()); + } + }); + } + } + } + return updatedSearchReplicas; + } +} diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 72afd44eadef8..9005c830167f9 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -492,7 +492,9 @@ public Iterator> settings() { ); public static final String SETTING_AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas"; + public static final String SETTING_AUTO_EXPAND_SEARCH_REPLICAS = "index.auto_expand_search_replicas"; public static final Setting INDEX_AUTO_EXPAND_REPLICAS_SETTING = AutoExpandReplicas.SETTING; + public static final Setting INDEX_AUTO_EXPAND_SEARCH_REPLICAS_SETTING = AutoExpandSearchReplicas.SETTING; /** * Blocks the API. diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 3483c14df6272..76b2b948ca164 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -1505,7 +1505,10 @@ List getIndexSettingsValidationErrors( Optional replicaValidationError = awarenessReplicaBalance.validate(replicaCount, autoExpandReplica); replicaValidationError.ifPresent(validationErrors::add); - Optional searchReplicaValidationError = awarenessReplicaBalance.validate(searchReplicaCount); + Optional searchReplicaValidationError = awarenessReplicaBalance.validate( + searchReplicaCount, + AutoExpandSearchReplicas.SETTING.get(settings) + ); searchReplicaValidationError.ifPresent(validationErrors::add); } return validationErrors; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java index fff704210ca7a..8eff5604045bc 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java @@ -304,7 +304,10 @@ public ClusterState execute(ClusterState currentState) { for (Index index : request.indices()) { if (index.getName().charAt(0) != '.') { // No replica count validation for system indices - Optional error = awarenessReplicaBalance.validate(updatedNumberOfSearchReplicas); + Optional error = awarenessReplicaBalance.validate( + updatedNumberOfSearchReplicas, + AutoExpandSearchReplicas.SETTING.get(openSettings) + ); if (error.isPresent()) { ValidationException ex = new ValidationException(); diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java index 78f17c9ff212b..efe51e36ec748 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java @@ -44,6 +44,7 @@ import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.health.ClusterStateHealth; import org.opensearch.cluster.metadata.AutoExpandReplicas; +import org.opensearch.cluster.metadata.AutoExpandSearchReplicas; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNode; @@ -373,11 +374,19 @@ public ClusterState adaptAutoExpandReplicas(ClusterState clusterState) { clusterState.metadata(), allocation ); - if (autoExpandReplicaChanges.isEmpty()) { + + final Map> autoExpandSearchReplicaChanges = AutoExpandSearchReplicas.getAutoExpandSearchReplicaChanges( + clusterState.metadata(), + allocation + ); + + if (autoExpandReplicaChanges.isEmpty() && autoExpandSearchReplicaChanges.isEmpty()) { return clusterState; } else { final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable()); final Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata()); + final Set updatedIndices = new HashSet<>(); + for (Map.Entry> entry : autoExpandReplicaChanges.entrySet()) { final int numberOfReplicas = entry.getKey(); final String[] indices = entry.getValue().toArray(new String[0]); @@ -385,21 +394,36 @@ public ClusterState adaptAutoExpandReplicas(ClusterState clusterState) { // operation which make these copies stale routingTableBuilder.updateNumberOfReplicas(numberOfReplicas, indices); metadataBuilder.updateNumberOfReplicas(numberOfReplicas, indices); - // update settings version for each index - for (final String index : indices) { - final IndexMetadata indexMetadata = metadataBuilder.get(index); - final IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder(indexMetadata).settingsVersion( - 1 + indexMetadata.getSettingsVersion() - ); - metadataBuilder.put(indexMetadataBuilder); - } + updatedIndices.addAll(Set.of(indices)); logger.info("updating number_of_replicas to [{}] for indices {}", numberOfReplicas, indices); } + + for (Map.Entry> entry : autoExpandSearchReplicaChanges.entrySet()) { + final int numberOfSearchReplicas = entry.getKey(); + final String[] indices = entry.getValue().toArray(new String[0]); + // we do *not* update the in sync allocation ids as they will be removed upon the first index + // operation which make these copies stale + routingTableBuilder.updateNumberOfSearchReplicas(numberOfSearchReplicas, indices); + metadataBuilder.updateNumberOfSearchReplicas(numberOfSearchReplicas, indices); + updatedIndices.addAll(Set.of(indices)); + logger.info("updating number_of_search_replicas to [{}] for indices {}", numberOfSearchReplicas, indices); + } + + // update settings version for each updated index + for (final String index : updatedIndices) { + final IndexMetadata indexMetadata = metadataBuilder.get(index); + final IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder(indexMetadata).settingsVersion( + 1 + indexMetadata.getSettingsVersion() + ); + metadataBuilder.put(indexMetadataBuilder); + } + final ClusterState fixedState = ClusterState.builder(clusterState) .routingTable(routingTableBuilder.build()) .metadata(metadataBuilder) .build(); assert AutoExpandReplicas.getAutoExpandReplicaChanges(fixedState.metadata(), allocation).isEmpty(); + assert AutoExpandSearchReplicas.getAutoExpandSearchReplicaChanges(fixedState.metadata(), allocation).isEmpty(); return fixedState; } } @@ -567,6 +591,9 @@ private void reroute(RoutingAllocation allocation) { assert hasDeadNodes(allocation) == false : "dead nodes should be explicitly cleaned up. See disassociateDeadNodes"; assert AutoExpandReplicas.getAutoExpandReplicaChanges(allocation.metadata(), allocation).isEmpty() : "auto-expand replicas out of sync with number of nodes in the cluster"; + assert AutoExpandSearchReplicas.getAutoExpandSearchReplicaChanges(allocation.metadata(), allocation).isEmpty() + : "auto-expand search replicas out of sync with number of search nodes in the cluster"; + assert assertInitialized(); long rerouteStartTimeNS = System.nanoTime(); removeDelayMarkers(allocation); diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalance.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalance.java index d2cf30bd31983..538d49d4e4701 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalance.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalance.java @@ -9,6 +9,7 @@ package org.opensearch.cluster.routing.allocation; import org.opensearch.cluster.metadata.AutoExpandReplicas; +import org.opensearch.cluster.metadata.AutoExpandSearchReplicas; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; @@ -122,13 +123,22 @@ public Optional validate(int replicaCount, AutoExpandReplicas autoExpand return Optional.empty(); } - public Optional validate(int searchReplicaCount) { - // TODO: For now Search replicas do not support auto expand, when we add support update this validation - if (searchReplicaCount > 0 && searchReplicaCount % maxAwarenessAttributes() != 0) { - String errorMessage = "total search replicas needs to be a multiple of total awareness attributes [" - + maxAwarenessAttributes() - + "]"; - return Optional.of(errorMessage); + public Optional validate(int searchReplicaCount, AutoExpandSearchReplicas autoExpandSearchReplicas) { + if (autoExpandSearchReplicas.isEnabled()) { + if ((autoExpandSearchReplicas.getMaxSearchReplicas() != Integer.MAX_VALUE) + && ((autoExpandSearchReplicas.getMaxSearchReplicas()) % maxAwarenessAttributes() != 0)) { + String errorMessage = "expected max cap on auto expand search replicas to be a multiple of total awareness attributes [" + + maxAwarenessAttributes() + + "]"; + return Optional.of(errorMessage); + } + } else { + if (searchReplicaCount > 0 && searchReplicaCount % maxAwarenessAttributes() != 0) { + String errorMessage = "total search replicas needs to be a multiple of total awareness attributes [" + + maxAwarenessAttributes() + + "]"; + return Optional.of(errorMessage); + } } return Optional.empty(); } diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 14cd7479866d2..3793b9b09e3b2 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -98,6 +98,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING, IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING, IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING, + IndexMetadata.INDEX_AUTO_EXPAND_SEARCH_REPLICAS_SETTING, IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING, IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING, IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING, diff --git a/server/src/test/java/org/opensearch/cluster/metadata/AutoExpandReplicasTests.java b/server/src/test/java/org/opensearch/cluster/metadata/AutoExpandReplicasTests.java index 4b7eaf0272a91..3c1bcf8449458 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/AutoExpandReplicasTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/AutoExpandReplicasTests.java @@ -52,7 +52,6 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Set; @@ -123,10 +122,8 @@ public void testInvalidValues() { private static final AtomicInteger nodeIdGenerator = new AtomicInteger(); protected DiscoveryNode createNode(Version version, DiscoveryNodeRole... mustHaveRoles) { - Set roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)); - Collections.addAll(roles, mustHaveRoles); final String id = String.format(Locale.ROOT, "node_%03d", nodeIdGenerator.incrementAndGet()); - return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), roles, version); + return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), Set.of(mustHaveRoles), version); } protected DiscoveryNode createNode(DiscoveryNodeRole... mustHaveRoles) { diff --git a/server/src/test/java/org/opensearch/cluster/metadata/AutoExpandSearchReplicasTests.java b/server/src/test/java/org/opensearch/cluster/metadata/AutoExpandSearchReplicasTests.java new file mode 100644 index 0000000000000..bfc0ec748c7d0 --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/metadata/AutoExpandSearchReplicasTests.java @@ -0,0 +1,141 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.cluster.metadata; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.OpenSearchAllocationTestCase; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.allocation.RoutingAllocation; +import org.opensearch.common.settings.Settings; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.OptionalInt; + +public class AutoExpandSearchReplicasTests extends OpenSearchAllocationTestCase { + + public void testParseAutoExpandSearchReplicaSettings() { + AutoExpandSearchReplicas autoExpandSearchReplicas = AutoExpandSearchReplicas.SETTING.get( + Settings.builder().put("index.auto_expand_search_replicas", "0-5").build() + ); + assertEquals(0, autoExpandSearchReplicas.getMinSearchReplicas()); + assertEquals(5, autoExpandSearchReplicas.getMaxSearchReplicas()); + + autoExpandSearchReplicas = AutoExpandSearchReplicas.SETTING.get( + Settings.builder().put("index.auto_expand_search_replicas", "0-all").build() + ); + assertEquals(0, autoExpandSearchReplicas.getMinSearchReplicas()); + assertEquals(Integer.MAX_VALUE, autoExpandSearchReplicas.getMaxSearchReplicas()); + + autoExpandSearchReplicas = AutoExpandSearchReplicas.SETTING.get( + Settings.builder().put("index.auto_expand_search_replicas", "1-all").build() + ); + assertEquals(1, autoExpandSearchReplicas.getMinSearchReplicas()); + assertEquals(Integer.MAX_VALUE, autoExpandSearchReplicas.getMaxSearchReplicas()); + } + + public void testInvalidValues() { + Throwable throwable = assertThrows(IllegalArgumentException.class, () -> { + AutoExpandSearchReplicas.SETTING.get(Settings.builder().put("index.auto_expand_search_replicas", "boom").build()); + }); + assertEquals("failed to parse [index.auto_expand_search_replicas] from value: [boom] at index -1", throwable.getMessage()); + + throwable = assertThrows(IllegalArgumentException.class, () -> { + AutoExpandSearchReplicas.SETTING.get(Settings.builder().put("index.auto_expand_search_replicas", "1-boom").build()); + }); + assertEquals("failed to parse [index.auto_expand_search_replicas] from value: [1-boom] at index 1", throwable.getMessage()); + assertEquals("For input string: \"boom\"", throwable.getCause().getMessage()); + + throwable = assertThrows(IllegalArgumentException.class, () -> { + AutoExpandSearchReplicas.SETTING.get(Settings.builder().put("index.auto_expand_search_replicas", "boom-1").build()); + }); + assertEquals("failed to parse [index.auto_expand_search_replicas] from value: [boom-1] at index 4", throwable.getMessage()); + assertEquals("For input string: \"boom\"", throwable.getCause().getMessage()); + + throwable = assertThrows(IllegalArgumentException.class, () -> { + AutoExpandSearchReplicas.SETTING.get(Settings.builder().put("index.auto_expand_search_replicas", "2-1").build()); + }); + assertEquals( + "[index.auto_expand_search_replicas] minSearchReplicas must be =< maxSearchReplicas but wasn't 2 > 1", + throwable.getMessage() + ); + } + + public void testCalculateNumberOfSearchReplicas() { + // when the number of matching search nodes is lesser than the maximum value of auto-expand + AutoExpandSearchReplicas autoExpandSearchReplicas = AutoExpandSearchReplicas.SETTING.get( + Settings.builder().put("index.auto_expand_search_replicas", "0-all").build() + ); + assertEquals(OptionalInt.of(5), autoExpandSearchReplicas.calculateNumberOfSearchReplicas(5)); + + // when the number of matching search nodes is equal to the maximum value of auto-expand + autoExpandSearchReplicas = AutoExpandSearchReplicas.SETTING.get( + Settings.builder().put("index.auto_expand_search_replicas", "0-5").build() + ); + assertEquals(OptionalInt.of(5), autoExpandSearchReplicas.calculateNumberOfSearchReplicas(5)); + + // when the number of matching search nodes is equal to the minimum value of auto-expand + autoExpandSearchReplicas = AutoExpandSearchReplicas.SETTING.get( + Settings.builder().put("index.auto_expand_search_replicas", "0-5").build() + ); + assertEquals(OptionalInt.of(0), autoExpandSearchReplicas.calculateNumberOfSearchReplicas(0)); + + // when the number of matching search nodes is greater than the maximum value of auto-expand + autoExpandSearchReplicas = AutoExpandSearchReplicas.SETTING.get( + Settings.builder().put("index.auto_expand_search_replicas", "0-5").build() + ); + assertEquals(OptionalInt.of(5), autoExpandSearchReplicas.calculateNumberOfSearchReplicas(8)); + + // when the number of matching search nodes is lesser than the minimum value of auto-expand, + // then the number of search replicas remains unchanged + autoExpandSearchReplicas = AutoExpandSearchReplicas.SETTING.get( + Settings.builder().put("index.auto_expand_search_replicas", "2-5").build() + ); + assertEquals(OptionalInt.empty(), autoExpandSearchReplicas.calculateNumberOfSearchReplicas(1)); + } + + public void testGetAutoExpandReplicaChanges() { + Metadata metadata = Metadata.builder() + .put( + IndexMetadata.builder("test") + .settings(settings(Version.CURRENT).put("index.auto_expand_search_replicas", "0-all")) + .numberOfShards(1) + .numberOfReplicas(0) + .numberOfSearchReplicas(1) + ) + .build(); + + RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); + ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metadata(metadata) + .routingTable(initialRoutingTable) + .nodes( + DiscoveryNodes.builder() + .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Collections.emptyMap(), SEARCH_ROLE, Version.CURRENT)) + .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Collections.emptyMap(), SEARCH_ROLE, Version.CURRENT)) + .add(new DiscoveryNode("node3", buildNewFakeTransportAddress(), Collections.emptyMap(), SEARCH_ROLE, Version.CURRENT)) + .build() + ) + .build(); + + RoutingAllocation allocation = new RoutingAllocation( + yesAllocationDeciders(), + clusterState.getRoutingNodes(), + clusterState, + null, + null, + System.nanoTime() + ); + + assertEquals(Map.of(3, List.of("test")), AutoExpandSearchReplicas.getAutoExpandSearchReplicaChanges(metadata, allocation)); + } +} diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationServiceTests.java index cce75105dd33f..b1f4b45bb2441 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationServiceTests.java @@ -40,6 +40,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; @@ -48,9 +49,11 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.UnassignedInfo; +import org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; import org.opensearch.cluster.routing.allocation.allocator.ShardsAllocator; import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders; import org.opensearch.cluster.routing.allocation.decider.Decision; +import org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; import org.opensearch.common.settings.ClusterSettings; @@ -66,12 +69,16 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_SEARCH_REPLICAS; import static org.opensearch.cluster.routing.UnassignedInfo.AllocationStatus.DECIDERS_NO; import static org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING; import static org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING; @@ -445,4 +452,116 @@ private static ClusterState rerouteAndStartShards(final AllocationService alloca ); } + public void testAdaptAutoExpandReplicasWhenAutoExpandChangesExists() { + final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + String indexName = "index1"; + Set SEARCH_NODE_ROLE = new HashSet<>(List.of(DiscoveryNodeRole.SEARCH_ROLE)); + Set DATA_NODE_ROLE = new HashSet<>(List.of(DiscoveryNodeRole.DATA_ROLE)); + + nodesBuilder.add( + new DiscoveryNode("node1", buildNewFakeTransportAddress(), Collections.emptyMap(), DATA_NODE_ROLE, Version.CURRENT) + ); + nodesBuilder.add( + new DiscoveryNode("node2", buildNewFakeTransportAddress(), Collections.emptyMap(), DATA_NODE_ROLE, Version.CURRENT) + ); + nodesBuilder.add( + new DiscoveryNode("node3", buildNewFakeTransportAddress(), Collections.emptyMap(), DATA_NODE_ROLE, Version.CURRENT) + ); + nodesBuilder.add( + new DiscoveryNode("node4", buildNewFakeTransportAddress(), Collections.emptyMap(), SEARCH_NODE_ROLE, Version.CURRENT) + ); + nodesBuilder.add( + new DiscoveryNode("node5", buildNewFakeTransportAddress(), Collections.emptyMap(), SEARCH_NODE_ROLE, Version.CURRENT) + ); + + Metadata.Builder metadataBuilder = Metadata.builder() + .put( + IndexMetadata.builder(indexName) + .settings( + settings(Version.CURRENT).put(SETTING_AUTO_EXPAND_REPLICAS, "0-all") + .put(SETTING_AUTO_EXPAND_SEARCH_REPLICAS, "0-all") + ) + .numberOfShards(1) + .numberOfReplicas(1) + .numberOfSearchReplicas(1) + ); + + final RoutingTable.Builder routingTableBuilder = RoutingTable.builder().addAsRecovery(metadataBuilder.get("index1")); + final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(nodesBuilder) + .metadata(metadataBuilder) + .routingTable(routingTableBuilder.build()) + .build(); + final AllocationService allocationService = new AllocationService( + new AllocationDeciders(Collections.singleton(new MaxRetryAllocationDecider())), + new TestGatewayAllocator(), + new BalancedShardsAllocator(Settings.EMPTY), + EmptyClusterInfoService.INSTANCE, + EmptySnapshotsInfoService.INSTANCE + ); + + ClusterState updatedClusterState = allocationService.adaptAutoExpandReplicas(clusterState); + assertEquals(2, updatedClusterState.routingTable().index(indexName).shard(0).writerReplicas().size()); + assertEquals(2, updatedClusterState.routingTable().index(indexName).shard(0).searchOnlyReplicas().size()); + assertEquals(2, updatedClusterState.metadata().index(indexName).getNumberOfReplicas()); + assertEquals(2, updatedClusterState.metadata().index(indexName).getNumberOfSearchOnlyReplicas()); + assertNotEquals(updatedClusterState, clusterState); + assertEquals( + clusterState.metadata().index(indexName).getSettingsVersion() + 1, + updatedClusterState.metadata().index(indexName).getSettingsVersion() + ); + } + + public void testAdaptAutoExpandReplicasWhenAutoExpandChangesNotExists() { + final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + String indexName = "index1"; + Set SEARCH_NODE_ROLE = Set.of(DiscoveryNodeRole.SEARCH_ROLE); + Set DATA_NODE_ROLE = Set.of(DiscoveryNodeRole.DATA_ROLE); + + nodesBuilder.add( + new DiscoveryNode("node1", buildNewFakeTransportAddress(), Collections.emptyMap(), DATA_NODE_ROLE, Version.CURRENT) + ); + nodesBuilder.add( + new DiscoveryNode("node2", buildNewFakeTransportAddress(), Collections.emptyMap(), DATA_NODE_ROLE, Version.CURRENT) + ); + nodesBuilder.add( + new DiscoveryNode("node3", buildNewFakeTransportAddress(), Collections.emptyMap(), DATA_NODE_ROLE, Version.CURRENT) + ); + nodesBuilder.add( + new DiscoveryNode("node4", buildNewFakeTransportAddress(), Collections.emptyMap(), SEARCH_NODE_ROLE, Version.CURRENT) + ); + nodesBuilder.add( + new DiscoveryNode("node5", buildNewFakeTransportAddress(), Collections.emptyMap(), SEARCH_NODE_ROLE, Version.CURRENT) + ); + + Metadata.Builder metadataBuilder = Metadata.builder() + .put( + IndexMetadata.builder(indexName) + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1) + .numberOfSearchReplicas(1) + ); + + final RoutingTable.Builder routingTableBuilder = RoutingTable.builder().addAsRecovery(metadataBuilder.get("index1")); + final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(nodesBuilder) + .metadata(metadataBuilder) + .routingTable(routingTableBuilder.build()) + .build(); + final AllocationService allocationService = new AllocationService( + new AllocationDeciders(Collections.singleton(new MaxRetryAllocationDecider())), + new TestGatewayAllocator(), + new BalancedShardsAllocator(Settings.EMPTY), + EmptyClusterInfoService.INSTANCE, + EmptySnapshotsInfoService.INSTANCE + ); + + ClusterState updatedClusterState = allocationService.adaptAutoExpandReplicas(clusterState); + assertEquals(1, updatedClusterState.routingTable().index(indexName).shard(0).writerReplicas().size()); + assertEquals(1, updatedClusterState.routingTable().index(indexName).shard(0).searchOnlyReplicas().size()); + assertEquals(1, updatedClusterState.metadata().index(indexName).getNumberOfReplicas()); + assertEquals(1, updatedClusterState.metadata().index(indexName).getNumberOfSearchOnlyReplicas()); + assertEquals(updatedClusterState, clusterState); + } } diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalanceTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalanceTests.java index c6134330727aa..aecd721542127 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalanceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalanceTests.java @@ -10,6 +10,7 @@ import org.opensearch.cluster.OpenSearchAllocationTestCase; import org.opensearch.cluster.metadata.AutoExpandReplicas; +import org.opensearch.cluster.metadata.AutoExpandSearchReplicas; import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -17,6 +18,8 @@ import java.util.Optional; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_SEARCH_REPLICAS; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; import static org.hamcrest.Matchers.equalTo; public class AwarenessReplicaBalanceTests extends OpenSearchAllocationTestCase { @@ -30,6 +33,7 @@ public void testNoForcedAwarenessAttribute() { Settings settings = Settings.builder() .put("cluster.routing.allocation.awareness.attributes", "rack_id") .put(SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .put(SETTING_AUTO_EXPAND_SEARCH_REPLICAS, "0-1") .build(); AutoExpandReplicas autoExpandReplica = AutoExpandReplicas.SETTING.get(settings); AwarenessReplicaBalance awarenessReplicaBalance = new AwarenessReplicaBalance(settings, EMPTY_CLUSTER_SETTINGS); @@ -38,8 +42,9 @@ public void testNoForcedAwarenessAttribute() { assertEquals(awarenessReplicaBalance.validate(0, autoExpandReplica), Optional.empty()); assertEquals(awarenessReplicaBalance.validate(1, autoExpandReplica), Optional.empty()); - assertEquals(awarenessReplicaBalance.validate(0), Optional.empty()); - assertEquals(awarenessReplicaBalance.validate(1), Optional.empty()); + AutoExpandSearchReplicas autoExpandSearchReplicas = AutoExpandSearchReplicas.SETTING.get(settings); + assertEquals(awarenessReplicaBalance.validate(0, autoExpandSearchReplicas), Optional.empty()); + assertEquals(awarenessReplicaBalance.validate(1, autoExpandSearchReplicas), Optional.empty()); } public void testForcedAwarenessAttribute() { @@ -50,6 +55,7 @@ public void testForcedAwarenessAttribute() { .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "rack.values", "c, d, e") .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), true) .put(SETTING_AUTO_EXPAND_REPLICAS, "0-2") + .put(SETTING_AUTO_EXPAND_SEARCH_REPLICAS, "0-3") .build(); AwarenessReplicaBalance awarenessReplicaBalance = new AwarenessReplicaBalance(settings, EMPTY_CLUSTER_SETTINGS); @@ -59,6 +65,9 @@ public void testForcedAwarenessAttribute() { assertEquals(awarenessReplicaBalance.validate(1, autoExpandReplica), Optional.empty()); assertEquals(awarenessReplicaBalance.validate(0, autoExpandReplica), Optional.empty()); + AutoExpandSearchReplicas autoExpandSearchReplicas = AutoExpandSearchReplicas.SETTING.get(settings); + assertEquals(awarenessReplicaBalance.validate(1, autoExpandSearchReplicas), Optional.empty()); + // When auto expand replica settings is passed as max cap settings = Settings.builder() .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone, rack") @@ -66,6 +75,7 @@ public void testForcedAwarenessAttribute() { .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "rack.values", "c, d, e") .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), true) .put(SETTING_AUTO_EXPAND_REPLICAS, "0-all") + .put(SETTING_AUTO_EXPAND_SEARCH_REPLICAS, "0-all") .build(); awarenessReplicaBalance = new AwarenessReplicaBalance(settings, EMPTY_CLUSTER_SETTINGS); @@ -76,6 +86,9 @@ public void testForcedAwarenessAttribute() { assertEquals(awarenessReplicaBalance.validate(1, autoExpandReplica), Optional.empty()); assertEquals(awarenessReplicaBalance.validate(0, autoExpandReplica), Optional.empty()); + autoExpandSearchReplicas = AutoExpandSearchReplicas.SETTING.get(settings); + assertEquals(awarenessReplicaBalance.validate(1, autoExpandSearchReplicas), Optional.empty()); + // when auto expand is not valid set as per zone awareness settings = Settings.builder() .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone, rack") @@ -83,6 +96,7 @@ public void testForcedAwarenessAttribute() { .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "rack.values", "c, d, e") .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), true) .put(SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .put(SETTING_AUTO_EXPAND_SEARCH_REPLICAS, "0-1") .build(); awarenessReplicaBalance = new AwarenessReplicaBalance(settings, EMPTY_CLUSTER_SETTINGS); @@ -97,6 +111,16 @@ public void testForcedAwarenessAttribute() { Optional.of("expected max cap on auto expand to be a multiple of total awareness attributes [3]") ); + autoExpandSearchReplicas = AutoExpandSearchReplicas.SETTING.get(settings); + assertEquals( + awarenessReplicaBalance.validate(1, autoExpandSearchReplicas), + Optional.of("expected max cap on auto expand search replicas to be a multiple of total awareness attributes [3]") + ); + assertEquals( + awarenessReplicaBalance.validate(2, autoExpandSearchReplicas), + Optional.of("expected max cap on auto expand search replicas to be a multiple of total awareness attributes [3]") + ); + // When auto expand replica is not present settings = Settings.builder() .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone, rack") @@ -118,14 +142,15 @@ public void testForcedAwarenessAttribute() { Optional.of("expected total copies needs to be a multiple of total awareness attributes [3]") ); - assertEquals(awarenessReplicaBalance.validate(3), Optional.empty()); - assertEquals(awarenessReplicaBalance.validate(0), Optional.empty()); + autoExpandSearchReplicas = AutoExpandSearchReplicas.SETTING.get(settings); + assertEquals(awarenessReplicaBalance.validate(3, autoExpandSearchReplicas), Optional.empty()); + assertEquals(awarenessReplicaBalance.validate(0, autoExpandSearchReplicas), Optional.empty()); assertEquals( - awarenessReplicaBalance.validate(2), + awarenessReplicaBalance.validate(2, autoExpandSearchReplicas), Optional.of("total search replicas needs to be a multiple of total awareness attributes [3]") ); assertEquals( - awarenessReplicaBalance.validate(1), + awarenessReplicaBalance.validate(1, autoExpandSearchReplicas), Optional.of("total search replicas needs to be a multiple of total awareness attributes [3]") ); } @@ -135,6 +160,7 @@ public void testForcedAwarenessAttributeDisabled() { .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone, rack") .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), true) .put(SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, "0-1") .build(); AwarenessReplicaBalance awarenessReplicaBalance = new AwarenessReplicaBalance(settings, EMPTY_CLUSTER_SETTINGS); @@ -143,6 +169,10 @@ public void testForcedAwarenessAttributeDisabled() { assertThat(awarenessReplicaBalance.maxAwarenessAttributes(), equalTo(1)); assertEquals(awarenessReplicaBalance.validate(0, autoExpandReplica), Optional.empty()); assertEquals(awarenessReplicaBalance.validate(1, autoExpandReplica), Optional.empty()); + + AutoExpandSearchReplicas autoExpandSearchReplicas = AutoExpandSearchReplicas.SETTING.get(settings); + assertEquals(awarenessReplicaBalance.validate(0, autoExpandSearchReplicas), Optional.empty()); + assertEquals(awarenessReplicaBalance.validate(1, autoExpandSearchReplicas), Optional.empty()); } } diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index e69b5984bce8d..60cddfc2972cb 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -217,6 +217,7 @@ import reactor.util.annotation.NonNull; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.common.unit.TimeValue.timeValueMillis; import static org.opensearch.core.common.util.CollectionUtils.eagerPartition; @@ -2276,7 +2277,9 @@ protected NumShards getNumShards(String index) { assertThat(metadata.hasIndex(index), equalTo(true)); int numShards = Integer.valueOf(metadata.index(index).getSettings().get(SETTING_NUMBER_OF_SHARDS)); int numReplicas = Integer.valueOf(metadata.index(index).getSettings().get(SETTING_NUMBER_OF_REPLICAS)); - return new NumShards(numShards, numReplicas); + String numSearchReplicasValue = metadata.index(index).getSettings().get(SETTING_NUMBER_OF_SEARCH_REPLICAS); + int numSearchReplicas = numSearchReplicasValue != null ? Integer.parseInt(numSearchReplicasValue) : 0; + return new NumShards(numShards, numReplicas, numSearchReplicas); } /** @@ -2317,13 +2320,15 @@ public void assertSortedSegments(String indexName, Sort expectedIndexSort) { protected static class NumShards { public final int numPrimaries; public final int numReplicas; + public final int numSearchReplicas; public final int totalNumShards; public final int dataCopies; - private NumShards(int numPrimaries, int numReplicas) { + private NumShards(int numPrimaries, int numReplicas, int numSearchReplicas) { this.numPrimaries = numPrimaries; this.numReplicas = numReplicas; - this.dataCopies = numReplicas + 1; + this.numSearchReplicas = numSearchReplicas; + this.dataCopies = numReplicas + numSearchReplicas + 1; this.totalNumShards = numPrimaries * dataCopies; } }