Skip to content

Commit 610e69c

Browse files
github-actions[bot]github-actions[bot]
committed
Add setting to limit total number of shards on a cluster (#6143)
Signed-off-by: Rishav Sagar <rissag@amazon.com> (cherry picked from commit e42b76f) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
1 parent e53f6c2 commit 610e69c

5 files changed

Lines changed: 248 additions & 35 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
1414
- Add support to disallow search request with preference parameter with strict weighted shard routing([#5874](https://github.com/opensearch-project/OpenSearch/pull/5874))
1515
- Added support to apply index create block ([#4603](https://github.com/opensearch-project/OpenSearch/issues/4603))
1616
- Adds support for minimum compatible version for extensions ([#6003](https://github.com/opensearch-project/OpenSearch/pull/6003))
17+
- Add a guardrail to limit maximum number of shard on the cluster ([#6143](https://github.com/opensearch-project/OpenSearch/pull/6143))
1718

1819
### Dependencies
1920
- Update nebula-publishing-plugin to 19.2.0 ([#5704](https://github.com/opensearch-project/OpenSearch/pull/5704))

‎server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java‎

Lines changed: 51 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.opensearch.action.support.master.AcknowledgedResponse;
4242
import org.opensearch.client.Client;
4343
import org.opensearch.cluster.ClusterState;
44+
import org.opensearch.cluster.metadata.IndexMetadata;
4445
import org.opensearch.cluster.metadata.Metadata;
4546
import org.opensearch.common.Priority;
4647
import org.opensearch.common.network.NetworkModule;
@@ -68,19 +69,21 @@
6869

6970
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
7071
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
72+
import static org.opensearch.indices.ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE;
73+
import static org.opensearch.indices.ShardLimitValidator.SETTING_MAX_SHARDS_PER_CLUSTER_KEY;
7174
import static org.opensearch.test.NodeRoles.dataNode;
7275
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
7376
import static org.hamcrest.Matchers.equalTo;
7477
import static org.hamcrest.Matchers.greaterThan;
7578

7679
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST)
7780
public class ClusterShardLimitIT extends OpenSearchIntegTestCase {
78-
private static final String shardsPerNodeKey = ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey();
81+
private static final String shardsPerNodeKey = SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey();
7982
private static final String ignoreDotIndexKey = ShardLimitValidator.SETTING_CLUSTER_IGNORE_DOT_INDEXES.getKey();
8083

8184
public void testSettingClusterMaxShards() {
8285
int shardsPerNode = between(1, 500_000);
83-
setShardsPerNode(shardsPerNode);
86+
setMaxShardLimit(shardsPerNode, shardsPerNodeKey);
8487
}
8588

8689
public void testSettingIgnoreDotIndexes() {
@@ -118,7 +121,7 @@ public void testIndexCreationOverLimit() {
118121

119122
ShardCounts counts = ShardCounts.forDataNodeCount(dataNodes);
120123

121-
setShardsPerNode(counts.getShardsPerNode());
124+
setMaxShardLimit(counts.getShardsPerNode(), shardsPerNodeKey);
122125
// Create an index that will bring us up to the limit
123126
createIndex(
124127
"test",
@@ -155,7 +158,7 @@ public void testIndexCreationOverLimitForDotIndexesSucceeds() {
155158
int dataNodes = client().admin().cluster().prepareState().get().getState().getNodes().getDataNodes().size();
156159

157160
// Setting the cluster.max_shards_per_node setting according to the data node count.
158-
setShardsPerNode(dataNodes);
161+
setMaxShardLimit(dataNodes, shardsPerNodeKey);
159162
setIgnoreDotIndex(true);
160163

161164
/*
@@ -176,9 +179,7 @@ public void testIndexCreationOverLimitForDotIndexesSucceeds() {
176179

177180
// Getting cluster.max_shards_per_node setting
178181
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
179-
String maxShardsPerNode = clusterState.getMetadata()
180-
.settings()
181-
.get(ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey());
182+
String maxShardsPerNode = clusterState.getMetadata().settings().get(SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey());
182183

183184
// Checking if the total shards created are equivalent to dataNodes * cluster.max_shards_per_node
184185
assertEquals(dataNodes * Integer.parseInt(maxShardsPerNode), currentActiveShards);
@@ -203,7 +204,7 @@ public void testIndexCreationOverLimitForDotIndexesFail() {
203204
int maxAllowedShards = dataNodes * dataNodes;
204205

205206
// Setting the cluster.max_shards_per_node setting according to the data node count.
206-
setShardsPerNode(dataNodes);
207+
setMaxShardLimit(dataNodes, shardsPerNodeKey);
207208

208209
/*
209210
Create an index that will bring us up to the limit. It would create index with primary equal to the
@@ -223,9 +224,7 @@ public void testIndexCreationOverLimitForDotIndexesFail() {
223224

224225
// Getting cluster.max_shards_per_node setting
225226
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
226-
String maxShardsPerNode = clusterState.getMetadata()
227-
.settings()
228-
.get(ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey());
227+
String maxShardsPerNode = clusterState.getMetadata().settings().get(SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey());
229228

230229
// Checking if the total shards created are equivalent to dataNodes * cluster.max_shards_per_node
231230
assertEquals(dataNodes * Integer.parseInt(maxShardsPerNode), currentActiveShards);
@@ -247,6 +246,27 @@ public void testIndexCreationOverLimitForDotIndexesFail() {
247246
assertFalse(clusterState.getMetadata().hasIndex(".test-index"));
248247
}
249248

249+
public void testCreateIndexWithMaxClusterShardSetting() {
250+
int dataNodes = client().admin().cluster().prepareState().get().getState().getNodes().getDataNodes().size();
251+
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
252+
setMaxShardLimit(dataNodes, shardsPerNodeKey);
253+
254+
int maxAllowedShards = dataNodes + 1;
255+
int extraShardCount = maxAllowedShards + 1;
256+
// Getting total active shards in the cluster.
257+
int currentActiveShards = client().admin().cluster().prepareHealth().get().getActiveShards();
258+
try {
259+
setMaxShardLimit(maxAllowedShards, SETTING_MAX_SHARDS_PER_CLUSTER_KEY);
260+
prepareCreate("test_index_with_cluster_shard_limit").setSettings(
261+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, extraShardCount).put(SETTING_NUMBER_OF_REPLICAS, 0).build()
262+
).get();
263+
} catch (final IllegalArgumentException ex) {
264+
verifyException(Math.min(maxAllowedShards, dataNodes * dataNodes), currentActiveShards, extraShardCount, ex);
265+
} finally {
266+
setMaxShardLimit(-1, SETTING_MAX_SHARDS_PER_CLUSTER_KEY);
267+
}
268+
}
269+
250270
/**
251271
* The test checks if the index starting with the .ds- can be created if the node has
252272
* number of shards equivalent to the cluster.max_shards_per_node and the cluster.ignore_Dot_indexes
@@ -258,7 +278,7 @@ public void testIndexCreationOverLimitForDataStreamIndexes() {
258278
int maxAllowedShards = dataNodes * dataNodes;
259279

260280
// Setting the cluster.max_shards_per_node setting according to the data node count.
261-
setShardsPerNode(dataNodes);
281+
setMaxShardLimit(dataNodes, shardsPerNodeKey);
262282
setIgnoreDotIndex(true);
263283

264284
/*
@@ -279,9 +299,7 @@ public void testIndexCreationOverLimitForDataStreamIndexes() {
279299

280300
// Getting cluster.max_shards_per_node setting
281301
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
282-
String maxShardsPerNode = clusterState.getMetadata()
283-
.settings()
284-
.get(ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey());
302+
String maxShardsPerNode = clusterState.getMetadata().settings().get(SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey());
285303

286304
// Checking if the total shards created are equivalent to dataNodes * cluster.max_shards_per_node
287305
assertEquals(dataNodes * Integer.parseInt(maxShardsPerNode), currentActiveShards);
@@ -308,7 +326,7 @@ public void testIndexCreationOverLimitFromTemplate() {
308326

309327
final ShardCounts counts = ShardCounts.forDataNodeCount(dataNodes);
310328

311-
setShardsPerNode(counts.getShardsPerNode());
329+
setMaxShardLimit(counts.getShardsPerNode(), shardsPerNodeKey);
312330

313331
if (counts.getFirstIndexShards() > 0) {
314332
createIndex(
@@ -351,7 +369,7 @@ public void testIncreaseReplicasOverLimit() {
351369

352370
int firstShardCount = between(2, 10);
353371
int shardsPerNode = firstShardCount - 1;
354-
setShardsPerNode(shardsPerNode);
372+
setMaxShardLimit(shardsPerNode, shardsPerNodeKey);
355373

356374
prepareCreate(
357375
"growing-should-fail",
@@ -397,7 +415,7 @@ public void testChangingMultipleIndicesOverLimit() {
397415
int secondIndexReplicas = dataNodes;
398416

399417
int shardsPerNode = firstIndexFactor + (secondIndexFactor * (1 + secondIndexReplicas));
400-
setShardsPerNode(shardsPerNode);
418+
setMaxShardLimit(shardsPerNode, shardsPerNodeKey);
401419

402420
createIndex(
403421
"test-1-index",
@@ -448,7 +466,7 @@ public void testPreserveExistingSkipsCheck() {
448466

449467
int firstShardCount = between(2, 10);
450468
int shardsPerNode = firstShardCount - 1;
451-
setShardsPerNode(shardsPerNode);
469+
setMaxShardLimit(shardsPerNode, shardsPerNodeKey);
452470

453471
prepareCreate(
454472
"test-index",
@@ -521,7 +539,7 @@ public void testRestoreSnapshotOverLimit() {
521539
cluster().wipeIndices("snapshot-index");
522540

523541
// Reduce the shard limit and fill it up
524-
setShardsPerNode(counts.getShardsPerNode());
542+
setMaxShardLimit(counts.getShardsPerNode(), shardsPerNodeKey);
525543
createIndex(
526544
"test-fill",
527545
Settings.builder()
@@ -570,7 +588,7 @@ public void testOpenIndexOverLimit() {
570588
assertTrue(closeIndexResponse.isAcknowledged());
571589

572590
// Fill up the cluster
573-
setShardsPerNode(counts.getShardsPerNode());
591+
setMaxShardLimit(counts.getShardsPerNode(), shardsPerNodeKey);
574592
createIndex(
575593
"test-fill",
576594
Settings.builder()
@@ -704,27 +722,34 @@ private int ensureMultipleDataNodes(int dataNodes) {
704722
return dataNodes;
705723
}
706724

707-
private void setShardsPerNode(int shardsPerNode) {
725+
/**
726+
* Set max shard limit on either per node level or on cluster level.
727+
*
728+
* @param limit the limit value to set.
729+
* @param key node level or cluster level setting key.
730+
*/
731+
private void setMaxShardLimit(int limit, String key) {
708732
try {
709733
ClusterUpdateSettingsResponse response;
710734
if (frequently()) {
711735
response = client().admin()
712736
.cluster()
713737
.prepareUpdateSettings()
714-
.setPersistentSettings(Settings.builder().put(shardsPerNodeKey, shardsPerNode).build())
738+
.setPersistentSettings(Settings.builder().put(key, limit).build())
715739
.get();
716-
assertEquals(shardsPerNode, response.getPersistentSettings().getAsInt(shardsPerNodeKey, -1).intValue());
740+
assertEquals(limit, response.getPersistentSettings().getAsInt(key, -1).intValue());
717741
} else {
718742
response = client().admin()
719743
.cluster()
720744
.prepareUpdateSettings()
721-
.setTransientSettings(Settings.builder().put(shardsPerNodeKey, shardsPerNode).build())
745+
.setTransientSettings(Settings.builder().put(key, limit).build())
722746
.get();
723-
assertEquals(shardsPerNode, response.getTransientSettings().getAsInt(shardsPerNodeKey, -1).intValue());
747+
assertEquals(limit, response.getTransientSettings().getAsInt(key, -1).intValue());
724748
}
725749
} catch (IllegalArgumentException ex) {
726750
fail(ex.getMessage());
727751
}
752+
728753
}
729754

730755
private void setIgnoreDotIndex(boolean ignoreDotIndex) {

‎server/src/main/java/org/opensearch/common/settings/ClusterSettings.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,7 @@ public void apply(Settings value, Settings current, Settings previous) {
263263
Metadata.DEFAULT_REPLICA_COUNT_SETTING,
264264
Metadata.SETTING_CREATE_INDEX_BLOCK_SETTING,
265265
ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE,
266+
ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_CLUSTER,
266267
ShardLimitValidator.SETTING_CLUSTER_IGNORE_DOT_INDEXES,
267268
RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING,
268269
RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING,

0 commit comments

Comments
 (0)