diff --git a/qa/smoke-test-http/src/test/java/org/opensearch/http/ShardIndexingPressureRestIT.java b/qa/smoke-test-http/src/test/java/org/opensearch/http/ShardIndexingPressureRestIT.java index 9e663ac042d91..8b582d2b28f15 100644 --- a/qa/smoke-test-http/src/test/java/org/opensearch/http/ShardIndexingPressureRestIT.java +++ b/qa/smoke-test-http/src/test/java/org/opensearch/http/ShardIndexingPressureRestIT.java @@ -16,6 +16,7 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.XContentTestUtils; +import java.io.InputStream; import java.util.ArrayList; import java.util.Map; @@ -29,8 +30,12 @@ /** * Test Shard Indexing Pressure Metrics and Statistics */ -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 2, - numClientNodes = 0) +@OpenSearchIntegTestCase.ClusterScope( + scope = OpenSearchIntegTestCase.Scope.SUITE, + supportsDedicatedMasters = false, + numDataNodes = 2, + numClientNodes = 0 +) public class ShardIndexingPressureRestIT extends HttpSmokeTestCase { @Override @@ -43,30 +48,61 @@ protected Settings nodeSettings(int nodeOrdinal) { .build(); } + /** + * Response is NOT AutoCloseable. To avoid leaking tracked HTTP channels, drain and close the entity stream. + * Best-effort: any exception is ignored since this is test cleanup. + */ + private static void consumeEntity(Response response) { + if (response == null) return; + try { + if (response.getEntity() == null) return; + try (InputStream is = response.getEntity().getContent()) { + byte[] buf = new byte[8192]; + while (is.read(buf) != -1) { + } + } + } catch (Exception ignored) { + } + } + @SuppressWarnings("unchecked") public void testShardIndexingPressureStats() throws Exception { Request createRequest = new Request("PUT", "/index_name"); - createRequest.setJsonEntity("{\"settings\": {\"index\": {\"number_of_shards\": 1, \"number_of_replicas\": 1, " + - "\"write.wait_for_active_shards\": 2}}}"); + createRequest.setJsonEntity( + "{\"settings\": {\"index\": {\"number_of_shards\": 1, \"number_of_replicas\": 1, \"write.wait_for_active_shards\": 2}}}" + ); final Response indexCreatedResponse = getRestClient().performRequest(createRequest); - assertThat(indexCreatedResponse.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); + try { + assertThat(indexCreatedResponse.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); + } finally { + consumeEntity(indexCreatedResponse); + } Request successfulIndexingRequest = new Request("POST", "/index_name/_doc/"); successfulIndexingRequest.setJsonEntity("{\"x\": \"small text\"}"); final Response indexSuccessFul = getRestClient().performRequest(successfulIndexingRequest); - assertThat(indexSuccessFul.getStatusLine().getStatusCode(), equalTo(CREATED.getStatus())); + try { + assertThat(indexSuccessFul.getStatusLine().getStatusCode(), equalTo(CREATED.getStatus())); + } finally { + consumeEntity(indexSuccessFul); + } Request getShardStats1 = new Request("GET", "/_nodes/stats/shard_indexing_pressure?include_all"); final Response shardStats1 = getRestClient().performRequest(getShardStats1); - Map shardStatsMap1 = XContentHelper.convertToMap(JsonXContent.jsonXContent, shardStats1.getEntity().getContent(), - true); + final Map shardStatsMap1; + try (InputStream is = shardStats1.getEntity().getContent()) { + shardStatsMap1 = XContentHelper.convertToMap(JsonXContent.jsonXContent, is, true); + } finally { + consumeEntity(shardStats1); + } ArrayList values1 = new ArrayList<>(((Map) shardStatsMap1.get("nodes")).values()); assertThat(values1.size(), equalTo(2)); XContentTestUtils.JsonMapView node1 = new XContentTestUtils.JsonMapView((Map) values1.get(0)); - ArrayList shard1IndexingPressureValues = new ArrayList<>(((Map) node1.get("shard_indexing_pressure.stats")) - .values()); + ArrayList shard1IndexingPressureValues = + new ArrayList<>(((Map) node1.get("shard_indexing_pressure.stats")).values()); assertThat(shard1IndexingPressureValues.size(), equalTo(1)); - XContentTestUtils.JsonMapView shard1 = new XContentTestUtils.JsonMapView((Map) shard1IndexingPressureValues.get(0)); + XContentTestUtils.JsonMapView shard1 = + new XContentTestUtils.JsonMapView((Map) shard1IndexingPressureValues.get(0)); Integer node1TotalLimitsRejections = node1.get("shard_indexing_pressure.total_rejections_breakup.node_limits"); Integer shard1CoordinatingBytes = shard1.get("memory.total.coordinating_in_bytes"); Integer shard1PrimaryBytes = shard1.get("memory.total.primary_in_bytes"); @@ -76,18 +112,17 @@ public void testShardIndexingPressureStats() throws Exception { Integer shard1CoordinatingNodeRejections = shard1.get("rejection.coordinating.breakup.node_limits"); XContentTestUtils.JsonMapView node2 = new XContentTestUtils.JsonMapView((Map) values1.get(1)); - ArrayList shard2IndexingPressureValues = new ArrayList<>(((Map) node2.get("shard_indexing_pressure.stats")) - .values()); + ArrayList shard2IndexingPressureValues = + new ArrayList<>(((Map) node2.get("shard_indexing_pressure.stats")).values()); assertThat(shard2IndexingPressureValues.size(), equalTo(1)); - XContentTestUtils.JsonMapView shard2 = new XContentTestUtils.JsonMapView((Map) shard2IndexingPressureValues.get(0)); + XContentTestUtils.JsonMapView shard2 = + new XContentTestUtils.JsonMapView((Map) shard2IndexingPressureValues.get(0)); Integer node2TotalLimitsRejections = node2.get("shard_indexing_pressure.total_rejections_breakup.node_limits"); Integer shard2CoordinatingBytes = shard2.get("memory.total.coordinating_in_bytes"); Integer shard2PrimaryBytes = shard2.get("memory.total.primary_in_bytes"); Integer shard2ReplicaBytes = shard2.get("memory.total.replica_in_bytes"); Integer shard2CoordinatingRejections = shard2.get("rejection.coordinating.coordinating_rejections"); Integer shard2PrimaryRejections = shard2.get("rejection.primary.primary_rejections"); - Integer shard2CoordinatingNodeRejections = shard2.get("rejection.coordinating.breakup.node_limits"); - if (shard1CoordinatingBytes == 0) { assertThat(shard2CoordinatingBytes, greaterThan(0)); @@ -121,21 +156,26 @@ public void testShardIndexingPressureStats() throws Exception { Request getNodeStats = new Request("GET", "/_nodes/stats/indexing_pressure"); final Response nodeStats = getRestClient().performRequest(getNodeStats); - Map nodeStatsMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, nodeStats.getEntity().getContent(), true); + final Map nodeStatsMap; + try (InputStream is = nodeStats.getEntity().getContent()) { + nodeStatsMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, is, true); + } finally { + consumeEntity(nodeStats); + } values1 = new ArrayList<>(((Map) nodeStatsMap.get("nodes")).values()); assertThat(values1.size(), equalTo(2)); node1 = new XContentTestUtils.JsonMapView((Map) values1.get(0)); Integer node1CombinedBytes = node1.get("indexing_pressure.memory.total.combined_coordinating_and_primary_in_bytes"); - Integer node1PrimaryBytes = node1.get("indexing_pressure.memory.total.primary_in_bytes"); - Integer node1ReplicaBytes = node1.get("indexing_pressure.memory.total.replica_in_bytes"); + Integer node1PrimaryBytes2 = node1.get("indexing_pressure.memory.total.primary_in_bytes"); + Integer node1ReplicaBytes2 = node1.get("indexing_pressure.memory.total.replica_in_bytes"); Integer node1CoordinatingRejections = node1.get("indexing_pressure.memory.total.coordinating_rejections"); - Integer node1PrimaryRejections = node1.get("indexing_pressure.memory.total.primary_rejections"); + Integer node1PrimaryRejections2 = node1.get("indexing_pressure.memory.total.primary_rejections"); node2 = new XContentTestUtils.JsonMapView((Map) values1.get(1)); Integer node2IndexingBytes = node2.get("indexing_pressure.memory.total.combined_coordinating_and_primary_in_bytes"); - Integer node2PrimaryBytes = node2.get("indexing_pressure.memory.total.primary_in_bytes"); - Integer node2ReplicaBytes = node2.get("indexing_pressure.memory.total.replica_in_bytes"); - Integer node2CoordinatingRejections = node2.get("indexing_pressure.memory.total.coordinating_rejections"); - Integer node2PrimaryRejections = node2.get("indexing_pressure.memory.total.primary_rejections"); + Integer node2PrimaryBytes2 = node2.get("indexing_pressure.memory.total.primary_in_bytes"); + Integer node2ReplicaBytes2 = node2.get("indexing_pressure.memory.total.replica_in_bytes"); + Integer node2CoordinatingRejections2 = node2.get("indexing_pressure.memory.total.coordinating_rejections"); + Integer node2PrimaryRejections2 = node2.get("indexing_pressure.memory.total.primary_rejections"); if (node1CombinedBytes == 0) { assertThat(node2IndexingBytes, greaterThan(0)); @@ -145,62 +185,72 @@ public void testShardIndexingPressureStats() throws Exception { assertThat(node1CombinedBytes, lessThan(1024)); } - if (node1ReplicaBytes == 0) { - assertThat(node1PrimaryBytes, greaterThan(0)); - assertThat(node1PrimaryBytes, lessThan(1024)); + if (node1ReplicaBytes2 == 0) { + assertThat(node1PrimaryBytes2, greaterThan(0)); + assertThat(node1PrimaryBytes2, lessThan(1024)); - assertThat(node2ReplicaBytes, greaterThan(0)); - assertThat(node2ReplicaBytes, lessThan(1024)); + assertThat(node2ReplicaBytes2, greaterThan(0)); + assertThat(node2ReplicaBytes2, lessThan(1024)); } else { - assertThat(node2PrimaryBytes, greaterThan(0)); - assertThat(node2PrimaryBytes, lessThan(1024)); + assertThat(node2PrimaryBytes2, greaterThan(0)); + assertThat(node2PrimaryBytes2, lessThan(1024)); - assertThat(node2ReplicaBytes, equalTo(0)); - assertThat(node1ReplicaBytes, lessThan(1024)); + assertThat(node2ReplicaBytes2, equalTo(0)); + assertThat(node1ReplicaBytes2, lessThan(1024)); } assertThat(node1CoordinatingRejections, equalTo(0)); - assertThat(node1PrimaryRejections, equalTo(0)); - assertThat(node2CoordinatingRejections, equalTo(0)); - assertThat(node2PrimaryRejections, equalTo(0)); + assertThat(node1PrimaryRejections2, equalTo(0)); + assertThat(node2CoordinatingRejections2, equalTo(0)); + assertThat(node2PrimaryRejections2, equalTo(0)); Request failedIndexingRequest = new Request("POST", "/index_name/_doc/"); String largeString = randomAlphaOfLength(10000); - failedIndexingRequest.setJsonEntity("{\"x\": " + largeString + "}"); + failedIndexingRequest.setJsonEntity("{\"x\": \"" + largeString + "\"}"); ResponseException exception = expectThrows(ResponseException.class, () -> getRestClient().performRequest(failedIndexingRequest)); - assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(TOO_MANY_REQUESTS.getStatus())); + try { + assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(TOO_MANY_REQUESTS.getStatus())); + } finally { + consumeEntity(exception.getResponse()); + } Request getShardStats2 = new Request("GET", "/_nodes/stats/shard_indexing_pressure?include_all"); final Response shardStats2 = getRestClient().performRequest(getShardStats2); - Map shardStatsMap2 = XContentHelper.convertToMap(JsonXContent.jsonXContent, shardStats2.getEntity().getContent(), - true); + final Map shardStatsMap2; + try (InputStream is = shardStats2.getEntity().getContent()) { + shardStatsMap2 = XContentHelper.convertToMap(JsonXContent.jsonXContent, is, true); + } finally { + consumeEntity(shardStats2); + } ArrayList values2 = new ArrayList<>(((Map) shardStatsMap2.get("nodes")).values()); assertThat(values2.size(), equalTo(2)); XContentTestUtils.JsonMapView node1AfterRejection = new XContentTestUtils.JsonMapView((Map) values2.get(0)); - ArrayList shard1IndexingPressureValuesAfterRejection = new ArrayList<>(((Map) node1AfterRejection - .get("shard_indexing_pressure.stats")).values()); + ArrayList shard1IndexingPressureValuesAfterRejection = new ArrayList<>( + ((Map) node1AfterRejection.get("shard_indexing_pressure.stats")).values() + ); assertThat(shard1IndexingPressureValuesAfterRejection.size(), equalTo(1)); - XContentTestUtils.JsonMapView shard1AfterRejection = new XContentTestUtils.JsonMapView((Map) - shard1IndexingPressureValuesAfterRejection.get(0)); + XContentTestUtils.JsonMapView shard1AfterRejection = + new XContentTestUtils.JsonMapView((Map) shard1IndexingPressureValuesAfterRejection.get(0)); node1TotalLimitsRejections = node1AfterRejection.get("shard_indexing_pressure.total_rejections_breakup.node_limits"); shard1CoordinatingRejections = shard1AfterRejection.get("rejection.coordinating.coordinating_rejections"); shard1PrimaryRejections = shard1AfterRejection.get("rejection.primary.primary_rejections"); shard1CoordinatingNodeRejections = shard1AfterRejection.get("rejection.coordinating.breakup.node_limits"); XContentTestUtils.JsonMapView node2AfterRejection = new XContentTestUtils.JsonMapView((Map) values2.get(1)); - ArrayList shard2IndexingPressureValuesAfterRejection = new ArrayList<>(((Map) node2AfterRejection - .get("shard_indexing_pressure.stats")).values()); + ArrayList shard2IndexingPressureValuesAfterRejection = new ArrayList<>( + ((Map) node2AfterRejection.get("shard_indexing_pressure.stats")).values() + ); assertThat(shard2IndexingPressureValuesAfterRejection.size(), equalTo(1)); - XContentTestUtils.JsonMapView shard2AfterRejection = new XContentTestUtils.JsonMapView((Map) - shard2IndexingPressureValuesAfterRejection.get(0)); + XContentTestUtils.JsonMapView shard2AfterRejection = + new XContentTestUtils.JsonMapView((Map) shard2IndexingPressureValuesAfterRejection.get(0)); node2TotalLimitsRejections = node2AfterRejection.get("shard_indexing_pressure.total_rejections_breakup.node_limits"); - shard2CoordinatingRejections = shard2AfterRejection.get("rejection.coordinating.coordinating_rejections"); - shard2PrimaryRejections = shard2AfterRejection.get("rejection.primary.primary_rejections"); - shard2CoordinatingNodeRejections = shard2AfterRejection.get("rejection.coordinating.breakup.node_limits"); + Integer shard2CoordinatingRejectionsAfterRej = shard2AfterRejection.get("rejection.coordinating.coordinating_rejections"); + Integer shard2PrimaryRejectionsAfterRej = shard2AfterRejection.get("rejection.primary.primary_rejections"); + Integer shard2CoordinatingNodeRejectionsAfterRej = shard2AfterRejection.get("rejection.coordinating.breakup.node_limits"); if (shard1CoordinatingRejections == 0) { - assertThat(shard2CoordinatingRejections, equalTo(1)); - assertThat(shard2CoordinatingNodeRejections, equalTo(1)); + assertThat(shard2CoordinatingRejectionsAfterRej, equalTo(1)); + assertThat(shard2CoordinatingNodeRejectionsAfterRej, equalTo(1)); assertThat(node2TotalLimitsRejections, equalTo(1)); } else { assertThat(shard1CoordinatingRejections, equalTo(1)); @@ -209,89 +259,112 @@ public void testShardIndexingPressureStats() throws Exception { } assertThat(shard1PrimaryRejections, equalTo(0)); - assertThat(shard2PrimaryRejections, equalTo(0)); + assertThat(shard2PrimaryRejectionsAfterRej, equalTo(0)); Request getNodeStats2 = new Request("GET", "/_nodes/stats/indexing_pressure"); final Response nodeStats2 = getRestClient().performRequest(getNodeStats2); - Map nodeStatsMap2 = XContentHelper.convertToMap(JsonXContent.jsonXContent, nodeStats2.getEntity().getContent(), - true); + final Map nodeStatsMap2; + try (InputStream is = nodeStats2.getEntity().getContent()) { + nodeStatsMap2 = XContentHelper.convertToMap(JsonXContent.jsonXContent, is, true); + } finally { + consumeEntity(nodeStats2); + } values2 = new ArrayList<>(((Map) nodeStatsMap2.get("nodes")).values()); assertThat(values2.size(), equalTo(2)); node1AfterRejection = new XContentTestUtils.JsonMapView((Map) values2.get(0)); node1CoordinatingRejections = node1AfterRejection.get("indexing_pressure.memory.total.coordinating_rejections"); - node1PrimaryRejections = node1.get("indexing_pressure.memory.total.primary_rejections"); + Integer node1PrimaryRejectionsAfter = node1AfterRejection.get("indexing_pressure.memory.total.primary_rejections"); node2AfterRejection = new XContentTestUtils.JsonMapView((Map) values2.get(1)); - node2CoordinatingRejections = node2AfterRejection.get("indexing_pressure.memory.total.coordinating_rejections"); - node2PrimaryRejections = node2AfterRejection.get("indexing_pressure.memory.total.primary_rejections"); + Integer node2CoordinatingRejectionsAfter = node2AfterRejection.get("indexing_pressure.memory.total.coordinating_rejections"); + Integer node2PrimaryRejectionsAfter = node2AfterRejection.get("indexing_pressure.memory.total.primary_rejections"); if (node1CoordinatingRejections == 0) { - assertThat(node2CoordinatingRejections, equalTo(1)); + assertThat(node2CoordinatingRejectionsAfter, equalTo(1)); } else { assertThat(node1CoordinatingRejections, equalTo(1)); } - assertThat(node1PrimaryRejections, equalTo(0)); - assertThat(node2PrimaryRejections, equalTo(0)); + assertThat(node1PrimaryRejectionsAfter, equalTo(0)); + assertThat(node2PrimaryRejectionsAfter, equalTo(0)); // Update cluster setting to enable shadow mode Request updateSettingRequest = new Request("PUT", "/_cluster/settings"); updateSettingRequest.setJsonEntity("{\"persistent\": {\"shard_indexing_pressure\": {\"enforced\": \"false\"}}}"); final Response updateSettingResponse = getRestClient().performRequest(updateSettingRequest); - assertThat(updateSettingResponse.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); + try { + assertThat(updateSettingResponse.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); + } finally { + consumeEntity(updateSettingResponse); + } Request shadowIndexingRequest = new Request("POST", "/index_name/_doc/"); shadowIndexingRequest.setJsonEntity("{\"x\": \"text\"}"); final Response shadowIndexingResponse = getRestClient().performRequest(shadowIndexingRequest); - assertThat(shadowIndexingResponse.getStatusLine().getStatusCode(), equalTo(CREATED.getStatus())); + try { + assertThat(shadowIndexingResponse.getStatusLine().getStatusCode(), equalTo(CREATED.getStatus())); + } finally { + consumeEntity(shadowIndexingResponse); + } Request getShardStats3 = new Request("GET", "/_nodes/stats/shard_indexing_pressure?include_all"); final Response shardStats3 = getRestClient().performRequest(getShardStats3); - Map shardStatsMap3 = XContentHelper.convertToMap(JsonXContent.jsonXContent, shardStats3.getEntity().getContent(), - true); + final Map shardStatsMap3; + try (InputStream is = shardStats3.getEntity().getContent()) { + shardStatsMap3 = XContentHelper.convertToMap(JsonXContent.jsonXContent, is, true); + } finally { + consumeEntity(shardStats3); + } ArrayList values3 = new ArrayList<>(((Map) shardStatsMap3.get("nodes")).values()); assertThat(values3.size(), equalTo(2)); - XContentTestUtils.JsonMapView node1AfterShadowMode = new XContentTestUtils.JsonMapView((Map) - values3.get(0)); - ArrayList shard1IndexingPressureValuesAfterShadowMode = new ArrayList<>(((Map) - node1AfterShadowMode.get("shard_indexing_pressure.stats")).values()); + XContentTestUtils.JsonMapView node1AfterShadowMode = new XContentTestUtils.JsonMapView((Map) values3.get(0)); + ArrayList shard1IndexingPressureValuesAfterShadowMode = new ArrayList<>( + ((Map) node1AfterShadowMode.get("shard_indexing_pressure.stats")).values() + ); assertThat(shard1IndexingPressureValuesAfterShadowMode.size(), equalTo(1)); - XContentTestUtils.JsonMapView shard1AfterShadowMode = new XContentTestUtils.JsonMapView((Map) - shard1IndexingPressureValuesAfterShadowMode.get(0)); - node1TotalLimitsRejections = node1AfterShadowMode.get("shard_indexing_pressure.total_rejections_breakup_shadow_mode" + - ".node_limits"); - shard1CoordinatingRejections = shard1AfterShadowMode.get("rejection.coordinating.coordinating_rejections"); - shard1PrimaryRejections = shard1AfterShadowMode.get("rejection.primary.primary_rejections"); - shard1CoordinatingNodeRejections = shard1AfterShadowMode.get("rejection.coordinating.breakup_shadow_mode.node_limits"); - - XContentTestUtils.JsonMapView node2AfterShadowMode = new XContentTestUtils.JsonMapView((Map) - values3.get(1)); - ArrayList shard2IndexingPressureValuesAfterShadowMode = new ArrayList<>(((Map) - node2AfterShadowMode.get("shard_indexing_pressure.stats")).values()); + XContentTestUtils.JsonMapView shard1AfterShadowMode = + new XContentTestUtils.JsonMapView((Map) shard1IndexingPressureValuesAfterShadowMode.get(0)); + Integer node1TotalLimitsRejectionsShadow = node1AfterShadowMode.get( + "shard_indexing_pressure.total_rejections_breakup_shadow_mode.node_limits" + ); + Integer shard1CoordinatingRejectionsShadow = shard1AfterShadowMode.get("rejection.coordinating.coordinating_rejections"); + Integer shard1PrimaryRejectionsShadow = shard1AfterShadowMode.get("rejection.primary.primary_rejections"); + Integer shard1CoordinatingNodeRejectionsShadow = shard1AfterShadowMode.get( + "rejection.coordinating.breakup_shadow_mode.node_limits" + ); + + XContentTestUtils.JsonMapView node2AfterShadowMode = new XContentTestUtils.JsonMapView((Map) values3.get(1)); + ArrayList shard2IndexingPressureValuesAfterShadowMode = new ArrayList<>( + ((Map) node2AfterShadowMode.get("shard_indexing_pressure.stats")).values() + ); assertThat(shard2IndexingPressureValuesAfterShadowMode.size(), equalTo(1)); - XContentTestUtils.JsonMapView shard2AfterShadowMode = new XContentTestUtils.JsonMapView((Map) - shard2IndexingPressureValuesAfterShadowMode.get(0)); - node2TotalLimitsRejections = node2AfterShadowMode.get("shard_indexing_pressure.total_rejections_breakup_shadow_mode" + - ".node_limits"); - shard2CoordinatingRejections = shard2AfterShadowMode.get("rejection.coordinating.coordinating_rejections"); - shard2PrimaryRejections = shard2AfterShadowMode.get("rejection.primary.primary_rejections"); - shard2CoordinatingNodeRejections = shard2AfterShadowMode.get("rejection.coordinating.breakup_shadow_mode.node_limits"); - - if (shard1CoordinatingRejections == 0) { - assertThat(shard2CoordinatingRejections, equalTo(1)); - assertThat(shard2CoordinatingNodeRejections, equalTo(1)); - assertThat(node2TotalLimitsRejections, equalTo(1)); + XContentTestUtils.JsonMapView shard2AfterShadowMode = + new XContentTestUtils.JsonMapView((Map) shard2IndexingPressureValuesAfterShadowMode.get(0)); + Integer node2TotalLimitsRejectionsShadow = node2AfterShadowMode.get( + "shard_indexing_pressure.total_rejections_breakup_shadow_mode.node_limits" + ); + Integer shard2CoordinatingRejectionsShadow = shard2AfterShadowMode.get("rejection.coordinating.coordinating_rejections"); + Integer shard2PrimaryRejectionsShadow = shard2AfterShadowMode.get("rejection.primary.primary_rejections"); + Integer shard2CoordinatingNodeRejectionsShadow = shard2AfterShadowMode.get( + "rejection.coordinating.breakup_shadow_mode.node_limits" + ); + + if (shard1CoordinatingRejectionsShadow == 0) { + assertThat(shard2CoordinatingRejectionsShadow, equalTo(1)); + assertThat(shard2CoordinatingNodeRejectionsShadow, equalTo(1)); + assertThat(node2TotalLimitsRejectionsShadow, equalTo(1)); } else { - assertThat(shard1CoordinatingRejections, equalTo(1)); - assertThat(shard1CoordinatingNodeRejections, equalTo(1)); - assertThat(node1TotalLimitsRejections, equalTo(1)); + assertThat(shard1CoordinatingRejectionsShadow, equalTo(1)); + assertThat(shard1CoordinatingNodeRejectionsShadow, equalTo(1)); + assertThat(node1TotalLimitsRejectionsShadow, equalTo(1)); } - assertThat(shard1PrimaryRejections, equalTo(0)); - assertThat(shard2PrimaryRejections, equalTo(0)); + assertThat(shard1PrimaryRejectionsShadow, equalTo(0)); + assertThat(shard2PrimaryRejectionsShadow, equalTo(0)); //Reset persistent setting to clear cluster metadata updateSettingRequest = new Request("PUT", "/_cluster/settings"); updateSettingRequest.setJsonEntity("{\"persistent\": {\"shard_indexing_pressure\": {\"enforced\": null}}}"); - getRestClient().performRequest(updateSettingRequest); + final Response resetResponse = getRestClient().performRequest(updateSettingRequest); + consumeEntity(resetResponse); } }