-
Notifications
You must be signed in to change notification settings - Fork 2.5k
[Remote Store] Add support for Remote Translog Store upload stats in _nodes/stats/ API #8908
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
sachinpkale
merged 2 commits into
opensearch-project:main
from
BhumikaSaini-Amazon:rts-stats-upload-flow
Sep 6, 2023
Merged
Changes from all commits
Commits
Show all changes
2 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
267 changes: 0 additions & 267 deletions
267
...ternalClusterTest/java/org/opensearch/remotestore/RemoteSegmentStatsFromNodesStatsIT.java
This file was deleted.
Oops, something went wrong.
209 changes: 209 additions & 0 deletions
209
...internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsFromNodesStatsIT.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,209 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.remotestore; | ||
|
|
||
| import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; | ||
| import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsResponse; | ||
| import org.opensearch.action.admin.indices.stats.CommonStatsFlags; | ||
| import org.opensearch.common.settings.Settings; | ||
| import org.opensearch.index.remote.RemoteSegmentStats; | ||
| import org.opensearch.index.translog.RemoteTranslogStats; | ||
| import org.opensearch.test.OpenSearchIntegTestCase; | ||
| import org.junit.Before; | ||
|
|
||
| import java.util.concurrent.TimeUnit; | ||
|
|
||
| @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) | ||
| public class RemoteStoreStatsFromNodesStatsIT extends RemoteStoreBaseIntegTestCase { | ||
| private static final String INDEX_NAME = "remote-index-1"; | ||
| private static final int DATA_NODE_COUNT = 2; | ||
| private static final int CLUSTER_MANAGER_NODE_COUNT = 3; | ||
|
|
||
| @Before | ||
| public void setup() { | ||
| setupCustomCluster(); | ||
| } | ||
|
|
||
| private void setupCustomCluster() { | ||
| internalCluster().startClusterManagerOnlyNodes(CLUSTER_MANAGER_NODE_COUNT); | ||
| internalCluster().startDataOnlyNodes(DATA_NODE_COUNT); | ||
| ensureStableCluster(DATA_NODE_COUNT + CLUSTER_MANAGER_NODE_COUNT); | ||
| } | ||
|
|
||
| /** | ||
| * - Creates two indices with single primary shard, pinned to a single node. | ||
| * - Index documents in both of them and forces a fresh for both | ||
| * - Polls the _remotestore/stats API for individual index level stats | ||
| * - Adds up requisite fields from the API output, repeats this for the 2nd index | ||
| * - Polls _nodes/stats and verifies that the total values at node level adds up | ||
| * to the values capture in the previous step | ||
| */ | ||
| public void testNodesStatsParityWithOnlyPrimaryShards() { | ||
| String[] dataNodes = internalCluster().getDataNodeNames().toArray(String[]::new); | ||
| String randomDataNode = dataNodes[randomIntBetween(0, dataNodes.length - 1)]; | ||
| String firstIndex = INDEX_NAME + "1"; | ||
| String secondIndex = INDEX_NAME + "2"; | ||
|
|
||
| // Create first index | ||
| createIndex( | ||
| firstIndex, | ||
| Settings.builder().put(remoteStoreIndexSettings(0, 1)).put("index.routing.allocation.require._name", randomDataNode).build() | ||
| ); | ||
| ensureGreen(firstIndex); | ||
| indexSingleDoc(firstIndex, true); | ||
|
|
||
| // Create second index | ||
| createIndex( | ||
| secondIndex, | ||
| Settings.builder().put(remoteStoreIndexSettings(0, 1)).put("index.routing.allocation.require._name", randomDataNode).build() | ||
| ); | ||
| ensureGreen(secondIndex); | ||
| indexSingleDoc(secondIndex, true); | ||
|
|
||
| assertNodeStatsParityOnNode(randomDataNode, firstIndex, secondIndex); | ||
| } | ||
|
|
||
| /** | ||
| * - Creates two indices with single primary shard and single replica | ||
| * - Index documents in both of them and forces a fresh for both | ||
| * - Polls the _remotestore/stats API for individual index level stats | ||
| * - Adds up requisite fields from the API output for both indices | ||
| * - Polls _nodes/stats and verifies that the total values at node level adds up | ||
| * to the values capture in the previous step | ||
| * - Repeats the above 3 steps for the second node | ||
| */ | ||
| public void testNodesStatsParityWithReplicaShards() throws Exception { | ||
| String firstIndex = INDEX_NAME + "1"; | ||
| String secondIndex = INDEX_NAME + "2"; | ||
|
|
||
| createIndex(firstIndex, Settings.builder().put(remoteStoreIndexSettings(1, 1)).build()); | ||
| ensureGreen(firstIndex); | ||
| indexSingleDoc(firstIndex, true); | ||
|
|
||
| // Create second index | ||
| createIndex(secondIndex, Settings.builder().put(remoteStoreIndexSettings(1, 1)).build()); | ||
| ensureGreen(secondIndex); | ||
| indexSingleDoc(secondIndex, true); | ||
|
|
||
| assertBusy(() -> assertNodeStatsParityAcrossNodes(firstIndex, secondIndex), 15, TimeUnit.SECONDS); | ||
| } | ||
|
|
||
| /** | ||
| * Ensures that node stats shows 0 values for dedicated cluster manager nodes | ||
| * since cluster manager nodes does not participate in indexing | ||
| */ | ||
| public void testZeroRemoteStatsOnNodesStatsForClusterManager() { | ||
| createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); | ||
| ensureGreen(INDEX_NAME); | ||
| indexSingleDoc(INDEX_NAME); | ||
| refresh(INDEX_NAME); | ||
|
|
||
| NodesStatsResponse nodesStatsResponseForClusterManager = client().admin() | ||
| .cluster() | ||
| .prepareNodesStats(internalCluster().getClusterManagerName()) | ||
| .setIndices(new CommonStatsFlags().set(CommonStatsFlags.Flag.Segments, true).set(CommonStatsFlags.Flag.Translog, true)) | ||
| .get(); | ||
|
|
||
| assertTrue( | ||
| nodesStatsResponseForClusterManager.getNodes().get(0).getNode().isClusterManagerNode() | ||
| && !nodesStatsResponseForClusterManager.getNodes().get(0).getNode().isDataNode() | ||
| ); | ||
| assertZeroRemoteSegmentStats( | ||
| nodesStatsResponseForClusterManager.getNodes().get(0).getIndices().getSegments().getRemoteSegmentStats() | ||
| ); | ||
| assertZeroRemoteTranslogStats( | ||
| nodesStatsResponseForClusterManager.getNodes().get(0).getIndices().getTranslog().getRemoteTranslogStats() | ||
| ); | ||
|
|
||
| NodesStatsResponse nodesStatsResponseForDataNode = client().admin() | ||
| .cluster() | ||
| .prepareNodesStats(primaryNodeName(INDEX_NAME)) | ||
| .setIndices(new CommonStatsFlags().set(CommonStatsFlags.Flag.Segments, true).set(CommonStatsFlags.Flag.Translog, true)) | ||
| .get(); | ||
|
|
||
| assertTrue(nodesStatsResponseForDataNode.getNodes().get(0).getNode().isDataNode()); | ||
| RemoteSegmentStats remoteSegmentStats = nodesStatsResponseForDataNode.getNodes() | ||
| .get(0) | ||
| .getIndices() | ||
| .getSegments() | ||
| .getRemoteSegmentStats(); | ||
| assertTrue(remoteSegmentStats.getUploadBytesStarted() > 0); | ||
| assertTrue(remoteSegmentStats.getUploadBytesSucceeded() > 0); | ||
|
|
||
| RemoteTranslogStats remoteTranslogStats = nodesStatsResponseForDataNode.getNodes() | ||
| .get(0) | ||
| .getIndices() | ||
| .getTranslog() | ||
| .getRemoteTranslogStats(); | ||
| assertTrue(remoteTranslogStats.getUploadBytesStarted() > 0); | ||
| assertTrue(remoteTranslogStats.getUploadBytesSucceeded() > 0); | ||
| } | ||
|
|
||
| private void assertZeroRemoteSegmentStats(RemoteSegmentStats remoteSegmentStats) { | ||
| // Compare with fresh object because all values default to 0 in default fresh object | ||
| assertEquals(new RemoteSegmentStats(), remoteSegmentStats); | ||
| } | ||
|
|
||
| private void assertZeroRemoteTranslogStats(RemoteTranslogStats remoteTranslogStats) { | ||
| // Compare with fresh object because all values default to 0 in default fresh object | ||
| assertEquals(new RemoteTranslogStats(), remoteTranslogStats); | ||
| } | ||
|
|
||
| private static void assertNodeStatsParityAcrossNodes(String... indices) { | ||
| for (String dataNode : internalCluster().getDataNodeNames()) { | ||
| assertNodeStatsParityOnNode(dataNode, indices); | ||
| } | ||
| } | ||
|
|
||
| private static void assertNodeStatsParityOnNode(String dataNode, String... indices) { | ||
| RemoteSegmentStats remoteSegmentStatsCumulative = new RemoteSegmentStats(); | ||
| RemoteTranslogStats remoteTranslogStatsCumulative = new RemoteTranslogStats(); | ||
| for (String index : indices) { | ||
| // Fetch _remotestore/stats | ||
| RemoteStoreStatsResponse remoteStoreStats = client(dataNode).admin() | ||
| .cluster() | ||
| .prepareRemoteStoreStats(index, "0") | ||
| .setLocal(true) | ||
| .get(); | ||
| remoteSegmentStatsCumulative.add(new RemoteSegmentStats(remoteStoreStats.getRemoteStoreStats()[0].getSegmentStats())); | ||
| remoteTranslogStatsCumulative.add(new RemoteTranslogStats(remoteStoreStats.getRemoteStoreStats()[0].getTranslogStats())); | ||
| } | ||
|
|
||
| // Fetch _nodes/stats | ||
| NodesStatsResponse nodesStatsResponse = client().admin() | ||
| .cluster() | ||
| .prepareNodesStats(dataNode) | ||
| .setIndices(new CommonStatsFlags().set(CommonStatsFlags.Flag.Segments, true).set(CommonStatsFlags.Flag.Translog, true)) | ||
| .get(); | ||
|
|
||
| // assert segment stats | ||
| RemoteSegmentStats remoteSegmentStatsFromNodesStats = nodesStatsResponse.getNodes() | ||
| .get(0) | ||
| .getIndices() | ||
| .getSegments() | ||
| .getRemoteSegmentStats(); | ||
| assertEquals(remoteSegmentStatsCumulative, remoteSegmentStatsFromNodesStats); | ||
| // Ensure that total upload time has non-zero value if there has been segments uploaded from the node | ||
| if (remoteSegmentStatsCumulative.getUploadBytesStarted() > 0) { | ||
| assertTrue(remoteSegmentStatsCumulative.getTotalUploadTime() > 0); | ||
| } | ||
| // Ensure that total download time has non-zero value if there has been segments downloaded to the node | ||
| if (remoteSegmentStatsCumulative.getDownloadBytesStarted() > 0) { | ||
| assertTrue(remoteSegmentStatsCumulative.getTotalDownloadTime() > 0); | ||
| } | ||
|
|
||
| // assert translog stats | ||
| RemoteTranslogStats remoteTranslogStatsFromNodesStats = nodesStatsResponse.getNodes() | ||
| .get(0) | ||
| .getIndices() | ||
| .getTranslog() | ||
| .getRemoteTranslogStats(); | ||
| assertEquals(remoteTranslogStatsCumulative, remoteTranslogStatsFromNodesStats); | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.