Skip to content

Commit c8cdacf

Browse files
linuxpilinuxpi
committed
[Remote Cluster State] Move remote cluster state related settings to node attrs (opensearch-project#9811)
--------- Signed-off-by: bansvaru <bansvaru@amazon.com> (cherry picked from commit d58943d)
1 parent c0e9f4f commit c8cdacf

File tree

16 files changed

+230
-81
lines changed

16 files changed

+230
-81
lines changed

‎server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import java.util.stream.Collectors;
3232

3333
import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING;
34+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
3435
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
3536
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
3637
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
@@ -91,6 +92,16 @@ public Settings buildRemoteStoreNodeAttributes(Path repoLocation, double ioFailu
9192
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
9293
TRANSLOG_REPOSITORY_NAME
9394
);
95+
String stateRepoTypeAttributeKey = String.format(
96+
Locale.getDefault(),
97+
"node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
98+
REPOSITORY_NAME
99+
);
100+
String stateRepoSettingsAttributeKeyPrefix = String.format(
101+
Locale.getDefault(),
102+
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
103+
REPOSITORY_NAME
104+
);
94105

95106
return Settings.builder()
96107
.put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME)
@@ -104,6 +115,9 @@ public Settings buildRemoteStoreNodeAttributes(Path repoLocation, double ioFailu
104115
.put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, TRANSLOG_REPOSITORY_NAME)
105116
.put(translogRepoTypeAttributeKey, "mock")
106117
.put(translogRepoSettingsAttributeKeyPrefix + "location", repoLocation)
118+
.put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME)
119+
.put(stateRepoTypeAttributeKey, "mock")
120+
.put(stateRepoSettingsAttributeKeyPrefix + "location", repoLocation)
107121
.build();
108122
}
109123

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import java.util.stream.Collectors;
5050

5151
import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING;
52+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
5253
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
5354
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
5455
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
@@ -222,14 +223,27 @@ public static Settings buildRemoteStoreNodeAttributes(
222223
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
223224
translogRepoName
224225
);
226+
String stateRepoTypeAttributeKey = String.format(
227+
Locale.getDefault(),
228+
"node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
229+
segmentRepoName
230+
);
231+
String stateRepoSettingsAttributeKeyPrefix = String.format(
232+
Locale.getDefault(),
233+
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
234+
segmentRepoName
235+
);
225236

226237
Settings.Builder settings = Settings.builder()
227238
.put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName)
228239
.put(segmentRepoTypeAttributeKey, FsRepository.TYPE)
229240
.put(segmentRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath)
230241
.put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName)
231242
.put(translogRepoTypeAttributeKey, FsRepository.TYPE)
232-
.put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath);
243+
.put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath)
244+
.put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName)
245+
.put(stateRepoTypeAttributeKey, FsRepository.TYPE)
246+
.put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath);
233247

234248
if (withRateLimiterAttributes) {
235249
settings.put(segmentRepoSettingsAttributeKeyPrefix + "compress", randomBoolean())

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import java.util.concurrent.ExecutionException;
2424

2525
import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
26-
import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_REPOSITORY_SETTING;
2726
import static org.opensearch.indices.ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE;
2827
import static org.opensearch.indices.ShardLimitValidator.SETTING_MAX_SHARDS_PER_CLUSTER_KEY;
2928

@@ -32,11 +31,7 @@ public class RemoteStoreClusterStateRestoreIT extends BaseRemoteStoreRestoreIT {
3231

3332
@Override
3433
protected Settings nodeSettings(int nodeOrdinal) {
35-
return Settings.builder()
36-
.put(super.nodeSettings(nodeOrdinal))
37-
.put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
38-
.put(REMOTE_CLUSTER_STATE_REPOSITORY_SETTING.getKey(), REPOSITORY_NAME)
39-
.build();
34+
return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true).build();
4035
}
4136

4237
private void addNewNodes(int dataNodeCount, int clusterManagerNodeCount) {

‎server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@
5252
import java.util.Set;
5353

5454
import static org.opensearch.cluster.coordination.Coordinator.ZEN1_BWC_TERM;
55-
import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
55+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled;
5656

5757
/**
5858
* The core class of the cluster state coordination algorithm, directly implementing the
@@ -101,7 +101,7 @@ public CoordinationState(
101101
.getLastAcceptedState()
102102
.getLastAcceptedConfiguration();
103103
this.publishVotes = new VoteCollection();
104-
this.isRemoteStateEnabled = REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings);
104+
this.isRemoteStateEnabled = isRemoteStoreClusterStateEnabled(settings);
105105
}
106106

107107
public long getCurrentTerm() {

‎server/src/main/java/org/opensearch/common/settings/ClusterSettings.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -673,7 +673,6 @@ public void apply(Settings value, Settings current, Settings previous) {
673673

674674
// Remote cluster state settings
675675
RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING,
676-
RemoteClusterStateService.REMOTE_CLUSTER_STATE_REPOSITORY_SETTING,
677676
RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING
678677
)
679678
)

‎server/src/main/java/org/opensearch/gateway/GatewayMetaState.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@
8585
import java.util.function.UnaryOperator;
8686

8787
import static org.opensearch.common.util.concurrent.OpenSearchExecutors.daemonThreadFactory;
88-
import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
88+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled;
8989

9090
/**
9191
* Loads (and maybe upgrades) cluster metadata at startup, and persistently stores cluster metadata for future restarts.
@@ -166,7 +166,7 @@ public void start(
166166

167167
if (DiscoveryNode.isClusterManagerNode(settings)) {
168168
persistedState = new LucenePersistedState(persistedClusterStateService, currentTerm, clusterState);
169-
if (REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) == true) {
169+
if (isRemoteStoreClusterStateEnabled(settings)) {
170170
remotePersistedState = new RemotePersistedState(remoteClusterStateService);
171171
}
172172
} else {

‎server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
import org.opensearch.core.index.Index;
2929
import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
3030
import org.opensearch.index.remote.RemoteStoreUtils;
31+
import org.opensearch.node.Node;
32+
import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
3133
import org.opensearch.repositories.RepositoriesService;
3234
import org.opensearch.repositories.Repository;
3335
import org.opensearch.repositories.blobstore.BlobStoreRepository;
@@ -52,6 +54,7 @@
5254
import java.util.stream.Collectors;
5355

5456
import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD;
57+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled;
5558

5659
/**
5760
* A Service which provides APIs to upload and download cluster metadata from remote store.
@@ -80,23 +83,13 @@ public class RemoteClusterStateService implements Closeable {
8083
/**
8184
* Used to specify if cluster state metadata should be published to remote store
8285
*/
83-
// TODO The remote state enabled and repository settings should be read from node attributes.
84-
// Dependent on https://github.com/opensearch-project/OpenSearch/pull/9105/
8586
public static final Setting<Boolean> REMOTE_CLUSTER_STATE_ENABLED_SETTING = Setting.boolSetting(
8687
"cluster.remote_store.state.enabled",
8788
false,
8889
Property.NodeScope,
8990
Property.Final
9091
);
91-
/**
92-
* Used to specify default repo to use for cluster state metadata upload
93-
*/
94-
public static final Setting<String> REMOTE_CLUSTER_STATE_REPOSITORY_SETTING = Setting.simpleString(
95-
"cluster.remote_store.state.repository",
96-
"",
97-
Property.NodeScope,
98-
Property.Final
99-
);
92+
10093
private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class);
10194

10295
public static final String DELIMITER = "__";
@@ -115,7 +108,7 @@ public RemoteClusterStateService(
115108
ClusterSettings clusterSettings,
116109
LongSupplier relativeTimeNanosSupplier
117110
) {
118-
assert REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) == true : "Remote cluster state is not enabled";
111+
assert isRemoteStoreClusterStateEnabled(settings) : "Remote cluster state is not enabled";
119112
this.nodeId = nodeId;
120113
this.repositoriesService = repositoriesService;
121114
this.settings = settings;
@@ -384,8 +377,10 @@ void ensureRepositorySet() {
384377
if (blobStoreRepository != null) {
385378
return;
386379
}
387-
assert REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) == true : "Remote cluster state is not enabled";
388-
final String remoteStoreRepo = REMOTE_CLUSTER_STATE_REPOSITORY_SETTING.get(settings);
380+
assert isRemoteStoreClusterStateEnabled(settings) : "Remote cluster state is not enabled";
381+
final String remoteStoreRepo = settings.get(
382+
Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY
383+
);
389384
assert remoteStoreRepo != null : "Remote Cluster State repository is not configured";
390385
final Repository repository = repositoriesService.get().repository(remoteStoreRepo);
391386
assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository";

‎server/src/main/java/org/opensearch/node/Node.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,7 @@
273273
import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath;
274274
import static org.opensearch.index.ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED_ATTRIBUTE_KEY;
275275
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreAttributePresent;
276+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled;
276277

277278
/**
278279
* A node represent a node within a cluster ({@code cluster.name}). The {@link #client()} can be used
@@ -683,7 +684,7 @@ protected Node(
683684
threadPool::relativeTimeInMillis
684685
);
685686
final RemoteClusterStateService remoteClusterStateService;
686-
if (RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) == true) {
687+
if (isRemoteStoreClusterStateEnabled(settings)) {
687688
remoteClusterStateService = new RemoteClusterStateService(
688689
nodeEnvironment.nodeId(),
689690
repositoriesServiceReference::get,

‎server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.opensearch.cluster.metadata.RepositoryMetadata;
1414
import org.opensearch.cluster.node.DiscoveryNode;
1515
import org.opensearch.common.settings.Settings;
16+
import org.opensearch.gateway.remote.RemoteClusterStateService;
1617
import org.opensearch.node.Node;
1718
import org.opensearch.repositories.blobstore.BlobStoreRepository;
1819

@@ -36,6 +37,7 @@ public class RemoteStoreNodeAttribute {
3637
public static final String REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX = "remote_store";
3738
public static final String REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.segment.repository";
3839
public static final String REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.translog.repository";
40+
public static final String REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.state.repository";
3941
public static final String REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s.type";
4042
public static final String REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s."
4143
+ CryptoMetadata.CRYPTO_METADATA_KEY;
@@ -134,6 +136,7 @@ private RepositoriesMetadata buildRepositoriesMetadata(DiscoveryNode node) {
134136

135137
repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY));
136138
repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY));
139+
repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY));
137140

138141
for (String repositoryName : repositoryNames) {
139142
repositoryMetadataList.add(buildRepositoryMetadata(node, repositoryName));
@@ -146,6 +149,10 @@ public static boolean isRemoteStoreAttributePresent(Settings settings) {
146149
return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX).isEmpty() == false;
147150
}
148151

152+
public static boolean isRemoteStoreClusterStateEnabled(Settings settings) {
153+
return RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) && isRemoteStoreAttributePresent(settings);
154+
}
155+
149156
public RepositoriesMetadata getRepositoriesMetadata() {
150157
return this.repositoriesMetadata;
151158
}

‎server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,14 @@
4848
import org.opensearch.gateway.GatewayMetaState.RemotePersistedState;
4949
import org.opensearch.gateway.remote.ClusterMetadataManifest;
5050
import org.opensearch.gateway.remote.RemoteClusterStateService;
51+
import org.opensearch.repositories.fs.FsRepository;
5152
import org.opensearch.test.EqualsHashCodeTestUtils;
5253
import org.opensearch.test.OpenSearchTestCase;
5354
import org.junit.Before;
5455

5556
import java.io.IOException;
5657
import java.util.Collections;
58+
import java.util.Locale;
5759
import java.util.Optional;
5860
import java.util.stream.Collectors;
5961
import java.util.stream.IntStream;
@@ -62,6 +64,9 @@
6264

6365
import static java.util.Collections.emptyMap;
6466
import static java.util.Collections.emptySet;
67+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
68+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
69+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
6570
import static org.hamcrest.Matchers.containsString;
6671
import static org.hamcrest.Matchers.equalTo;
6772
import static org.hamcrest.Matchers.nullValue;
@@ -937,9 +942,26 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep
937942
final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry();
938943
persistedStateRegistry.addPersistedState(PersistedStateType.LOCAL, ps1);
939944
persistedStateRegistry.addPersistedState(PersistedStateType.REMOTE, new RemotePersistedState(remoteClusterStateService));
940-
final Settings settings = Settings.builder()
945+
946+
String randomRepoName = "randomRepoName";
947+
String stateRepoTypeAttributeKey = String.format(
948+
Locale.getDefault(),
949+
"node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
950+
randomRepoName
951+
);
952+
String stateRepoSettingsAttributeKeyPrefix = String.format(
953+
Locale.getDefault(),
954+
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
955+
randomRepoName
956+
);
957+
958+
Settings settings = Settings.builder()
959+
.put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, randomRepoName)
960+
.put(stateRepoTypeAttributeKey, FsRepository.TYPE)
961+
.put(stateRepoSettingsAttributeKeyPrefix + "location", "randomRepoPath")
941962
.put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
942963
.build();
964+
943965
final CoordinationState coordinationState = createCoordinationState(persistedStateRegistry, node1, settings);
944966
coordinationState.handlePrePublish(clusterState);
945967
Mockito.verifyNoInteractions(remoteClusterStateService);

0 commit comments

Comments
 (0)