diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index fabf6645fb6eb..37fad1a124e27 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add filter function for AbstractQueryBuilder, BoolQueryBuilder, ConstantScoreQueryBuilder([#17409](https://github.com/opensearch-project/OpenSearch/pull/17409)) - [Star Tree] [Search] Resolving keyword & numeric bucket aggregation with metric aggregation using star-tree ([#17165](https://github.com/opensearch-project/OpenSearch/pull/17165)) - Added error handling support for the pull-based ingestion ([#17427](https://github.com/opensearch-project/OpenSearch/pull/17427)) +- Added Warm index setting and Updated nomenclature to differentiate between hot and warm tiering implementation ([#17490](https://github.com/opensearch-project/OpenSearch/pull/17490)) ### Dependencies diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/WarmIndexSegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/WarmIndexSegmentReplicationIT.java index d7f1c2209f798..a50ec7a0d0fa1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/WarmIndexSegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/WarmIndexSegmentReplicationIT.java @@ -150,7 +150,7 @@ protected Settings nodeSettings(int nodeOrdinal) { @Override protected Settings featureFlagSettings() { Settings.Builder featureSettings = Settings.builder(); - featureSettings.put(FeatureFlags.TIERED_REMOTE_INDEX, true); + featureSettings.put(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG, true); return featureSettings.build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/WritableWarmIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/WritableWarmIT.java index 88c9ae436e85f..9050b7eff008d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/WritableWarmIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/WritableWarmIT.java @@ -63,7 +63,7 @@ protected boolean addMockIndexStorePlugin() { @Override protected Settings featureFlagSettings() { Settings.Builder featureSettings = Settings.builder(); - featureSettings.put(FeatureFlags.TIERED_REMOTE_INDEX, true); + featureSettings.put(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG, true); return featureSettings.build(); } @@ -77,7 +77,11 @@ protected Settings nodeSettings(int nodeOrdinal) { } public void testWritableWarmFeatureFlagDisabled() { - Settings clusterSettings = Settings.builder().put(super.nodeSettings(0)).put(FeatureFlags.TIERED_REMOTE_INDEX, false).build(); + Settings clusterSettings = Settings.builder() + .put(super.nodeSettings(0)) + .put(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG, false) + .build(); + InternalTestCluster internalTestCluster = internalCluster(); internalTestCluster.startClusterManagerOnlyNode(clusterSettings); internalTestCluster.startDataAndSearchNodes(1); @@ -85,7 +89,7 @@ public void testWritableWarmFeatureFlagDisabled() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexModule.INDEX_STORE_LOCALITY_SETTING.getKey(), IndexModule.DataLocalityType.PARTIAL.name()) + .put(IndexModule.IS_WARM_INDEX_SETTING.getKey(), false) .build(); try { @@ -94,7 +98,7 @@ public void testWritableWarmFeatureFlagDisabled() { } catch (SettingsException ex) { assertEquals( "unknown setting [" - + IndexModule.INDEX_STORE_LOCALITY_SETTING.getKey() + + IndexModule.IS_WARM_INDEX_SETTING.getKey() + "] please check that any required plugins are installed, or check the " + "breaking changes documentation for removed settings", ex.getMessage() @@ -109,7 +113,7 @@ public void testWritableWarmBasic() throws Exception { Settings settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexModule.INDEX_STORE_LOCALITY_SETTING.getKey(), IndexModule.DataLocalityType.PARTIAL.name()) + .put(IndexModule.IS_WARM_INDEX_SETTING.getKey(), true) .build(); assertAcked(client().admin().indices().prepareCreate(INDEX_NAME).setSettings(settings).get()); @@ -119,7 +123,7 @@ public void testWritableWarmBasic() throws Exception { .getIndex(new GetIndexRequest().indices(INDEX_NAME).includeDefaults(true)) .get(); Settings indexSettings = getIndexResponse.settings().get(INDEX_NAME); - assertEquals(IndexModule.DataLocalityType.PARTIAL.name(), indexSettings.get(IndexModule.INDEX_STORE_LOCALITY_SETTING.getKey())); + assertTrue(indexSettings.getAsBoolean(IndexModule.IS_WARM_INDEX_SETTING.getKey(), false)); // Ingesting some docs indexBulk(INDEX_NAME, NUM_DOCS_IN_BULK); diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index baf15d25dbfd4..2a8a675c6fcea 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -654,7 +654,7 @@ public void reg actions.register(CreateSnapshotAction.INSTANCE, TransportCreateSnapshotAction.class); actions.register(CloneSnapshotAction.INSTANCE, TransportCloneSnapshotAction.class); actions.register(RestoreSnapshotAction.INSTANCE, TransportRestoreSnapshotAction.class); - if (FeatureFlags.isEnabled(FeatureFlags.TIERED_REMOTE_INDEX)) { + if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG)) { actions.register(HotToWarmTieringAction.INSTANCE, TransportHotToWarmTieringAction.class); } actions.register(SnapshotsStatusAction.INSTANCE, TransportSnapshotsStatusAction.class); @@ -996,7 +996,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestNodeAttrsAction()); registerHandler.accept(new RestRepositoriesAction()); registerHandler.accept(new RestSnapshotAction()); - if (FeatureFlags.isEnabled(FeatureFlags.TIERED_REMOTE_INDEX)) { + if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG)) { registerHandler.accept(new RestWarmTieringAction()); } registerHandler.accept(new RestTemplatesAction()); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/tiering/TieringUtils.java b/server/src/main/java/org/opensearch/action/admin/indices/tiering/TieringUtils.java index 46912de17f213..5c77f1091c1c6 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/tiering/TieringUtils.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/tiering/TieringUtils.java @@ -22,26 +22,25 @@ public class TieringUtils { /** * Checks if the specified shard is a partial shard by - * checking the INDEX_STORE_LOCALITY_SETTING for its index. - * see {@link #isPartialIndex(IndexMetadata)} + * checking the WARM_INDEX_ENABLED_SETTING for its index. + * see {@link #isWarmIndex(IndexMetadata)} (IndexMetadata)} * @param shard ShardRouting object representing the shard * @param allocation RoutingAllocation object representing the allocation * @return true if the shard is a partial shard, false otherwise */ public static boolean isPartialShard(ShardRouting shard, RoutingAllocation allocation) { IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shard.index()); - return isPartialIndex(indexMetadata); + return isWarmIndex(indexMetadata); } /** - * Checks if the specified index is a partial index by - * checking the INDEX_STORE_LOCALITY_SETTING for the index. + * Checks if the specified index is a warm index by + * checking the WARM_INDEX_ENABLED_SETTING for the index. * * @param indexMetadata the metadata of the index - * @return true if the index is a partial index, false otherwise + * @return true if the index is a warm index, false otherwise */ - public static boolean isPartialIndex(final IndexMetadata indexMetadata) { - return IndexModule.DataLocalityType.PARTIAL.name() - .equals(indexMetadata.getSettings().get(IndexModule.INDEX_STORE_LOCALITY_SETTING.getKey())); + public static boolean isWarmIndex(final IndexMetadata indexMetadata) { + return indexMetadata.getSettings().getAsBoolean(IndexModule.IS_WARM_INDEX_SETTING.getKey(), false); } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java index eac6f41acde4c..9f23ba3a01539 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java +++ b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java @@ -256,7 +256,7 @@ public GroupShardsIterator searchShards( preference = Preference.PRIMARY.type(); } - if (FeatureFlags.isEnabled(FeatureFlags.TIERED_REMOTE_INDEX) + if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG) && IndexModule.DataLocalityType.PARTIAL.name() .equals(indexMetadataForShard.getSettings().get(IndexModule.INDEX_STORE_LOCALITY_SETTING.getKey())) && (preference == null || preference.isEmpty())) { diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingPool.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingPool.java index 647e993339476..53788c6559ee5 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingPool.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingPool.java @@ -13,7 +13,7 @@ import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.common.util.FeatureFlags; -import static org.opensearch.action.admin.indices.tiering.TieringUtils.isPartialIndex; +import static org.opensearch.action.admin.indices.tiering.TieringUtils.isWarmIndex; /** * {@link RoutingPool} defines the different node types based on the assigned capabilities. The methods @@ -62,6 +62,9 @@ public static RoutingPool getShardPool(ShardRouting shard, RoutingAllocation all */ public static RoutingPool getIndexPool(IndexMetadata indexMetadata) { return indexMetadata.isRemoteSnapshot() - || (FeatureFlags.isEnabled(FeatureFlags.TIERED_REMOTE_INDEX) && isPartialIndex(indexMetadata)) ? REMOTE_CAPABLE : LOCAL_ONLY; + || (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG) && isWarmIndex(indexMetadata)) + ? REMOTE_CAPABLE + : LOCAL_ONLY; + } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java index 7f6a7790d1db0..f6e3e94e9b9a6 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java @@ -561,7 +561,7 @@ private void checkAndAddInEligibleTargetNode(RoutingNode targetNode) { */ private boolean canShardBeSkipped(ShardRouting shardRouting) { return (RoutingPool.REMOTE_CAPABLE.equals(RoutingPool.getShardPool(shardRouting, allocation)) - && !(FeatureFlags.isEnabled(FeatureFlags.TIERED_REMOTE_INDEX) && isPartialShard(shardRouting, allocation))); + && !(FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG) && isPartialShard(shardRouting, allocation))); } /** @@ -771,7 +771,7 @@ private Map buildModelFromAssigned() assert rn.nodeId().equals(shard.currentNodeId()); /* we skip relocating shards here since we expect an initializing shard with the same id coming in */ if ((RoutingPool.LOCAL_ONLY.equals(RoutingPool.getShardPool(shard, allocation)) - || (FeatureFlags.isEnabled(FeatureFlags.TIERED_REMOTE_INDEX) && isPartialShard(shard, allocation))) + || (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG) && isPartialShard(shard, allocation))) && shard.state() != RELOCATING) { node.addShard(shard); ++totalShardCount; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/RemoteShardsBalancer.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/RemoteShardsBalancer.java index e0543b396728e..5bd663826ea5f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/RemoteShardsBalancer.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/RemoteShardsBalancer.java @@ -33,7 +33,7 @@ import java.util.Queue; import java.util.Set; -import static org.opensearch.action.admin.indices.tiering.TieringUtils.isPartialIndex; +import static org.opensearch.action.admin.indices.tiering.TieringUtils.isWarmIndex; /** * A {@link RemoteShardsBalancer} used by the {@link BalancedShardsAllocator} to perform allocation operations @@ -348,7 +348,7 @@ private void unassignIgnoredRemoteShards(RoutingAllocation routingAllocation) { // to re-fetch any shard blocks from the repository. if (shard.primary()) { if (RecoverySource.Type.SNAPSHOT.equals(shard.recoverySource().getType()) == false - && isPartialIndex(allocation.metadata().getIndexSafe(shard.index())) == false) { + && isWarmIndex(allocation.metadata().getIndexSafe(shard.index())) == false) { unassignedShard = shard.updateUnassigned(shard.unassignedInfo(), RecoverySource.EmptyStoreRecoverySource.INSTANCE); } } diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index b1d13b1ae8693..ba35a8bd1133a 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -33,7 +33,7 @@ protected FeatureFlagSettings( FeatureFlags.EXTENSIONS_SETTING, FeatureFlags.TELEMETRY_SETTING, FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING, - FeatureFlags.TIERED_REMOTE_INDEX_SETTING, + FeatureFlags.WRITABLE_WARM_INDEX_SETTING, FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, FeatureFlags.STAR_TREE_INDEX_SETTING, FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 12bee5cd14f57..14cd7479866d2 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -291,8 +291,9 @@ public final class IndexScopedSettings extends AbstractScopedSettings { * setting should be moved to {@link #BUILT_IN_INDEX_SETTINGS}. */ public static final Map> FEATURE_FLAGGED_INDEX_SETTINGS = Map.of( - FeatureFlags.TIERED_REMOTE_INDEX, - List.of(IndexModule.INDEX_STORE_LOCALITY_SETTING, IndexModule.INDEX_TIERING_STATE), + FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG, + // TODO: Create a separate feature flag for hot tiering index state. + List.of(IndexModule.INDEX_STORE_LOCALITY_SETTING, IndexModule.INDEX_TIERING_STATE, IndexModule.IS_WARM_INDEX_SETTING), FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL, List.of(IndexMetadata.INDEX_NUMBER_OF_SEARCH_REPLICAS_SETTING) ); diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index f0b26d562c52b..4ff81cf0c1c96 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -51,10 +51,10 @@ public class FeatureFlags { public static final String DATETIME_FORMATTER_CACHING = "opensearch.experimental.optimization.datetime_formatter_caching.enabled"; /** - * Gates the functionality of remote index having the capability to move across different tiers + * Gates the functionality of warm index having the capability to store data remotely. * Once the feature is ready for release, this feature flag can be removed. */ - public static final String TIERED_REMOTE_INDEX = "opensearch.experimental.feature.tiered_remote_index.enabled"; + public static final String WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG = "opensearch.experimental.feature.writable_warm_index.enabled"; /** * Gates the functionality of background task execution. @@ -79,7 +79,11 @@ public class FeatureFlags { Property.NodeScope ); - public static final Setting TIERED_REMOTE_INDEX_SETTING = Setting.boolSetting(TIERED_REMOTE_INDEX, false, Property.NodeScope); + public static final Setting WRITABLE_WARM_INDEX_SETTING = Setting.boolSetting( + WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG, + false, + Property.NodeScope + ); public static final Setting READER_WRITER_SPLIT_EXPERIMENTAL_SETTING = Setting.boolSetting( READER_WRITER_SPLIT_EXPERIMENTAL, @@ -128,7 +132,7 @@ public class FeatureFlags { EXTENSIONS_SETTING, TELEMETRY_SETTING, DATETIME_FORMATTER_CACHING_SETTING, - TIERED_REMOTE_INDEX_SETTING, + WRITABLE_WARM_INDEX_SETTING, STAR_TREE_INDEX_SETTING, APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, READER_WRITER_SPLIT_EXPERIMENTAL_SETTING, diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index 7016ddb8e59b8..961b77ac20c5e 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -145,7 +145,7 @@ public final class IndexModule { ); /** - * Index setting which used to determine how the data is cached locally fully or partially + * Index setting which used to determine how the data is cached locally fully or partially. */ public static final Setting INDEX_STORE_LOCALITY_SETTING = new Setting<>( "index.store.data_locality", @@ -155,6 +155,8 @@ public final class IndexModule { Property.NodeScope ); + public static final Setting IS_WARM_INDEX_SETTING = Setting.boolSetting("index.warm", false, Property.IndexScope); + public static final Setting INDEX_RECOVERY_TYPE_SETTING = new Setting<>( "index.recovery.type", "", diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index e265ce3590121..0696058e86f08 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -652,9 +652,9 @@ protected void closeInternal() { } Directory directory = null; - if (FeatureFlags.isEnabled(FeatureFlags.TIERED_REMOTE_INDEX_SETTING) && + if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_SETTING) && // TODO : Need to remove this check after support for hot indices is added in Composite Directory - this.indexSettings.isStoreLocalityPartial()) { + this.indexSettings.isWarmIndex()) { Directory localDirectory = directoryFactory.newDirectory(this.indexSettings, path); directory = new CompositeDirectory(localDirectory, remoteDirectory, fileCache); } else { diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 554e99764c1a1..38604ffd8bf8f 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -790,7 +790,8 @@ public static IndexMergePolicy fromString(String text) { private final int numberOfShards; private final ReplicationType replicationType; private volatile boolean isRemoteStoreEnabled; - private final boolean isStoreLocalityPartial; + // For warm index we would partially store files in local. + private final boolean isWarmIndex; private volatile TimeValue remoteTranslogUploadBufferInterval; private volatile String remoteStoreTranslogRepository; private volatile String remoteStoreRepository; @@ -994,10 +995,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); replicationType = IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.get(settings); isRemoteStoreEnabled = settings.getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false); - isStoreLocalityPartial = settings.get( - IndexModule.INDEX_STORE_LOCALITY_SETTING.getKey(), - IndexModule.DataLocalityType.FULL.toString() - ).equalsIgnoreCase(IndexModule.DataLocalityType.PARTIAL.toString()); + + isWarmIndex = settings.getAsBoolean(IndexModule.IS_WARM_INDEX_SETTING.getKey(), false); + remoteStoreTranslogRepository = settings.get(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY); remoteTranslogUploadBufferInterval = INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings); remoteStoreRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY); @@ -1372,10 +1372,10 @@ public String getRemoteStoreTranslogRepository() { } /** - * Returns true if the store locality is partial + * Returns true if the index is writable warm index and has partial store locality. */ - public boolean isStoreLocalityPartial() { - return isStoreLocalityPartial; + public boolean isWarmIndex() { + return isWarmIndex; } /** diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java index 7f3010ff0937a..80e24fa0c5a7e 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java @@ -370,7 +370,7 @@ public void flush(boolean force, boolean waitIfOngoing) throws EngineException { ensureOpen(); // Skip flushing for indices with partial locality (warm indices) // For these indices, we don't need to commit as we will sync from the remote store on re-open - if (engineConfig.getIndexSettings().isStoreLocalityPartial()) { + if (engineConfig.getIndexSettings().isWarmIndex()) { return; } // readLock is held here to wait/block any concurrent close that acquires the writeLock. @@ -447,7 +447,7 @@ protected final void closeNoLock(String reason, CountDownLatch closedLatch) { latestSegmentInfos.changed(); } try { - if (engineConfig.getIndexSettings().isStoreLocalityPartial() == false) { + if (engineConfig.getIndexSettings().isWarmIndex() == false) { commitSegmentInfos(latestSegmentInfos); } } catch (IOException e) { diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 173268997895e..f861dbdab360b 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -5142,7 +5142,7 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, final Runn } else { storeDirectory = store.directory(); } - if (indexSettings.isStoreLocalityPartial() == false) { + if (indexSettings.isWarmIndex() == false) { copySegmentFiles(storeDirectory, remoteDirectory, null, uploadedSegments, overrideLocal, onFileSync); } @@ -5160,7 +5160,7 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, final Runn } } assert Arrays.stream(store.directory().listAll()).filter(f -> f.startsWith(IndexFileNames.SEGMENTS)).findAny().isEmpty() - || indexSettings.isStoreLocalityPartial() : "There should not be any segments file in the dir"; + || indexSettings.isWarmIndex() : "There should not be any segments file in the dir"; store.commitSegmentInfos(infosSnapshot, processedLocalCheckpoint, processedLocalCheckpoint); } syncSegmentSuccess = true; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 6922ade22b714..550be9fb12965 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -206,7 +206,7 @@ private List getFiles(CheckpointInfoResponse checkpointInfo) // Return an empty list for warm indices, In this case, replica shards don't require downloading files from remote storage // as replicas will sync all files from remote in case of failure. - if (indexShard.indexSettings().isStoreLocalityPartial()) { + if (indexShard.indexSettings().isWarmIndex()) { return Collections.emptyList(); } final Store.RecoveryDiff diff = Store.segmentReplicationDiff(checkpointInfo.getMetadataMap(), indexShard.getSegmentMetadataMap()); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/tiering/TransportHotToWarmTieringActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/tiering/TransportHotToWarmTieringActionTests.java index 10273366af804..ec0c1a8ebe3b9 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/tiering/TransportHotToWarmTieringActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/tiering/TransportHotToWarmTieringActionTests.java @@ -44,7 +44,7 @@ public class TransportHotToWarmTieringActionTests extends OpenSearchIntegTestCas @Override protected Settings featureFlagSettings() { Settings.Builder featureSettings = Settings.builder(); - featureSettings.put(FeatureFlags.TIERED_REMOTE_INDEX, true); + featureSettings.put(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG, true); return featureSettings.build(); } diff --git a/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java index 4263e1aa347dc..8cfdcce45c523 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java @@ -1060,7 +1060,7 @@ public void testSearchableSnapshotPrimaryDefault() throws Exception { @SuppressForbidden(reason = "feature flag overrides") public void testPartialIndexPrimaryDefault() throws Exception { - System.setProperty(FeatureFlags.TIERED_REMOTE_INDEX, "true"); + System.setProperty(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG, "true"); final int numIndices = 1; final int numShards = 2; final int numReplicas = 2; @@ -1116,7 +1116,7 @@ public void testPartialIndexPrimaryDefault() throws Exception { } finally { IOUtils.close(clusterService); terminate(threadPool); - System.setProperty(FeatureFlags.TIERED_REMOTE_INDEX, "false"); + System.setProperty(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG, "false"); } } diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/ShardsTieringAllocationTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/ShardsTieringAllocationTests.java index 8d45ebd2781b1..765d88f7af360 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/ShardsTieringAllocationTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/ShardsTieringAllocationTests.java @@ -29,7 +29,7 @@ public class ShardsTieringAllocationTests extends TieringAllocationBaseTestCase @Before public void setup() { - FeatureFlagSetter.set(FeatureFlags.TIERED_REMOTE_INDEX); + FeatureFlagSetter.set(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG); } public void testShardsInLocalPool() { @@ -107,7 +107,10 @@ public void testShardsWithTiering() { public void testShardPoolForPartialIndices() { String index = "test-index"; IndexMetadata indexMetadata = IndexMetadata.builder(index) - .settings(settings(Version.CURRENT).put(INDEX_STORE_LOCALITY_SETTING.getKey(), IndexModule.DataLocalityType.PARTIAL.name())) + .settings( + settings(Version.CURRENT).put(INDEX_STORE_LOCALITY_SETTING.getKey(), IndexModule.DataLocalityType.PARTIAL.name()) + .put(IndexModule.IS_WARM_INDEX_SETTING.getKey(), true) + ) .numberOfShards(PRIMARIES) .numberOfReplicas(REPLICAS) .build(); diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/TieringAllocationBaseTestCase.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/TieringAllocationBaseTestCase.java index aba6fe74e0634..e90c959dc0b18 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/TieringAllocationBaseTestCase.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/TieringAllocationBaseTestCase.java @@ -13,6 +13,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexModule; import static org.opensearch.index.IndexModule.INDEX_STORE_LOCALITY_SETTING; import static org.opensearch.index.IndexModule.INDEX_TIERING_STATE; @@ -37,6 +38,7 @@ public ClusterState updateIndexMetadataForTiering( .put(settings) .put(settings) .put(INDEX_TIERING_STATE.getKey(), tieringState) + .put(IndexModule.IS_WARM_INDEX_SETTING.getKey(), true) .put(INDEX_STORE_LOCALITY_SETTING.getKey(), dataLocality) ) );