Skip to content

Commit c3a7c4e

Browse files
joegallojoegallo
authored
Add setting to enforce a default TIER_PREFERENCE (#79210)
1 parent 9c5f148 commit c3a7c4e

10 files changed

Lines changed: 227 additions & 26 deletions

File tree

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java‎

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.elasticsearch.cluster.routing.ShardRouting;
3535
import org.elasticsearch.cluster.routing.ShardRoutingState;
3636
import org.elasticsearch.cluster.routing.allocation.AllocationService;
37+
import org.elasticsearch.cluster.routing.allocation.DataTier;
3738
import org.elasticsearch.cluster.service.ClusterService;
3839
import org.elasticsearch.common.Priority;
3940
import org.elasticsearch.common.Strings;
@@ -45,7 +46,6 @@
4546
import org.elasticsearch.common.settings.IndexScopedSettings;
4647
import org.elasticsearch.common.settings.Setting;
4748
import org.elasticsearch.common.settings.Settings;
48-
import org.elasticsearch.xcontent.NamedXContentRegistry;
4949
import org.elasticsearch.common.xcontent.XContentHelper;
5050
import org.elasticsearch.core.Nullable;
5151
import org.elasticsearch.core.PathUtils;
@@ -66,6 +66,7 @@
6666
import org.elasticsearch.indices.ShardLimitValidator;
6767
import org.elasticsearch.indices.SystemIndices;
6868
import org.elasticsearch.threadpool.ThreadPool;
69+
import org.elasticsearch.xcontent.NamedXContentRegistry;
6970

7071
import java.io.IOException;
7172
import java.io.UnsupportedEncodingException;
@@ -128,6 +129,8 @@ public class MetadataCreateIndexService {
128129
private final boolean forbidPrivateIndexSettings;
129130
private final Set<IndexSettingProvider> indexSettingProviders = new HashSet<>();
130131

132+
private volatile boolean enforceDefaultTierPreference;
133+
131134
public MetadataCreateIndexService(
132135
final Settings settings,
133136
final ClusterService clusterService,
@@ -153,6 +156,14 @@ public MetadataCreateIndexService(
153156
this.systemIndices = systemIndices;
154157
this.forbidPrivateIndexSettings = forbidPrivateIndexSettings;
155158
this.shardLimitValidator = shardLimitValidator;
159+
160+
enforceDefaultTierPreference = DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING.get(settings);
161+
clusterService.getClusterSettings().addSettingsUpdateConsumer(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING,
162+
this::setEnforceDefaultTierPreference);
163+
}
164+
165+
public void setEnforceDefaultTierPreference(boolean enforceDefaultTierPreference) {
166+
this.enforceDefaultTierPreference = enforceDefaultTierPreference;
156167
}
157168

158169
/**
@@ -481,7 +492,8 @@ private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState c
481492

482493
final Settings aggregatedIndexSettings =
483494
aggregateIndexSettings(currentState, request, resolveSettings(templates),
484-
null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders);
495+
null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders,
496+
this.enforceDefaultTierPreference);
485497
int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
486498
IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
487499

@@ -516,7 +528,8 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu
516528
final Settings aggregatedIndexSettings =
517529
aggregateIndexSettings(currentState, request,
518530
resolveSettings(currentState.metadata(), templateName),
519-
null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders);
531+
null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders,
532+
this.enforceDefaultTierPreference);
520533
int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
521534
IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
522535

@@ -570,7 +583,8 @@ private ClusterState applyCreateIndexRequestForSystemDataStream(final ClusterSta
570583
settings,
571584
indexScopedSettings,
572585
shardLimitValidator,
573-
indexSettingProviders
586+
indexSettingProviders,
587+
this.enforceDefaultTierPreference
574588
);
575589
final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
576590
final IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
@@ -634,7 +648,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterSt
634648
}
635649

636650
final Settings aggregatedIndexSettings = aggregateIndexSettings(currentState, request, Settings.EMPTY,
637-
sourceMetadata, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders);
651+
sourceMetadata, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders, this.enforceDefaultTierPreference);
638652
final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata);
639653
IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
640654

@@ -696,7 +710,9 @@ static Map<String, Object> parseV1Mappings(String mappingsJson, List<CompressedX
696710
static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClusterStateUpdateRequest request,
697711
Settings combinedTemplateSettings, @Nullable IndexMetadata sourceMetadata, Settings settings,
698712
IndexScopedSettings indexScopedSettings, ShardLimitValidator shardLimitValidator,
699-
Set<IndexSettingProvider> indexSettingProviders) {
713+
Set<IndexSettingProvider> indexSettingProviders, boolean enforceDefaultTierPreference) {
714+
final boolean isDataStreamIndex = request.dataStreamName() != null;
715+
700716
// Create builders for the template and request settings. We transform these into builders
701717
// because we may want settings to be "removed" from these prior to being set on the new
702718
// index (see more comments below)
@@ -711,7 +727,6 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu
711727
.put(request.settings())
712728
.build();
713729

714-
final boolean isDataStreamIndex = request.dataStreamName() != null;
715730
// Loop through all the explicit index setting providers, adding them to the
716731
// additionalIndexSettings map
717732
for (IndexSettingProvider provider : indexSettingProviders) {
@@ -754,6 +769,20 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu
754769
// now, put the request settings, so they override templates
755770
indexSettingsBuilder.put(requestSettings.build());
756771

772+
if (sourceMetadata == null) { // not for shrink/split/clone
773+
if (enforceDefaultTierPreference) {
774+
// regardless of any previous logic, we're going to force there
775+
// to be an appropriate non-empty value for the tier preference
776+
String currentTierPreference = indexSettingsBuilder.get(DataTier.TIER_PREFERENCE);
777+
if (DataTier.parseTierList(currentTierPreference).isEmpty()) {
778+
String newTierPreference = isDataStreamIndex ? DataTier.DATA_HOT : DataTier.DATA_CONTENT;
779+
logger.debug("enforcing default [{}] setting for [{}] creation, replacing [{}] with [{}]",
780+
DataTier.TIER_PREFERENCE, request.index(), currentTierPreference, newTierPreference);
781+
indexSettingsBuilder.put(DataTier.TIER_PREFERENCE, newTierPreference);
782+
}
783+
}
784+
}
785+
757786
if (indexSettingsBuilder.get(IndexMetadata.SETTING_VERSION_CREATED) == null) {
758787
final DiscoveryNodes nodes = currentState.nodes();
759788
final Version createdVersion = Version.min(Version.CURRENT, nodes.getSmallestNonClientNodeVersion());

‎server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
1616
import org.elasticsearch.common.Strings;
1717
import org.elasticsearch.common.settings.Setting;
18+
import org.elasticsearch.common.settings.Setting.Property;
1819
import org.elasticsearch.common.settings.Settings;
1920
import org.elasticsearch.index.IndexModule;
2021
import org.elasticsearch.index.shard.IndexSettingProvider;
@@ -44,6 +45,13 @@ public class DataTier {
4445

4546
public static final Set<String> ALL_DATA_TIERS = Set.of(DATA_CONTENT, DATA_HOT, DATA_WARM, DATA_COLD, DATA_FROZEN);
4647

48+
// this setting is for migrating from 7.x (where a tier preference was not required, and did not necessarily
49+
// have a default value), to 8.x (where a tier preference will be required, and a default value will be injected).
50+
// it will be removed as a breaking change in some future version, likely 9.0.
51+
public static final String ENFORCE_DEFAULT_TIER_PREFERENCE = "cluster.routing.allocation.enforce_default_tier_preference";
52+
public static final Setting<Boolean> ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING =
53+
Setting.boolSetting(ENFORCE_DEFAULT_TIER_PREFERENCE, false, Property.Dynamic, Property.NodeScope);
54+
4755
public static final String TIER_PREFERENCE = "index.routing.allocation.include._tier_preference";
4856

4957
private static final Settings DATA_CONTENT_TIER_PREFERENCE_SETTINGS = Settings.builder().put(TIER_PREFERENCE, DATA_CONTENT).build();
@@ -57,8 +65,8 @@ public class DataTier {
5765
DataTierSettingValidator::getDefaultTierPreference,
5866
Function.identity(),
5967
new DataTierSettingValidator(),
60-
Setting.Property.Dynamic,
61-
Setting.Property.IndexScope
68+
Property.Dynamic,
69+
Property.IndexScope
6270
);
6371

6472
static {

‎server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.elasticsearch.cluster.metadata.IndexGraveyard;
3434
import org.elasticsearch.cluster.metadata.Metadata;
3535
import org.elasticsearch.cluster.routing.OperationRouting;
36+
import org.elasticsearch.cluster.routing.allocation.DataTier;
3637
import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
3738
import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
3839
import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
@@ -485,7 +486,9 @@ public void apply(Settings value, Settings current, Settings previous) {
485486
FsHealthService.REFRESH_INTERVAL_SETTING,
486487
FsHealthService.SLOW_PATH_LOGGING_THRESHOLD_SETTING,
487488
IndexingPressure.MAX_INDEXING_BYTES,
488-
ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE_FROZEN);
489+
ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE_FROZEN,
490+
DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING
491+
);
489492

490493
static List<SettingUpgrader<?>> BUILT_IN_SETTING_UPGRADERS = Collections.emptyList();
491494

0 commit comments

Comments
 (0)