diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index 2f702494960ac..434db543e54e0 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -10,8 +10,10 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; @@ -19,6 +21,7 @@ import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.indices.InvalidIndexTemplateException; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xcontent.XContentType; @@ -28,6 +31,7 @@ import java.util.List; import java.util.concurrent.CountDownLatch; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class TSDBIndexingIT extends ESSingleNodeTestCase { @@ -76,21 +80,48 @@ public void testTimeRanges() throws Exception { } } }"""; - Settings templateSettings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "metricset").build(); - var request = new PutComposableIndexTemplateAction.Request("id"); - request.indexTemplate( - new ComposableIndexTemplate( - List.of("k8s*"), - new Template(templateSettings, new CompressedXContent(mappingTemplate), null), - null, - null, - null, - null, - new ComposableIndexTemplate.DataStreamTemplate(false, false), - null - ) - ); - client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); + var templateSettings = Settings.builder().put("index.mode", "time_series"); + if (randomBoolean()) { + templateSettings.put("index.routing_path", "metricset"); + } + + if (randomBoolean()) { + var request = new PutComposableIndexTemplateAction.Request("id"); + request.indexTemplate( + new ComposableIndexTemplate( + List.of("k8s*"), + new Template(templateSettings.build(), new CompressedXContent(mappingTemplate), null), + null, + null, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(false, false), + null + ) + ); + client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); + } else { + var putComponentTemplateRequest = new PutComponentTemplateAction.Request("1"); + putComponentTemplateRequest.componentTemplate( + new ComponentTemplate(new Template(null, new CompressedXContent(mappingTemplate), null), null, null) + ); + client().execute(PutComponentTemplateAction.INSTANCE, putComponentTemplateRequest).actionGet(); + + var putTemplateRequest = new PutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + new ComposableIndexTemplate( + List.of("k8s*"), + new Template(templateSettings.build(), null, null), + List.of("1"), + null, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(false, false), + null + ) + ); + client().execute(PutComposableIndexTemplateAction.INSTANCE, putTemplateRequest).actionGet(); + } // index doc Instant time = Instant.now(); @@ -176,34 +207,60 @@ public void testInvalidTsdbTemplatesNoTimeSeriesDimensionAttribute() throws Exce } } }"""; - var request = new PutComposableIndexTemplateAction.Request("id"); - request.indexTemplate( - new ComposableIndexTemplate( - List.of("k8s*"), - new Template( - Settings.builder().put("index.mode", "time_series").put("index.routing_path", "metricset").build(), - new CompressedXContent(mappingTemplate), + { + var request = new PutComposableIndexTemplateAction.Request("id"); + request.indexTemplate( + new ComposableIndexTemplate( + List.of("k8s*"), + new Template( + Settings.builder().put("index.mode", "time_series").put("index.routing_path", "metricset").build(), + new CompressedXContent(mappingTemplate), + null + ), + null, + null, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(false, false), null - ), - null, - null, - null, - null, - new ComposableIndexTemplate.DataStreamTemplate(false, false), - null - ) - ); - Exception e = expectThrows( - IllegalArgumentException.class, - () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet() - ); - assertThat( - e.getCause().getCause().getMessage(), - equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] and " - + "without the [script] parameter. [metricset] was not [time_series_dimension: true]." - ) - ); + ) + ); + var e = expectThrows( + IllegalArgumentException.class, + () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet() + ); + assertThat( + e.getCause().getCause().getMessage(), + equalTo( + "All fields that match routing_path must be keywords with [time_series_dimension: true] and " + + "without the [script] parameter. [metricset] was not [time_series_dimension: true]." + ) + ); + } + { + var request = new PutComposableIndexTemplateAction.Request("id"); + request.indexTemplate( + new ComposableIndexTemplate( + List.of("k8s*"), + new Template( + Settings.builder().put("index.mode", "time_series").build(), + new CompressedXContent(mappingTemplate), + null + ), + null, + null, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(false, false), + null + ) + ); + var e = expectThrows( + InvalidIndexTemplateException.class, + () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet() + ); + assertThat(e.getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]")); + } } public void testInvalidTsdbTemplatesNoKeywordFieldType() throws Exception { @@ -260,54 +317,28 @@ public void testInvalidTsdbTemplatesMissingSettings() throws Exception { } } }"""; - { - var request = new PutComposableIndexTemplateAction.Request("id"); - request.indexTemplate( - new ComposableIndexTemplate( - List.of("k8s*"), - new Template( - Settings.builder().put("index.mode", "time_series").build(), - new CompressedXContent(mappingTemplate), - null - ), - null, - null, - null, - null, - new ComposableIndexTemplate.DataStreamTemplate(false, false), - null - ) - ); - var e = expectThrows( - IllegalArgumentException.class, - () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet() - ); - assertThat(e.getMessage(), equalTo("[index.mode=time_series] requires a non-empty [index.routing_path]")); - } - { - var request = new PutComposableIndexTemplateAction.Request("id"); - request.indexTemplate( - new ComposableIndexTemplate( - List.of("k8s*"), - new Template( - Settings.builder().put("index.routing_path", "metricset").build(), - new CompressedXContent(mappingTemplate), - null - ), - null, - null, - null, - null, - new ComposableIndexTemplate.DataStreamTemplate(false, false), + var request = new PutComposableIndexTemplateAction.Request("id"); + request.indexTemplate( + new ComposableIndexTemplate( + List.of("k8s*"), + new Template( + Settings.builder().put("index.routing_path", "metricset").build(), + new CompressedXContent(mappingTemplate), null - ) - ); - var e = expectThrows( - IllegalArgumentException.class, - () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet() - ); - assertThat(e.getMessage(), equalTo("[index.routing_path] requires [index.mode=time_series]")); - } + ), + null, + null, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(false, false), + null + ) + ); + var e = expectThrows( + IllegalArgumentException.class, + () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet() + ); + assertThat(e.getCause().getMessage(), equalTo("[index.routing_path] requires [index.mode=time_series]")); } static String formatInstant(Instant instant) { diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java index 6f579361622cc..6b58bc631a905 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java @@ -23,7 +23,7 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; import static org.hamcrest.Matchers.aMapWithSize; -import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -42,8 +42,7 @@ public class TsdbDataStreamRestIT extends ESRestTestCase { "index": { "number_of_replicas": 0, "number_of_shards": 2, - "mode": "time_series", - "routing_path": ["metricset", "time_series_dimension"] + "mode": "time_series" } }, "mappings":{ @@ -328,7 +327,7 @@ public void testSimulateTsdbDataStreamTemplate() throws Exception { assertThat(ObjectPath.evaluate(responseBody, "template.settings.index.time_series.end_time"), notNullValue()); assertThat( ObjectPath.evaluate(responseBody, "template.settings.index.routing_path"), - contains("metricset", "time_series_dimension") + containsInAnyOrder("metricset", "k8s.pod.uid") ); assertThat(ObjectPath.evaluate(responseBody, "overlapping"), empty()); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index db26e7c2ee072..c3a933b5941e1 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -7,25 +7,48 @@ */ package org.elasticsearch.datastreams; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MapperService; +import java.io.IOException; +import java.io.UncheckedIOException; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; +import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH; + +/** + * An {@link IndexSettingProvider} implementation that adds the index.time_series.start_time, + * index.time_series.end_time and index.routing_path index settings to backing indices of + * data streams in time series index mode. + */ public class DataStreamIndexSettingsProvider implements IndexSettingProvider { static final DateFormatter FORMATTER = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; + private final CheckedFunction mapperServiceFactory; + + DataStreamIndexSettingsProvider(CheckedFunction mapperServiceFactory) { + this.mapperServiceFactory = mapperServiceFactory; + } + @Override public Settings getAdditionalIndexSettings( String indexName, @@ -33,7 +56,8 @@ public Settings getAdditionalIndexSettings( boolean timeSeries, Metadata metadata, Instant resolvedAt, - Settings allSettings + Settings allSettings, + List combinedTemplateMappings ) { if (dataStreamName != null) { DataStream dataStream = metadata.dataStreams().get(dataStreamName); @@ -84,6 +108,14 @@ public Settings getAdditionalIndexSettings( assert start.isBefore(end) : "data stream backing index's start time is not before end time"; builder.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), FORMATTER.format(start)); builder.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), FORMATTER.format(end)); + + if (allSettings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey()) == false + && combinedTemplateMappings.isEmpty() == false) { + List routingPaths = findRoutingPaths(indexName, allSettings, combinedTemplateMappings); + if (routingPaths != null) { + builder.putList(INDEX_ROUTING_PATH.getKey(), routingPaths); + } + } return builder.build(); } } @@ -92,4 +124,56 @@ public Settings getAdditionalIndexSettings( return Settings.EMPTY; } + /** + * Find fields in mapping that are of type keyword and time_series_dimension enabled. + * Using MapperService here has an overhead, but allows the mappings from template to + * be merged correctly and fetching the fields without manually parsing the mappings. + * + * Alternatively this method can instead parse mappings into map of maps and merge that and + * iterate over all values to find the field that can serve as routing value. But this requires + * mapping specific logic to exist here. + */ + private List findRoutingPaths(String indexName, Settings allSettings, List combinedTemplateMappings) { + var tmpIndexMetadata = IndexMetadata.builder(indexName); + + int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(allSettings); + int dummyShards = allSettings.getAsInt( + IndexMetadata.SETTING_NUMBER_OF_SHARDS, + dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1 + ); + int shardReplicas = allSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0); + var finalResolvedSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(allSettings) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + // Avoid failing because index.routing_path is missing + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .build(); + + tmpIndexMetadata.settings(finalResolvedSettings); + // Create MapperService just to extract keyword dimension fields: + try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata.build())) { + for (var mapping : combinedTemplateMappings) { + mapperService.merge(MapperService.SINGLE_MAPPING_NAME, mapping, MapperService.MergeReason.INDEX_TEMPLATE); + } + + List routingPaths = null; + for (var fieldMapper : mapperService.documentMapper().mappers().fieldMappers()) { + if (fieldMapper instanceof KeywordFieldMapper keywordFieldMapper) { + if (keywordFieldMapper.fieldType().isDimension()) { + if (routingPaths == null) { + routingPaths = new ArrayList<>(); + } + routingPaths.add(keywordFieldMapper.name()); + } + } + } + return routingPaths; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index e32775ce58d42..4cbfe44e6751c 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -134,7 +134,7 @@ public List getRestHandlers( } @Override - public Collection getAdditionalIndexSettingProviders() { - return List.of(new DataStreamIndexSettingsProvider()); + public Collection getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) { + return List.of(new DataStreamIndexSettingsProvider(parameters.mapperServiceFactory())); } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java index 85c294cb03158..953d6d8d809e0 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java @@ -58,6 +58,7 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.datastreams.MetadataDataStreamRolloverServiceTests.createSettingsProvider; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.ArgumentMatchers.any; @@ -257,7 +258,7 @@ public void setup() throws Exception { null, EmptySystemIndices.INSTANCE, false, - new IndexSettingProviders(Set.of(new DataStreamIndexSettingsProvider())) + new IndexSettingProviders(Set.of(createSettingsProvider(xContentRegistry()))) ); } { diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index 537282bc2d08a..a4e357b0d68af 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -9,13 +9,17 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.test.ESTestCase; +import org.junit.Before; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -23,53 +27,213 @@ import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.FORMATTER; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; public class DataStreamIndexSettingsProviderTests extends ESTestCase { - public void testGetAdditionalIndexSettings() { + DataStreamIndexSettingsProvider provider; + + @Before + public void setup() { + provider = new DataStreamIndexSettingsProvider( + im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()) + ); + } + + public void testGetAdditionalIndexSettings() throws Exception { + Metadata metadata = Metadata.EMPTY_METADATA; + String dataStreamName = "logs-app1"; + + Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); + TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default + Settings settings = Settings.EMPTY; + String mapping = """ + { + "_doc": { + "properties": { + "field1": { + "type": "long" + }, + "field2": { + "type": "keyword" + }, + "field3": { + "type": "keyword", + "time_series_dimension": true + } + } + } + } + """; + Settings result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 1), + dataStreamName, + true, + metadata, + now, + settings, + List.of(new CompressedXContent(mapping)) + ); + assertThat(result.size(), equalTo(3)); + assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis()))); + assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis()))); + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("field3")); + } + + public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throws Exception { + Metadata metadata = Metadata.EMPTY_METADATA; + String dataStreamName = "logs-app1"; + + Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); + TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default + Settings settings = builder().putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field2").build(); + String mapping = """ + { + "_doc": { + "properties": { + "field1": { + "type": "keyword" + "time_series_dimension": true + }, + "field2": { + "type": "keyword", + "time_series_dimension": true + }, + "field3": { + "type": "keyword", + "time_series_dimension": true + } + } + } + } + """; + Settings result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 1), + dataStreamName, + true, + metadata, + now, + settings, + List.of(new CompressedXContent(mapping)) + ); + assertThat(result.size(), equalTo(2)); + assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis()))); + assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis()))); + } + + public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception { + Metadata metadata = Metadata.EMPTY_METADATA; + String dataStreamName = "logs-app1"; + + Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); + TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default + Settings settings = Settings.EMPTY; + String mapping1 = """ + { + "_doc": { + "properties": { + "field1": { + "type": "keyword", + "time_series_dimension": true + }, + "field2": { + "type": "keyword", + "time_series_dimension": true + } + } + } + } + """; + String mapping2 = """ + { + "_doc": { + "properties": { + "field3": { + "type": "keyword", + "time_series_dimension": true + }, + "field4": { + "type": "keyword" + } + } + } + } + """; + String mapping3 = """ + { + "_doc": { + "properties": { + "field2": { + "type": "keyword", + "time_series_dimension": false + }, + "field5": { + "type": "long" + } + } + } + } + """; + Settings result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 1), + dataStreamName, + true, + metadata, + now, + settings, + List.of(new CompressedXContent(mapping1), new CompressedXContent(mapping2), new CompressedXContent(mapping3)) + ); + assertThat(result.size(), equalTo(3)); + assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis()))); + assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis()))); + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field1", "field3")); + } + + public void testGetAdditionalIndexSettingsNoMappings() { Metadata metadata = Metadata.EMPTY_METADATA; String dataStreamName = "logs-app1"; Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default Settings settings = Settings.EMPTY; - var provider = new DataStreamIndexSettingsProvider(); Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, true, metadata, now, - settings + settings, + List.of() ); assertThat(result.size(), equalTo(2)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis()))); } - public void testGetAdditionalIndexSettingsLookAheadTime() { + public void testGetAdditionalIndexSettingsLookAheadTime() throws Exception { Metadata metadata = Metadata.EMPTY_METADATA; String dataStreamName = "logs-app1"; Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); TimeValue lookAheadTime = TimeValue.timeValueMinutes(30); Settings settings = builder().put("index.mode", "time_series").put("index.look_ahead_time", lookAheadTime.getStringRep()).build(); - var provider = new DataStreamIndexSettingsProvider(); Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, true, metadata, now, - settings + settings, + List.of(new CompressedXContent("{}")) ); assertThat(result.size(), equalTo(2)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis()))); } - public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() { + public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() throws Exception { String dataStreamName = "logs-app1"; TimeValue lookAheadTime = TimeValue.timeValueHours(2); @@ -82,14 +246,14 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() { Instant now = sixHoursAgo.plus(6, ChronoUnit.HOURS); Settings settings = Settings.EMPTY; - var provider = new DataStreamIndexSettingsProvider(); var result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, true, metadata, now, - settings + settings, + List.of(new CompressedXContent("{}")) ); assertThat(result.size(), equalTo(2)); assertThat(result.get(IndexSettings.TIME_SERIES_START_TIME.getKey()), equalTo(FORMATTER.format(currentEnd))); @@ -129,7 +293,6 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi Instant now = twoHoursAgo.plus(2, ChronoUnit.HOURS); Settings settings = Settings.EMPTY; - var provider = new DataStreamIndexSettingsProvider(); Exception e = expectThrows( IllegalStateException.class, () -> provider.getAdditionalIndexSettings( @@ -138,7 +301,8 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi true, metadata, now, - settings + settings, + null ) ); assertThat( @@ -156,14 +320,14 @@ public void testGetAdditionalIndexSettingsNonTsdbTemplate() { String dataStreamName = "logs-app1"; Settings settings = Settings.EMPTY; - var provider = new DataStreamIndexSettingsProvider(); Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, false, metadata, Instant.ofEpochMilli(1L), - settings + settings, + null ); assertThat(result.size(), equalTo(0)); } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java index c6d677c089b22..67c763a31b0b7 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java @@ -26,9 +26,11 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -92,7 +94,7 @@ public void testRolloverClusterStateForDataStream() throws Exception { MetadataRolloverService rolloverService = DataStreamTestHelper.getMetadataRolloverService( dataStream, testThreadPool, - Set.of(new DataStreamIndexSettingsProvider()), + Set.of(createSettingsProvider(xContentRegistry())), xContentRegistry() ); MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong()); @@ -187,7 +189,7 @@ public void testRolloverAndMigrateDataStream() throws Exception { MetadataRolloverService rolloverService = DataStreamTestHelper.getMetadataRolloverService( dataStream, testThreadPool, - Set.of(new DataStreamIndexSettingsProvider()), + Set.of(createSettingsProvider(xContentRegistry())), xContentRegistry() ); MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong()); @@ -278,7 +280,7 @@ public void testChangingIndexModeFromTimeSeriesToSomethingElseNoEffectOnExisting MetadataRolloverService rolloverService = DataStreamTestHelper.getMetadataRolloverService( dataStream, testThreadPool, - Set.of(new DataStreamIndexSettingsProvider()), + Set.of(createSettingsProvider(xContentRegistry())), xContentRegistry() ); MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong()); @@ -328,4 +330,10 @@ public void testChangingIndexModeFromTimeSeriesToSomethingElseNoEffectOnExisting } } + static DataStreamIndexSettingsProvider createSettingsProvider(NamedXContentRegistry xContentRegistry) { + return new DataStreamIndexSettingsProvider( + im -> MapperTestUtils.newMapperService(xContentRegistry, createTempDir(), im.getSettings(), im.getIndex().getName()) + ); + } + } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java index cfa1323ca98f2..aebaf1a6c017c 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.datastreams; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.Metadata; @@ -25,6 +26,7 @@ import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.InvalidIndexTemplateException; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -36,8 +38,10 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.generateTsdbMapping; import static org.elasticsearch.common.settings.Settings.builder; +import static org.elasticsearch.datastreams.MetadataDataStreamRolloverServiceTests.createSettingsProvider; import static org.elasticsearch.indices.ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -101,10 +105,105 @@ public void testValidateTsdbDataStreamsReferringTsdbTemplate() throws Exception ); } + public void testRequireRoutingPath() throws Exception { + final var service = getMetadataIndexTemplateService(); + { + // Missing routing path should fail validation + var componentTemplate = new ComponentTemplate(new Template(null, new CompressedXContent("{}"), null), null, null); + var state = service.addComponentTemplate(ClusterState.EMPTY_STATE, true, "1", componentTemplate); + var indexTemplate = new ComposableIndexTemplate( + Collections.singletonList("logs-*-*"), + new Template(builder().put("index.mode", "time_series").build(), null, null), + List.of("1"), + 100L, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(false, false), + null + ); + var e = expectThrows(InvalidIndexTemplateException.class, () -> service.addIndexTemplateV2(state, false, "1", indexTemplate)); + assertThat(e.getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]")); + } + { + // Routing path fetched from mapping of component template + var state = ClusterState.EMPTY_STATE; + var componentTemplate = new ComponentTemplate( + new Template(null, new CompressedXContent(generateTsdbMapping()), null), + null, + null + ); + state = service.addComponentTemplate(state, true, "1", componentTemplate); + var indexTemplate = new ComposableIndexTemplate( + Collections.singletonList("logs-*-*"), + new Template(builder().put("index.mode", "time_series").build(), null, null), + List.of("1"), + 100L, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(false, false), + null + ); + state = service.addIndexTemplateV2(state, false, "1", indexTemplate); + assertThat(state.getMetadata().templatesV2().get("1"), equalTo(indexTemplate)); + } + { + // Routing path defined in component template + var state = ClusterState.EMPTY_STATE; + var componentTemplate = new ComponentTemplate( + new Template(builder().put("index.mode", "time_series").put("index.routing_path", "uid").build(), null, null), + null, + null + ); + state = service.addComponentTemplate(state, true, "1", componentTemplate); + var indexTemplate = new ComposableIndexTemplate( + Collections.singletonList("logs-*-*"), + new Template(null, null, null), + List.of("1"), + 100L, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(false, false), + null + ); + state = service.addIndexTemplateV2(state, false, "1", indexTemplate); + assertThat(state.getMetadata().templatesV2().get("1"), equalTo(indexTemplate)); + } + { + // Routing path defined in index template + var indexTemplate = new ComposableIndexTemplate( + Collections.singletonList("logs-*-*"), + new Template(builder().put("index.mode", "time_series").put("index.routing_path", "uid").build(), null, null), + List.of("1"), + 100L, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(false, false), + null + ); + var state = service.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "1", indexTemplate); + assertThat(state.getMetadata().templatesV2().get("1"), equalTo(indexTemplate)); + } + { + // Routing fetched from mapping in index template + var indexTemplate = new ComposableIndexTemplate( + Collections.singletonList("logs-*-*"), + new Template(builder().put("index.mode", "time_series").build(), new CompressedXContent(generateTsdbMapping()), null), + List.of("1"), + 100L, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(false, false), + null + ); + var state = service.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "1", indexTemplate); + assertThat(state.getMetadata().templatesV2().get("1"), equalTo(indexTemplate)); + } + } + private MetadataIndexTemplateService getMetadataIndexTemplateService() { var indicesService = getInstanceFromNode(IndicesService.class); var clusterService = getInstanceFromNode(ClusterService.class); - var indexSettingProviders = new IndexSettingProviders(Set.of(new DataStreamIndexSettingsProvider())); + var indexSettingProviders = new IndexSettingProviders(Set.of(createSettingsProvider(xContentRegistry()))); var createIndexService = new MetadataCreateIndexService( Settings.EMPTY, clusterService, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 9ffbdec2422af..826f9ccc95956 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -223,6 +223,15 @@ public static Template resolveTemplate( .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); + // empty request mapping as the user can't specify any explicit mappings via the simulate api + List mappings = MetadataCreateIndexService.collectV2Mappings( + null, + simulatedState, + matchingTemplate, + xContentRegistry, + indexName + ); + // First apply settings sourced from index settings providers final var now = Instant.now(); Settings.Builder additionalSettings = Settings.builder(); @@ -233,7 +242,8 @@ public static Template resolveTemplate( template.getDataStreamTemplate() != null && metadata.isTimeSeriesTemplate(template), simulatedState.getMetadata(), now, - templateSettings + templateSettings, + mappings ); dummySettings.put(result); additionalSettings.put(result); @@ -265,15 +275,6 @@ public static Template resolveTemplate( Map aliasesByName = aliases.stream().collect(Collectors.toMap(AliasMetadata::getAlias, Function.identity())); - // empty request mapping as the user can't specify any explicit mappings via the simulate api - List mappings = MetadataCreateIndexService.collectV2Mappings( - null, - simulatedState, - matchingTemplate, - xContentRegistry, - indexName - ); - CompressedXContent mergedMapping = indicesService.withTempIndexService( indexMetadata, tempIndexService -> { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 1e59454860e26..8a39221e5abc7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -62,6 +62,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -902,7 +903,9 @@ public boolean isTimeSeriesTemplate(ComposableIndexTemplate indexTemplate) { } var settings = MetadataIndexTemplateService.resolveSettings(indexTemplate, componentTemplates()); - var indexMode = IndexSettings.MODE.get(settings); + // Not using IndexSettings.MODE.get() to avoid validation that may fail at this point. + var rawIndexMode = settings.get(IndexSettings.MODE.getKey()); + var indexMode = rawIndexMode != null ? Enum.valueOf(IndexMode.class, rawIndexMode.toUpperCase(Locale.ROOT)) : null; if (indexMode == IndexMode.TIME_SERIES) { // No need to check for the existence of index.routing_path here, because index.mode=time_series can't be specified without it. // Setting validation takes care of this. diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index cdae690930c0d..7510de0ecee5d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -531,6 +531,7 @@ private ClusterState applyCreateIndexRequestWithV1Templates( currentState, request, resolveSettings(templates), + mappings == null ? List.of() : List.of(mappings), null, settings, indexScopedSettings, @@ -597,6 +598,7 @@ private ClusterState applyCreateIndexRequestWithV2Template( currentState, request, resolveSettings(currentState.metadata(), templateName), + mappings, null, settings, indexScopedSettings, @@ -653,6 +655,7 @@ private ClusterState applyCreateIndexRequestForSystemDataStream( currentState, request, resolveSettings(template, componentTemplates), + mappings, null, settings, indexScopedSettings, @@ -747,6 +750,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata( currentState, request, Settings.EMPTY, + null, sourceMetadata, settings, indexScopedSettings, @@ -831,6 +835,7 @@ static Settings aggregateIndexSettings( ClusterState currentState, CreateIndexClusterStateUpdateRequest request, Settings combinedTemplateSettings, + List combinedTemplateMappings, @Nullable IndexMetadata sourceMetadata, Settings settings, IndexScopedSettings indexScopedSettings, @@ -867,7 +872,8 @@ static Settings aggregateIndexSettings( timeSeriesTemplate, currentState.getMetadata(), resolvedAt, - templateAndRequestSettings + templateAndRequestSettings, + combinedTemplateMappings ) ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index b0bdacb97f8b0..dd546378be32b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -236,8 +236,8 @@ public ClusterState execute(ClusterState currentState) throws Exception { ); } - // Package visible for testing - ClusterState addComponentTemplate( + // Public visible for testing + public ClusterState addComponentTemplate( final ClusterState currentState, final boolean create, final String name, @@ -628,6 +628,8 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT final var now = Instant.now(); final var metadata = currentState.getMetadata(); + final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), "tmp_idx"); + final var combinedSettings = resolveSettings(indexTemplate, metadata.componentTemplates()); // First apply settings sourced from index setting providers: for (var provider : indexSettingProviders) { finalSettings.put( @@ -637,7 +639,8 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT indexTemplate.getDataStreamTemplate() != null && metadata.isTimeSeriesTemplate(indexTemplate), currentState.getMetadata(), now, - finalTemplate.map(Template::settings).orElse(Settings.EMPTY) + combinedSettings, + combinedMappings ) ); } @@ -1222,8 +1225,7 @@ public static String findV2Template(Metadata metadata, String indexName, boolean /** * Collect the given v2 template into an ordered list of mappings. */ - public static List collectMappings(final ClusterState state, final String templateName, final String indexName) - throws Exception { + public static List collectMappings(final ClusterState state, final String templateName, final String indexName) { final ComposableIndexTemplate template = state.metadata().templatesV2().get(templateName); assert template != null : "attempted to resolve mappings for a template [" + templateName + "] that did not exist in the cluster state"; @@ -1242,7 +1244,7 @@ public static List collectMappings( final ComposableIndexTemplate template, final Map componentTemplates, final String indexName - ) throws Exception { + ) { Objects.requireNonNull(template, "Composable index template must be provided"); List mappings = template.composedOf() .stream() diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java index 54839eaad9d77..69b8f5b29d937 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; @@ -238,7 +239,8 @@ public Settings getAdditionalIndexSettings( boolean timeSeries, Metadata metadata, Instant resolvedAt, - Settings allSettings + Settings allSettings, + List combinedTemplateMappings ) { Set settings = allSettings.keySet(); if (settings.contains(TIER_PREFERENCE)) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java index aa972cdf5a011..e67196c9090c9 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java @@ -8,10 +8,16 @@ package org.elasticsearch.index; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.index.mapper.MapperService; +import java.io.IOException; import java.time.Instant; +import java.util.List; /** * An {@link IndexSettingProvider} is a provider for index level settings that can be set @@ -21,12 +27,16 @@ public interface IndexSettingProvider { /** * Returns explicitly set default index {@link Settings} for the given index. This should not * return null. - * @param indexName The name of the new index being created - * @param dataStreamName The name of the data stream if the index being created is part of a data stream otherwise null - * @param timeSeries Whether the template is in time series mode. - * @param metadata The current metadata instance that doesn't yet contain the index to be created - * @param resolvedAt The time the request to create this new index was accepted. - * @param allSettings All the setting resolved from the template that matches and any setting defined on the create index request + * + * @param indexName The name of the new index being created + * @param dataStreamName The name of the data stream if the index being created is part of a data stream otherwise + * null + * @param timeSeries Whether the template is in time series mode. + * @param metadata The current metadata instance that doesn't yet contain the index to be created + * @param resolvedAt The time the request to create this new index was accepted. + * @param allSettings All the setting resolved from the template that matches and any setting defined on the create index + * request + * @param combinedTemplateMappings All the mappings resolved from the template that matches */ Settings getAdditionalIndexSettings( String indexName, @@ -34,6 +44,14 @@ Settings getAdditionalIndexSettings( boolean timeSeries, Metadata metadata, Instant resolvedAt, - Settings allSettings + Settings allSettings, + List combinedTemplateMappings ); + + /** + * Infrastructure class that holds services that can be used by {@link IndexSettingProvider} instances. + */ + record Parameters(CheckedFunction mapperServiceFactory) { + + } } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index c31796717ff14..21310c25f909c 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -102,6 +102,7 @@ import org.elasticsearch.health.HealthIndicatorService; import org.elasticsearch.health.HealthService; import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexingPressure; @@ -628,8 +629,9 @@ protected Node( searchModule.getRequestCacheKeyDifferentiator() ); + final var parameters = new IndexSettingProvider.Parameters(indicesService::createIndexMapperServiceForValidation); IndexSettingProviders indexSettingProviders = new IndexSettingProviders( - pluginsService.flatMap(Plugin::getAdditionalIndexSettingProviders).collect(Collectors.toSet()) + pluginsService.flatMap(p -> p.getAdditionalIndexSettingProviders(parameters)).collect(Collectors.toSet()) ); final ShardLimitValidator shardLimitValidator = new ShardLimitValidator(settings, clusterService); diff --git a/server/src/main/java/org/elasticsearch/plugins/Plugin.java b/server/src/main/java/org/elasticsearch/plugins/Plugin.java index 77f1c36ac1368..79a5c846be017 100644 --- a/server/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -202,7 +202,7 @@ public void close() throws IOException { * the default values for an index-level setting, these act as though the setting has been set * explicitly, but still allow the setting to be overridden by a template or creation request body. */ - public Collection getAdditionalIndexSettingProviders() { + public Collection getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) { return Collections.emptyList(); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 8a9f155127fc8..d02016e59494c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -715,6 +715,7 @@ public void testAggregateSettingsAppliesSettingsFromTemplatesAndRequest() { request, templateMetadata.settings(), null, + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), @@ -800,6 +801,7 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception { request, templateMetadata.settings(), null, + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), @@ -822,6 +824,7 @@ public void testDefaultSettings() { request, Settings.EMPTY, null, + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), @@ -837,6 +840,7 @@ public void testSettingsFromClusterState() { request, Settings.EMPTY, null, + null, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 15).build(), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), @@ -875,6 +879,7 @@ public void testTemplateOrder() throws Exception { request, MetadataIndexTemplateService.resolveSettings(templates), null, + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), @@ -955,6 +960,7 @@ public void testAggregateIndexSettingsIgnoresTemplatesOnCreateFromSourceIndex() clusterState, request, templateMetadata.settings(), + List.of(templateMetadata.getMappings()), clusterState.metadata().index("sourceIndex"), Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, @@ -1207,6 +1213,7 @@ private Optional aggregatedTierPreference(Settings settings, boolean isD request, templateSettings, null, + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), @@ -1267,6 +1274,7 @@ public void testRejectWithSoftDeletesDisabled() { request, Settings.EMPTY, null, + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), @@ -1301,6 +1309,7 @@ public void testRejectTranslogRetentionSettings() { request, Settings.EMPTY, null, + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), @@ -1331,6 +1340,7 @@ public void testDeprecateTranslogRetentionSettings() { request, Settings.EMPTY, null, + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), @@ -1353,6 +1363,7 @@ public void testDeprecateSimpleFS() { request, Settings.EMPTY, null, + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index 5168460c094f0..375fa13fe36d5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -447,7 +447,7 @@ public Collection createAllocationDeciders(Settings settings, } @Override - public Collection getAdditionalIndexSettingProviders() { + public Collection getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) { return Collections.singleton(new DataTier.DefaultHotAllocationSettingProvider()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java index 38f64c7c407f0..40e6b3fe2bf6f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java @@ -476,10 +476,10 @@ public Map getElectionStrategies() { } @Override - public Collection getAdditionalIndexSettingProviders() { + public Collection getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) { Set providers = new HashSet<>(); - filterPlugins(Plugin.class).stream().forEach(p -> providers.addAll(p.getAdditionalIndexSettingProviders())); - providers.addAll(super.getAdditionalIndexSettingProviders()); + filterPlugins(Plugin.class).stream().forEach(p -> providers.addAll(p.getAdditionalIndexSettingProviders(parameters))); + providers.addAll(super.getAdditionalIndexSettingProviders(parameters)); return providers; }