diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java index 8d3bb4940faf2..b3817bc842cac 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java @@ -31,10 +31,10 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.search.QueryParserHelper; @@ -184,7 +184,7 @@ protected final MapperService createMapperService(String mappings) { similarityService, mapperRegistry, () -> { throw new UnsupportedOperationException(); }, - new IdFieldMapper(() -> true), + new ProvidedIdFieldMapper(() -> true), new ScriptCompiler() { @Override public T compile(Script script, ScriptContext scriptContext) { diff --git a/docs/changelog/82633.yaml b/docs/changelog/82633.yaml new file mode 100644 index 0000000000000..b598d3ab28dd6 --- /dev/null +++ b/docs/changelog/82633.yaml @@ -0,0 +1,5 @@ +pr: 82633 +summary: "TSDB: Support GET and DELETE and doc versioning" +area: TSDB +type: feature +issues: [] diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java index d6b81f4877b5b..26e623d16f5a6 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java @@ -17,7 +17,9 @@ import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; import static org.hamcrest.Matchers.aMapWithSize; @@ -301,10 +303,15 @@ public void testMigrateRegularDataStreamToTsdbDataStream() throws Exception { int numDocs = 32; var currentTime = Instant.now(); var currentMinus30Days = currentTime.minus(30, ChronoUnit.DAYS); + Set times = new HashSet<>(); for (int i = 0; i < numRollovers; i++) { for (int j = 0; j < numDocs; j++) { var indexRequest = new Request("POST", "/k8s/_doc"); - var time = Instant.ofEpochMilli(randomLongBetween(currentMinus30Days.toEpochMilli(), currentTime.toEpochMilli())); + var time = randomValueOtherThanMany( + times::contains, + () -> Instant.ofEpochMilli(randomLongBetween(currentMinus30Days.toEpochMilli(), currentTime.toEpochMilli())) + ); + times.add(time); indexRequest.setJsonEntity(DOC.replace("$time", formatInstant(time))); var response = client().performRequest(indexRequest); assertOK(response); @@ -350,13 +357,15 @@ public void testMigrateRegularDataStreamToTsdbDataStream() throws Exception { assertThat(newIndex, backingIndexEqualTo("k8s", 6)); // Ingest documents that will land in the new tsdb backing index: + var t = currentTime; for (int i = 0; i < numDocs; i++) { var indexRequest = new Request("POST", "/k8s/_doc"); - indexRequest.setJsonEntity(DOC.replace("$time", formatInstant(currentTime))); + indexRequest.setJsonEntity(DOC.replace("$time", formatInstant(t))); var response = client().performRequest(indexRequest); assertOK(response); var responseBody = entityAsMap(response); assertThat((String) responseBody.get("_index"), backingIndexEqualTo("k8s", 6)); + t = t.plusMillis(1000); } // Fail if documents target older non tsdb backing index: diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index b2b00b5b8b2ac..1f6f8a707110b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -48,8 +48,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.DocumentParser; -import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.query.AbstractQueryBuilder; @@ -701,22 +700,22 @@ private static Response prepareRamIndex( CheckedBiFunction handler, IndexService indexService ) throws IOException { - Analyzer defaultAnalyzer = indexService.getIndexAnalyzers().getDefaultIndexAnalyzer(); try (Directory directory = new ByteBuffersDirectory()) { try (IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(defaultAnalyzer))) { - String index = indexService.index().getName(); BytesReference document = request.contextSetup.document; XContentType xContentType = request.contextSetup.xContentType; SourceToParse sourceToParse = new SourceToParse("_id", document, xContentType); - MappingLookup mappingLookup = indexService.mapperService().mappingLookup(); - DocumentParser documentParser = indexService.mapperService().documentParser(); + DocumentMapper documentMapper = indexService.mapperService().documentMapper(); + if (documentMapper == null) { + documentMapper = DocumentMapper.createEmpty(indexService.mapperService()); + } // Note that we are not doing anything with dynamic mapping updates, hence fields that are not mapped but are present // in the sample doc are not accessible from the script through doc['field']. // This is a problem especially for indices that have no mappings, as no fields will be accessible, neither through doc // nor _source (if there are no mappings there are no metadata fields). - ParsedDocument parsedDocument = documentParser.parseDocument(sourceToParse, mappingLookup); + ParsedDocument parsedDocument = documentMapper.parse(sourceToParse); indexWriter.addDocuments(parsedDocument.docs()); try (IndexReader indexReader = DirectoryReader.open(indexWriter)) { final IndexSearcher searcher = new IndexSearcher(indexReader); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index dd0e6035f0ea9..5f5105c243c9c 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -70,10 +70,12 @@ public static class Defaults { } private static void checkIndexCompatibility(IndexSettings settings, String name) { + String indexName = settings.getIndex().getName(); if (settings.getIndexMetadata().isRoutingPartitionedIndex()) { - throw new IllegalStateException( - "cannot create join field [" + name + "] " + "for the partitioned index " + "[" + settings.getIndex().getName() + "]" - ); + throw new IllegalStateException("cannot create join field [" + name + "] for the partitioned index [" + indexName + "]"); + } + if (settings.getIndexMetadata().getRoutingPaths().isEmpty() == false) { + throw new IllegalStateException("cannot create join field [" + name + "] for the index [" + indexName + "] with routing_path"); } } @@ -141,7 +143,7 @@ public ParentJoinFieldMapper build(MapperBuilderContext context) { } } - public static TypeParser PARSER = new TypeParser((n, c) -> { + public static final TypeParser PARSER = new TypeParser((n, c) -> { checkIndexCompatibility(c.getIndexSettings(), n); return new Builder(n); }); @@ -293,7 +295,7 @@ public void parse(DocumentParserContext context) throws IOException { if (fieldType().joiner.parentTypeExists(name)) { // Index the document as a parent String fieldName = fieldType().joiner.childJoinField(name); - parentIdFields.get(fieldName).indexValue(context, context.sourceToParse().id()); + parentIdFields.get(fieldName).indexValue(context, context.id()); } BytesRef binaryValue = new BytesRef(name); diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/10_basic.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/10_basic.yml index 90c09c4a23741..8ae4d151635e6 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/10_basic.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/10_basic.yml @@ -499,12 +499,9 @@ tsdb: - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "cow", "uid":"1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39", "ip": "10.10.55.4", "network": {"tx": 1434521831, "rx": 530575198}}}}' - do: - catch: bad_request delete_by_query: index: tsdb body: query: match_all: {} - - - match: {failures.0.status: 400} - - match: {failures.0.cause.reason: "delete is not supported because the destination index [tsdb] is in time series mode"} + - match: {deleted: 1} diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml index d01b93edb55d1..93e57e5f84a55 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: introduced in 8.2.0 - do: indices.create: @@ -12,9 +12,6 @@ setup: index: mode: time_series routing_path: [metricset, k8s.pod.uid] - time_series: - start_time: 2021-04-28T00:00:00Z - end_time: 2021-04-29T00:00:00Z number_of_replicas: 0 number_of_shards: 2 mappings: @@ -56,25 +53,50 @@ setup: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "cow", "uid":"1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39", "ip": "10.10.55.4", "network": {"tx": 1434595272, "rx": 530605511}}}}' ---- -teardown: - - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 - - do: - indices.delete: - index: tsdb, standard, tsdb_new - ignore_unavailable: true + indices.create: + index: tsdb_new + body: + settings: + index: + mode: time_series + routing_path: [metricset, k8s.pod.uid] + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + k8s: + properties: + pod: + properties: + uid: + type: keyword + time_series_dimension: true + name: + type: keyword + ip: + type: ip + network: + properties: + tx: + type: long + rx: + type: long --- -reindex tsdb index: +from tsdb to standard: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: introduced in 8.2.0 - do: reindex: + refresh: true body: source: index: tsdb @@ -90,9 +112,6 @@ reindex tsdb index: - is_false: task - is_false: deleted - - do: - indices.refresh: {} - - do: search: index: standard @@ -105,49 +124,81 @@ reindex tsdb index: - match: {hits.hits.2._source.@timestamp: 2021-04-28T18:50:53.142Z} - match: {hits.hits.3._source.@timestamp: 2021-04-28T18:51:03.142Z} +--- +from tsdb to tsdb: + - skip: + version: " - 8.1.99" + reason: introduced in 8.2.0 + - do: - indices.create: + reindex: + refresh: true + body: + source: + index: tsdb + dest: + index: tsdb_new + - match: {created: 4} + - match: {updated: 0} + - match: {version_conflicts: 0} + - match: {batches: 1} + - match: {failures: []} + - match: {throttled_millis: 0} + - gte: { took: 0 } + - is_false: task + - is_false: deleted + + - do: + search: index: tsdb_new body: - settings: - index: - mode: time_series - routing_path: [metricset, k8s.pod.uid] - time_series: - start_time: 2021-04-28T00:00:00Z - end_time: 2021-04-29T00:00:00Z - number_of_replicas: 0 - number_of_shards: 2 - mappings: - properties: - "@timestamp": - type: date - metricset: - type: keyword - time_series_dimension: true - k8s: - properties: - pod: - properties: - uid: - type: keyword - time_series_dimension: true - name: - type: keyword - ip: - type: ip - network: - properties: - tx: - type: long - rx: - type: long + sort: '@timestamp' + aggs: + tsids: + terms: + field: _tsid + order: + _key: asc + + - match: {hits.total.value: 4} + - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: pod}} + - match: {aggregations.tsids.buckets.0.doc_count: 4} + - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} + - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} + - match: {hits.hits.2._source.@timestamp: 2021-04-28T18:50:53.142Z} + - match: {hits.hits.3._source.@timestamp: 2021-04-28T18:51:03.142Z} +--- +from standard to tsdb: + - skip: + version: " - 8.1.99" + reason: introduced in 8.2.0 + # Populate the standard index - do: reindex: + refresh: true body: source: + index: tsdb + dest: index: standard + - match: {created: 4} + - match: {updated: 0} + - match: {version_conflicts: 0} + - match: {batches: 1} + - match: {failures: []} + - match: {throttled_millis: 0} + - gte: { took: 0 } + - is_false: task + - is_false: deleted + + # Now test reindexing from it to tsdb + - do: + reindex: + refresh: true + body: + source: + index: tsdb dest: index: tsdb_new - match: {created: 4} @@ -167,7 +218,7 @@ reindex tsdb index: search: index: tsdb_new body: - size: 0 + sort: '@timestamp' aggs: tsids: terms: @@ -178,3 +229,43 @@ reindex tsdb index: - match: {hits.total.value: 4} - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: pod}} - match: {aggregations.tsids.buckets.0.doc_count: 4} + - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} + - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} + - match: {hits.hits.2._source.@timestamp: 2021-04-28T18:50:53.142Z} + - match: {hits.hits.3._source.@timestamp: 2021-04-28T18:51:03.142Z} + +--- +from tsdb to tsdb modifying timestamp: + - skip: + version: " - 8.1.99" + reason: introduced in 8.2.0 + + - do: + catch: bad_request # TODO make this work + reindex: + refresh: true + body: + source: + index: tsdb + dest: + index: tsdb_new + script: + source: ctx._source["@timestamp"] = ctx._source["@timestamp"].replace("-04-", "-05-") + +--- +from tsdb to tsdb modifying dimension: + - skip: + version: " - 8.1.99" + reason: introduced in 8.2.0 + + - do: + catch: bad_request # TODO make this work + reindex: + refresh: true + body: + source: + index: tsdb + dest: + index: tsdb_new + script: + source: ctx._source["metricset"] = "bubbles" diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/10_basic.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/10_basic.yml index a2334139c20e9..f1d7b481f5298 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/10_basic.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/10_basic.yml @@ -357,79 +357,3 @@ id: "1" - match: { _source: {} } - match: { _version: 2 } - ---- -tsdb: - - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 - - - do: - indices.create: - index: tsdb - body: - settings: - index: - mode: time_series - routing_path: [metricset, k8s.pod.uid] - time_series: - start_time: 2021-04-28T00:00:00Z - end_time: 2021-04-29T00:00:00Z - number_of_replicas: 0 - number_of_shards: 2 - mappings: - properties: - "@timestamp": - type: date - metricset: - type: keyword - time_series_dimension: true - k8s: - properties: - pod: - properties: - uid: - type: keyword - time_series_dimension: true - name: - type: keyword - ip: - type: ip - network: - properties: - tx: - type: long - rx: - type: long - - - do: - bulk: - refresh: true - index: tsdb - body: - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "cow", "uid":"1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39", "ip": "10.10.55.4", "network": {"tx": 1434521831, "rx": 530575198}}}}' - - - do: - update_by_query: - index: tsdb - body: - script: - lang: painless - source: ctx._source.k8s.pod.ip = "10.10.55.5" - - - match: {updated: 1} - - match: {version_conflicts: 0} - - match: {batches: 1} - - match: {failures: []} - - match: {throttled_millis: 0} - - gte: { took: 0 } - - - do: - indices.refresh: {} - - - do: - search: - index: tsdb - - match: {hits.total.value: 1} - - match: {hits.hits.0._source.k8s.pod.ip: 10.10.55.5} diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/90_tsdb.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/90_tsdb.yml new file mode 100644 index 0000000000000..cee4395eb87f8 --- /dev/null +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/90_tsdb.yml @@ -0,0 +1,112 @@ +setup: + - skip: + version: " - 8.0.99" + reason: introduced in 8.1.0 + + - do: + indices.create: + index: tsdb + body: + settings: + index: + mode: time_series + routing_path: [metricset, k8s.pod.uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + k8s: + properties: + pod: + properties: + uid: + type: keyword + time_series_dimension: true + name: + type: keyword + ip: + type: ip + network: + properties: + tx: + type: long + rx: + type: long + + - do: + bulk: + refresh: true + index: tsdb + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "cow", "uid":"1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39", "ip": "10.10.55.4", "network": {"tx": 1434521831, "rx": 530575198}}}}' + +--- +update tag field: + - do: + update_by_query: + index: tsdb + refresh: true + body: + script: + lang: painless + source: ctx._source.k8s.pod.ip = "10.10.55.5" + + - match: {updated: 1} + - match: {version_conflicts: 0} + - match: {batches: 1} + - match: {failures: []} + - match: {throttled_millis: 0} + - gte: { took: 0 } + + - do: + search: + index: tsdb + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.k8s.pod.ip: 10.10.55.5} + +--- +update dimension field: + # TODO better error message + - do: + catch: bad_request + update_by_query: + index: tsdb + body: + script: + lang: painless + source: ctx._source.k8s.pod.uid = "12342134" + + - match: {updated: 0} + - match: {version_conflicts: 0} + - match: {batches: 1} + - match: {throttled_millis: 0} + - gte: { took: 0 } + - match: {failures.0.cause.caused_by.reason: /_id\ must\ be\ unset\ or\ set\ to\ .+/} + +--- +update timestamp: + # TODO better error message + - do: + catch: bad_request + update_by_query: + index: tsdb + body: + script: + lang: painless + source: ctx._source["@timestamp"] = "2021-04-28T18:50:33.142Z" + + - match: {updated: 0} + - match: {version_conflicts: 0} + - match: {batches: 1} + - match: {throttled_millis: 0} + - gte: { took: 0 } + - match: {failures.0.cause.caused_by.reason: /_id\ must\ be\ unset\ or\ set\ to\ .+/} diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 81c50e8ff7ed8..f6eb3048ba279 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -226,7 +226,7 @@ public void testNewReplicas() throws Exception { } public void testSearchTimeSeriesMode() throws Exception { - assumeTrue("time series index sort by _tsid introduced in 8.1.0", getOldClusterVersion().onOrAfter(Version.V_8_1_0)); + assumeTrue("indexing time series indices changed in 8.2.0", getOldClusterVersion().onOrAfter(Version.V_8_2_0)); int numDocs; if (isRunningAgainstOldCluster()) { numDocs = createTimeSeriesModeIndex(1); @@ -268,7 +268,7 @@ public void testSearchTimeSeriesMode() throws Exception { } public void testNewReplicasTimeSeriesMode() throws Exception { - assumeTrue("time series index sort by _tsid introduced in 8.1.0", getOldClusterVersion().onOrAfter(Version.V_8_1_0)); + assumeTrue("indexing time series indices changed in 8.2.0", getOldClusterVersion().onOrAfter(Version.V_8_2_0)); if (isRunningAgainstOldCluster()) { createTimeSeriesModeIndex(0); } else { diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java index 2adc334983c93..3e6bc10b89b1b 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java @@ -247,7 +247,7 @@ private void bulk(String index, String valueSuffix, int count) throws IOExceptio } public void testTsdb() throws IOException { - assumeTrue("sort by _tsid added in 8.1.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_1_0)); + assumeTrue("indexing time series indices changed in 8.2.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_2_0)); StringBuilder bulk = new StringBuilder(); switch (CLUSTER_TYPE) { diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_time_series.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_time_series.yml index a9284120ebddc..ba9fc88eef5e4 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_time_series.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_time_series.yml @@ -1,7 +1,7 @@ setup: - skip: - version: " - 8.1.0" - reason: Suipport for time_series aggs was added in 8.1.0 + version: " - 8.2.0" + reason: Time series indexing changed in 8.2.0 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml index f16b1dd71f188..259ff3fb3c66f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml @@ -144,3 +144,20 @@ fetch nested source: - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 } - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 } + +--- +disabling stored fields removes fetch sub phases: + - skip: + version: ' - 7.15.99' + reason: fetch profiling implemented in 7.16.0 + + - do: + search: + index: test + body: + stored_fields: _none_ + profile: true + + - match: { hits.hits.0._index: test } + - match: { profile.shards.0.fetch.debug.stored_fields: [] } + - is_false: profile.shards.0.fetch.children diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml index 03808bd7cb0f3..600ee9868907a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml @@ -1,7 +1,7 @@ setup: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -63,8 +63,8 @@ setup: --- composite aggregation on tsid: - skip: - version: " - 8.0.99" - reason: _tsid introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: search: @@ -113,8 +113,8 @@ composite aggregation on tsid: --- composite aggregation on tsid with after: - skip: - version: " - 8.0.99" - reason: _tsid introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: search: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index 3d36467a48f6a..d19ae05f2e402 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -241,8 +241,9 @@ empty start end times: --- set start_time and end_time: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 + - do: indices.create: index: test_index @@ -303,8 +304,9 @@ set start_time and end_time: --- set start_time and end_time without timeseries mode: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 + - do: catch: /\[index.time_series.start_time\] requires \[index.mode=time_series\]/ indices.create: @@ -328,8 +330,9 @@ set start_time and end_time without timeseries mode: --- set bad start_time and end_time: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 + - do: indices.create: index: test_index @@ -368,8 +371,9 @@ set bad start_time and end_time: --- check start_time and end_time with data_nano: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 + - do: indices.create: index: test_index @@ -427,8 +431,9 @@ check start_time and end_time with data_nano: --- check start_time boundary with data_nano: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 + - do: indices.create: index: test_index @@ -477,8 +482,9 @@ check start_time boundary with data_nano: --- check end_time boundary with data_nano: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 + - do: indices.create: index: test_index @@ -529,7 +535,8 @@ check end_time boundary with data_nano: check time_series default time bound value: - skip: version: " - 8.1.99" - reason: behavior was changed in 8.2.0 + reason: tsdb indexing changed in 8.2.0 + - do: indices.create: index: test_index @@ -561,7 +568,8 @@ check time_series default time bound value: check time_series empty time bound value: - skip: version: " - 8.1.99" - reason: introduced in 8.2.0 + reason: tsdb indexing changed in 8.2.0 + - do: indices.create: index: test_index diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml index 76fb0282df902..b7d8a4b8aad5d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml @@ -2,8 +2,8 @@ --- date: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -51,8 +51,8 @@ date: --- date_nanos: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -100,8 +100,8 @@ date_nanos: --- automatically add with date: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -248,8 +248,8 @@ reject bad timestamp meta field: --- write without timestamp: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -288,8 +288,8 @@ write without timestamp: --- explicitly enable timestamp meta field: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index 59f480e42c98a..f36d210194f24 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -192,8 +192,8 @@ non keyword matches routing_path: --- runtime field matching routing path: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -234,8 +234,8 @@ runtime field matching routing path: --- "dynamic: runtime matches routing_path": - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -271,8 +271,8 @@ runtime field matching routing path: --- "dynamic: false matches routing_path": - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml new file mode 100644 index 0000000000000..db1a8de789fa8 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -0,0 +1,422 @@ +setup: + - skip: + version: " - 8.1.99" + reason: id generation changed in 8.2 + + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + routing_path: [metricset, k8s.pod.uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + k8s: + properties: + pod: + properties: + uid: + type: keyword + time_series_dimension: true + name: + type: keyword + ip: + type: ip + network: + properties: + tx: + type: long + rx: + type: long + + - do: + bulk: + refresh: true + index: test + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}' + +--- +generates a consistent id: + - skip: + version: " - 8.1.99" + reason: ID generation added in 8.2 + + - do: + bulk: + refresh: true + index: test + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - match: {items.0.index._id: cZZNs4NdV58ePSPI8-3SGXkBAAA} + + - do: + bulk: + refresh: true + index: test + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - match: {items.0.index._id: cZZNs4NdV58ePSPI8-3SGXkBAAA} + +--- +index a new document on top of an old one: + - skip: + version: " - 8.1.99" + reason: indexing on top of another document support added in 8.2 + + - do: + search: + index: test + body: + size: 0 + aggs: + max_tx: + max: + field: k8s.pod.network.tx + max_rx: + min: + field: k8s.pod.network.rx + - match: {aggregations.max_tx.value: 2.012916202E9} + - match: {aggregations.max_rx.value: 5.30575198E8} + + - do: + index: + refresh: true + index: test + op_type: index + body: + "@timestamp": "2021-04-28T18:51:03.142Z" + metricset: pod + k8s: + pod: + name: dog + uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 + ip: 10.10.55.3 + network: + tx: 111434595272 + rx: 430605511 + - match: {_id: cn4exTOUtxytuLkQZv7RGXkBAAA} + + - do: + search: + index: test + body: + size: 0 + aggs: + max_tx: + max: + field: k8s.pod.network.tx + max_rx: + min: + field: k8s.pod.network.rx + - match: {aggregations.max_tx.value: 1.11434595272E11} + - match: {aggregations.max_rx.value: 4.30605511E8} + +--- +index a new document on top of an old one over bulk: + - skip: + version: " - 8.1.99" + reason: indexing on top of another document support added in 8.2 + + - do: + search: + index: test + body: + size: 0 + aggs: + max_tx: + max: + field: k8s.pod.network.tx + max_rx: + min: + field: k8s.pod.network.rx + - match: {aggregations.max_tx.value: 2.012916202E9} + - match: {aggregations.max_rx.value: 5.30575198E8} + + - do: + bulk: + refresh: true + index: test + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' + - match: {items.0.index._id: cn4exTOUtxytuLkQZv7RGXkBAAA} + + - do: + search: + index: test + body: + size: 0 + aggs: + max_tx: + max: + field: k8s.pod.network.tx + max_rx: + min: + field: k8s.pod.network.rx + - match: {aggregations.max_tx.value: 1.11434595272E11} + - match: {aggregations.max_rx.value: 4.30605511E8} + +--- +ids query: + - skip: + version: " - 8.1.99" + reason: ids generation changed in 8.2 + + - do: + search: + index: test + body: + fields: + - field: k8s.pod.network.tx + query: + ids: + values: ["cn4exTOUtxytuLkQZv7RGXkBAAA", "cZZNs4NdV58ePSPIkwPSGXkBAAA"] + sort: ["@timestamp"] + - match: {hits.total.value: 2} + - match: {hits.hits.0._id: "cn4exTOUtxytuLkQZv7RGXkBAAA"} + - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434595272]} + - match: {hits.hits.1._id: "cZZNs4NdV58ePSPIkwPSGXkBAAA"} + - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [2012916202]} + +--- +get: + - skip: + version: " - 8.1.99" + reason: ids generation changed in 8.2 + + - do: + get: + index: test + id: cZZNs4NdV58ePSPIkwPSGXkBAAA + - match: {_index: test} + - match: {_id: cZZNs4NdV58ePSPIkwPSGXkBAAA} + - match: + _source: + "@timestamp": "2021-04-28T18:51:04.467Z" + metricset: pod + k8s: + pod: + name: cat + uid: 947e4ced-1786-4e53-9e0c-5c447e959507 + ip: 10.10.55.2 + network: + tx: 2012916202 + rx: 803685721 + +--- +get not found: + - skip: + version: " - 8.1.99" + reason: ids generation changed in 8.2 + + - do: + catch: missing + get: + index: test + id: not found + +--- +get with routing: + - skip: + version: " - 8.1.99" + reason: ids generation changed in 8.2 + + - do: + catch: bad_request + get: + index: test + id: cZZNs4NdV58ePSPIkwPSGXkBAAA + routing: routing + +--- +delete: + - skip: + version: " - 8.1.99" + reason: ids generation changed in 8.2 + + - do: + delete: + index: test + id: cZZNs4NdV58ePSPIkwPSGXkBAAA + - match: {result: deleted} + +--- +delete not found: + - skip: + version: " - 8.1.99" + reason: ids generation changed in 8.2 + + - do: + catch: missing + delete: + index: test + id: not found + +--- +delete with routing: + - skip: + version: " - 8.1.99" + reason: ids generation changed in 8.2 + + - do: + catch: bad_request + delete: + index: test + id: not found + routing: routing + +--- +delete over _bulk: + - skip: + version: " - 8.1.99" + reason: ids generation changed in 8.2 + + - do: + bulk: + refresh: true + index: test + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}' + + - do: + bulk: + index: test + body: + - '{"delete": {"_id": "cn4exTOUtxytuLkQBhTRGXkBAAA"}}' + - '{"delete": {"_id": "cZZNs4NdV58ePSPIkwPSGXkBAAA"}}' + - '{"delete": {"_id": "not found ++ not found"}}' + - match: {items.0.delete.result: deleted} + - match: {items.1.delete.result: deleted} + - match: {items.2.delete.status: 404} + - match: {items.2.delete.error.reason: "invalid id [not found ++ not found] for index [test] in time series mode"} + +--- +routing_path matches deep object: + - skip: + version: " - 8.1.99" + reason: id generation changed in 8.2 + + - do: + indices.create: + index: test2 + body: + settings: + index: + mode: time_series + routing_path: [dim.**.uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date + dim: + properties: + foo: + properties: + bar: + properties: + baz: + properties: + uid: + type: keyword + time_series_dimension: true + + - do: + bulk: + refresh: true + index: test2 + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"bar": {"baz": {"uid": "uid1"}}}}}' + - match: {items.0.index.result: created} + - match: {items.0.index._id: OcEOGaxBa0saxogMMxnRGXkBAAA} + +--- +routing_path matches object: + - skip: + version: " - 8.1.99" + reason: id generation changed in 8.2 + + - do: + indices.create: + index: test2 + body: + settings: + index: + mode: time_series + routing_path: [dim.*.uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date + dim: + properties: + foo: + properties: + uid: + type: keyword + time_series_dimension: true + + - do: + bulk: + refresh: true + index: test2 + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"uid": "uid1"}}}' + - match: {items.0.index.result: created} + - match: {items.0.index._id: 8bgiqUyQKH6n8noAMxnRGXkBAAA} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index 3384f175617c6..1e6dd9243fbe3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -17,8 +17,8 @@ teardown: --- "Create a snapshot and then restore it": - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 features: ["allowed_warnings"] # Create index diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index e3b2614cefc95..51e6125558da3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -1,7 +1,7 @@ setup: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -63,8 +63,8 @@ setup: --- query a dimension: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: search: @@ -79,8 +79,8 @@ query a dimension: --- query a metric: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: search: @@ -96,8 +96,8 @@ query a metric: --- "query tsid fails": - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: catch: /\[_tsid\] is not searchable/ @@ -111,8 +111,8 @@ query a metric: --- fetch a dimension: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: search: @@ -130,8 +130,8 @@ fetch a dimension: --- fetch a metric: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: search: @@ -150,8 +150,8 @@ fetch a metric: --- fetch a tag: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: search: @@ -170,8 +170,8 @@ fetch a tag: --- "fetch the tsid": - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: search: @@ -189,8 +189,8 @@ fetch a tag: --- aggregate a dimension: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: search: @@ -211,8 +211,8 @@ aggregate a dimension: --- aggregate a metric: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: search: @@ -239,8 +239,8 @@ aggregate a metric: --- aggregate a tag: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: search: @@ -265,8 +265,8 @@ aggregate a tag: --- "aggregate the tsid": - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: search: @@ -289,8 +289,8 @@ aggregate a tag: --- "aggregate filter the tsid fails": - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: catch: /\[_tsid\] is not searchable/ @@ -307,8 +307,8 @@ aggregate a tag: --- field capabilities: - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: field_caps: @@ -344,46 +344,11 @@ field capabilities: - is_false: fields._tsid._tsid.non_searchable_indices - is_false: fields._tsid._tsid.non_aggregatable_indices ---- -ids query: - - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 - - # Ingest documents assigning custom ids so we can query them - - do: - bulk: - refresh: true - body: - - '{"index": {"_index": "test", "_id": "u1"}}' - - '{"@timestamp": "2021-04-28T18:45:04.467Z", "metricset": "foo", "k8s": {"pod": {"name": "cat", "uid":"6483d28c-24ee-44f2-926b-63b89d6d8b1b", "ip": "10.10.55.1", "network": {"tx": 2001828691, "rx": 802133794}}}}' - - '{"index": {"_index": "test", "_id": "u2"}}' - - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "foo", "k8s": {"pod": {"name": "cat", "uid":"6483d28c-24ee-44f2-926b-63b89d6d8b1b", "ip": "10.10.55.1", "network": {"tx": 2001838691, "rx": 801479970}}}}' - - '{"index": {"_index": "test", "_id": "u3"}}' - - '{"@timestamp": "2021-04-28T18:55:24.467Z", "metricset": "foo", "k8s": {"pod": {"name": "cat", "uid":"6483d28c-24ee-44f2-926b-63b89d6d8b1b", "ip": "10.10.55.1", "network": {"tx": 2001848691, "rx": 801479970}}}}' - - - do: - search: - index: test - body: - fields: - - field: k8s.pod.network.tx - query: - ids: - values: ["u1", "u3"] - sort: ["@timestamp"] - - - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "u1"} - - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [2001828691]} - - match: {hits.hits.1._id: "u3"} - - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [2001848691]} - --- sort by tsid: - skip: - version: " - 8.0.99" - reason: _tsid introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: search: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml index e87338bf7327a..0035f9be3cfda 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml @@ -1,7 +1,7 @@ setup: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -63,8 +63,8 @@ setup: --- search an alias: - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.put_alias: @@ -92,8 +92,8 @@ search an alias: --- index into alias: - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.put_alias: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index 5e16cabaac9fd..2c151b9b7139f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -1,8 +1,8 @@ --- add dimensions with put_mapping: - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -54,8 +54,8 @@ add dimensions with put_mapping: --- add dimensions to no dims with dynamic_template over index: - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -102,8 +102,8 @@ add dimensions to no dims with dynamic_template over index: --- add dimensions to no dims with dynamic_template over bulk: - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -150,8 +150,8 @@ add dimensions to no dims with dynamic_template over bulk: --- add dimensions to some dims with dynamic_template over index: - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -202,8 +202,8 @@ add dimensions to some dims with dynamic_template over index: --- add dimensions to some dims with dynamic_template over bulk: - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index 8a2ec5497d867..0544dea0174c6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -1,8 +1,9 @@ keyword dimension: - skip: features: close_to - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 + - do: indices.create: @@ -74,8 +75,8 @@ keyword dimension: long dimension: - skip: features: close_to - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -149,8 +150,8 @@ long dimension: ip dimension: - skip: features: close_to - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index ed7e859a851ee..03872f79a037a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -1,7 +1,7 @@ setup: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 features: "arbitrary_key" # Force allocating all shards to a single node so that we can shrink later. @@ -85,8 +85,8 @@ setup: --- split: - skip: - version: " - 8.0.99" - reason: index-split check introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: catch: /index-split is not supported because the destination index \[test\] is in time series mode/ @@ -101,8 +101,8 @@ split: --- shrink: - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.shrink: @@ -128,8 +128,8 @@ shrink: --- clone: - skip: - version: " - 8.0.99" - reason: _tsid support introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.clone: @@ -152,8 +152,8 @@ clone: --- clone no source index: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml index af48b28c75c4a..782e8226ebb31 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml @@ -1,7 +1,7 @@ setup: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: indices.create: @@ -64,34 +64,34 @@ setup: - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}' --- -index with specified routing: +index with routing: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: - catch: /indexing with a specified routing is not supported because the destination index \[test\] is in time series mode/ + catch: /specifying routing is not supported because the destination index \[test\] is in time series mode/ index: index: test routing: foo body: - doc: - "@timestamp": "2021-04-28T18:35:24.467Z" - metricset: "pod" - k8s: - pod: - name: "cat" - uid: "947e4ced-1786-4e53-9e0c-5c447e959507" - ip: "10.10.55.1" - network: - tx: 2001818691 - rx: 802133794 + "@timestamp": "2021-04-28T18:35:24.467Z" + metricset: "pod" + k8s: + pod: + name: "cat" + uid: "947e4ced-1786-4e53-9e0c-5c447e959507" + ip: "10.10.55.1" + network: + tx: 2001818691 + rx: 802133794 --- -index with specified routing over _bulk: +index with routing over _bulk: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 + - do: bulk: refresh: true @@ -99,39 +99,13 @@ index with specified routing over _bulk: body: - '{"index": {"routing": "foo"}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index.error.reason: "indexing with a specified routing is not supported because the destination index [test] is in time series mode"} - ---- -delete: - - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 - - - do: - catch: /delete is not supported because the destination index \[test\] is in time series mode/ - delete: - index: test - id: "1" - ---- -delete over _bulk: - - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 - - - do: - bulk: - index: test - body: - - '{"delete": {"_id": 1}}' - - '{"delete": {"_id": 2}}' - - match: {items.0.delete.error.reason: "delete is not supported because the destination index [test] is in time series mode"} + - match: {items.0.index.error.reason: "specifying routing is not supported because the destination index [test] is in time series mode"} --- noop update: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: search: @@ -152,8 +126,8 @@ noop update: --- update: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 # We fail even though the document isn't found. - do: @@ -177,8 +151,8 @@ update: --- update over _bulk: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: bulk: @@ -191,8 +165,8 @@ update over _bulk: --- search with routing: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 # We fail even though the document isn't found. - do: @@ -204,8 +178,8 @@ search with routing: --- alias with routing: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: catch: /routing is forbidden on CRUD operations that target indices in \[index.mode=time_series\]/ @@ -218,8 +192,8 @@ alias with routing: --- alias with search_routing: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 - do: catch: /routing is forbidden on CRUD operations that target indices in \[index.mode=time_series\]/ @@ -229,3 +203,33 @@ alias with search_routing: body: search_routing: foo +--- +sort by _id: + - skip: + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 + + - do: + catch: /Fielddata is not supported on \[_id\] field in \[time_series\] indices/ + search: + index: test + body: + size: 1 + sort: _id + +--- +aggregate on _id: + - skip: + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 + + - do: + catch: /Fielddata is not supported on \[_id\] field in \[time_series\] indices/ + search: + index: test + body: + size: 1 + aggs: + id: + terms: + field: _id diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index cde1c2e19f794..17949982a88a4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -113,7 +113,7 @@ public void testBulkWithWriteIndexAndRouting() { // allowing the auto-generated timestamp to externally be set would allow making the index inconsistent with duplicate docs public void testExternallySetAutoGeneratedTimestamp() { IndexRequest indexRequest = new IndexRequest("index1").source(Collections.singletonMap("foo", "baz")); - indexRequest.process(); // sets the timestamp + indexRequest.autoGenerateId(); if (randomBoolean()) { indexRequest.id("test"); } diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java b/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java index 739d94fbb545a..da790fa9dd801 100644 --- a/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java @@ -147,7 +147,7 @@ public interface DocWriteRequest extends IndicesRequest, Accountable { /** * Finalize the request before executing or routing it. */ - void process(); + void process(IndexRouting indexRouting); /** * Pick the appropriate shard id to receive this request. diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java index 144880796a3d3..a8a543d2ec4a1 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java @@ -245,7 +245,7 @@ public void markOperationAsExecuted(Engine.Result result) { Engine.IndexResult indexResult = (Engine.IndexResult) result; response = new IndexResponse( primary.shardId(), - requestToExecute.id(), + indexResult.getId(), result.getSeqNo(), result.getTerm(), indexResult.getVersion(), @@ -270,20 +270,19 @@ public void markOperationAsExecuted(Engine.Result result) { executionResult.getResponse().setShardInfo(new ReplicationResponse.ShardInfo()); locationToSync = TransportWriteAction.locationToSync(locationToSync, result.getTranslogLocation()); } - case FAILURE -> executionResult = BulkItemResponse.failure( - current.id(), - docWriteRequest.opType(), - // Make sure to use request.index() here, if you - // use docWriteRequest.index() it will use the - // concrete index instead of an alias if used! - new BulkItemResponse.Failure( - request.index(), - docWriteRequest.id(), - result.getFailure(), - result.getSeqNo(), - result.getTerm() - ) - ); + case FAILURE -> { + /* + * Make sure to use request.index() here, if you + * use docWriteRequest.index() it will use the + * concrete index instead of an alias if used! + */ + String index = request.index(); + executionResult = BulkItemResponse.failure( + current.id(), + docWriteRequest.opType(), + new BulkItemResponse.Failure(index, result.getId(), result.getFailure(), result.getSeqNo(), result.getTerm()) + ); + } default -> throw new AssertionError("unknown result type for " + getCurrentItem() + ": " + result.getResultType()); } currentItemState = ItemProcessingState.EXECUTED; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index ee40476e94b79..1ee1d0dcf7030 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; @@ -532,20 +533,20 @@ protected void doRun() { prohibitCustomRoutingOnDataStream(docWriteRequest, metadata); prohibitAppendWritesInBackingIndices(docWriteRequest, metadata); docWriteRequest.routing(metadata.resolveWriteIndexRouting(docWriteRequest.routing(), docWriteRequest.index())); - docWriteRequest.process(); final Index concreteIndex = docWriteRequest.getConcreteWriteIndex(ia, metadata); if (addFailureIfIndexIsClosed(docWriteRequest, concreteIndex, i, metadata)) { continue; } IndexRouting indexRouting = concreteIndices.routing(concreteIndex); + docWriteRequest.process(indexRouting); int shardId = docWriteRequest.route(indexRouting); List shardRequests = requestsByShard.computeIfAbsent( new ShardId(concreteIndex, shardId), shard -> new ArrayList<>() ); shardRequests.add(new BulkItemRequest(i, docWriteRequest)); - } catch (ElasticsearchParseException | IllegalArgumentException | IndexNotFoundException | RoutingMissingException e) { + } catch (ElasticsearchParseException | IllegalArgumentException | RoutingMissingException | ResourceNotFoundException e) { String name = ia != null ? ia.getName() : docWriteRequest.index(); BulkItemResponse.Failure failure = new BulkItemResponse.Failure(name, docWriteRequest.id(), e); BulkItemResponse bulkItemResponse = BulkItemResponse.failure(i, docWriteRequest.opType(), failure); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index cf15b273cb0b5..c157963023e59 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -218,7 +218,8 @@ protected void doRun() { e, primary, docWriteRequest.opType() == DocWriteRequest.OpType.DELETE, - docWriteRequest.version() + docWriteRequest.version(), + docWriteRequest.id() ), context, null @@ -274,7 +275,7 @@ static boolean executeBulkItemRequest( } catch (Exception failure) { // we may fail translating a update to index or delete operation // we use index result to communicate failure while translating update request - final Engine.Result result = new Engine.IndexResult(failure, updateRequest.version()); + final Engine.Result result = new Engine.IndexResult(failure, updateRequest.version(), updateRequest.id()); context.setRequestToExecute(updateRequest); context.markOperationAsExecuted(result); context.markAsCompleted(context.getExecutionResult()); @@ -285,9 +286,7 @@ static boolean executeBulkItemRequest( context.markAsCompleted(context.getExecutionResult()); return true; } - DocWriteRequest translated = updateResult.action(); - translated.process(); - context.setRequestToExecute(translated); + context.setRequestToExecute(updateResult.action()); } else { context.setRequestToExecute(context.getCurrent()); updateResult = null; @@ -338,7 +337,8 @@ static boolean executeBulkItemRequest( ); } catch (Exception e) { logger.info(() -> new ParameterizedMessage("{} mapping update rejected by primary", primary.shardId()), e); - onComplete(exceptionToResult(e, primary, isDelete, version), context, updateResult); + assert result.getId() != null; + onComplete(exceptionToResult(e, primary, isDelete, version, result.getId()), context, updateResult); return true; } @@ -362,7 +362,7 @@ public void onFailure(Exception e) { @Override public void onFailure(Exception e) { - onComplete(exceptionToResult(e, primary, isDelete, version), context, updateResult); + onComplete(exceptionToResult(e, primary, isDelete, version, result.getId()), context, updateResult); // Requesting mapping update failed, so we don't have to wait for a cluster state update assert context.isInitial(); itemDoneListener.onResponse(null); @@ -375,8 +375,9 @@ public void onFailure(Exception e) { return true; } - private static Engine.Result exceptionToResult(Exception e, IndexShard primary, boolean isDelete, long version) { - return isDelete ? primary.getFailedDeleteResult(e, version) : primary.getFailedIndexResult(e, version); + private static Engine.Result exceptionToResult(Exception e, IndexShard primary, boolean isDelete, long version, String id) { + assert id != null; + return isDelete ? primary.getFailedDeleteResult(e, version, id) : primary.getFailedIndexResult(e, version, id); } private static void onComplete(Engine.Result r, BulkPrimaryExecutionContext context, UpdateHelper.Result updateResult) { diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java index 1053eddced6c0..42914d85bdeba 100644 --- a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java @@ -234,7 +234,7 @@ public boolean isRequireAlias() { } @Override - public void process() { + public void process(IndexRouting indexRouting) { // Nothing to do } diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 0f3b7d755976c..6d3269a5854c2 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -591,20 +591,29 @@ public VersionType versionType() { } @Override - public void process() { - if ("".equals(id)) { - throw new IllegalArgumentException("if _id is specified it must not be empty"); - } + public void process(IndexRouting indexRouting) { + indexRouting.process(this); + } - // generate id if not already provided - if (id == null) { - assert autoGeneratedTimestamp == UNSET_AUTO_GENERATED_TIMESTAMP : "timestamp has already been generated!"; - assert ifSeqNo == UNASSIGNED_SEQ_NO; - assert ifPrimaryTerm == UNASSIGNED_PRIMARY_TERM; - autoGeneratedTimestamp = Math.max(0, System.currentTimeMillis()); // extra paranoia - String uid = UUIDs.base64UUID(); - id(uid); - } + /** + * Set the {@code #id()} to an automatically generated one and make this + * request compatible with the append-only optimization. + */ + public void autoGenerateId() { + assert id == null; + assert autoGeneratedTimestamp == UNSET_AUTO_GENERATED_TIMESTAMP : "timestamp has already been generated!"; + assert ifSeqNo == UNASSIGNED_SEQ_NO; + assert ifPrimaryTerm == UNASSIGNED_PRIMARY_TERM; + /* + * Set the auto generated timestamp so the append only optimization + * can quickly test if this request *must* be unique without reaching + * into the Lucene index. We lock it >0 because UNSET_AUTO_GENERATED_TIMESTAMP + * has a special meaning and is a negative value. This optimiation will + * never work before 1970, but that's ok. It's after 1970. + */ + autoGeneratedTimestamp = Math.max(0, System.currentTimeMillis()); + String uid = UUIDs.base64UUID(); + id(uid); } public void checkAutoIdWithOpTypeCreateSupportedByVersion(Version version) { @@ -728,7 +737,6 @@ public Index getConcreteWriteIndex(IndexAbstraction ia, Metadata metadata) { @Override public int route(IndexRouting indexRouting) { - assert id != null : "route must be called after process"; return indexRouting.indexShard(id, routing, contentType, source); } diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index a0014e8e46d5a..5de40117b7c76 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -832,7 +832,7 @@ public boolean isRequireAlias() { } @Override - public void process() { + public void process(IndexRouting indexRouting) { // Nothing to do } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java index 100e8782f9b48..0a99a36dfc1f2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java @@ -22,7 +22,6 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; @@ -191,7 +190,7 @@ public Set> entrySet() { similarityService, mapperRegistry, () -> null, - IdFieldMapper.NO_FIELD_DATA, + indexSettings.getMode().buildNoFieldDataIdFieldMapper(), scriptService ); mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 8bf20281844f6..899501f22cb5d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -10,11 +10,15 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.RoutingMissingException; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.XContentParser; @@ -24,9 +28,11 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Base64; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.IntConsumer; @@ -59,6 +65,8 @@ private IndexRouting(IndexMetadata metadata) { this.routingFactor = metadata.getRoutingFactor(); } + public abstract void process(IndexRequest indexRequest); + /** * Called when indexing a document to generate the shard id that should contain * a document with the provided parameters. @@ -129,8 +137,23 @@ private abstract static class IdAndRoutingOnly extends IndexRouting { protected abstract int shardId(String id, @Nullable String routing); + @Override + public void process(IndexRequest indexRequest) { + if ("".equals(indexRequest.id())) { + throw new IllegalArgumentException("if _id is specified it must not be empty"); + } + + // generate id if not already provided + if (indexRequest.id() == null) { + indexRequest.autoGenerateId(); + } + } + @Override public int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source) { + if (id == null) { + throw new IllegalStateException("id is required and should have been set by process"); + } checkRoutingRequired(id, routing); return shardId(id, routing); } @@ -208,7 +231,8 @@ public void collectSearchShards(String routing, IntConsumer consumer) { } } - private static class ExtractFromSource extends IndexRouting { + public static class ExtractFromSource extends IndexRouting { + private final List routingPaths; private final XContentParserConfiguration parserConfig; ExtractFromSource(IndexMetadata metadata) { @@ -216,16 +240,36 @@ private static class ExtractFromSource extends IndexRouting { if (metadata.isRoutingPartitionedIndex()) { throw new IllegalArgumentException("routing_partition_size is incompatible with routing_path"); } - this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(metadata.getRoutingPaths()), null, true); + this.routingPaths = metadata.getRoutingPaths(); + this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(routingPaths), null, true); } + @Override + public void process(IndexRequest indexRequest) {} + @Override public int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source) { - if (routing != null) { - throw new IllegalArgumentException(error("indexing with a specified routing")); - } assert Transports.assertNotTransportThread("parsing the _source can get slow"); + checkNoRouting(routing); + return hashToShardId(hashSource(sourceType, source)); + } + + public String createId(XContentType sourceType, BytesReference source, byte[] suffix) { + return createId(hashSource(sourceType, source), suffix); + } + + public String createId(Map flat, byte[] suffix) { + return createId(hashSource(flat), suffix); + } + private String createId(int routingHash, byte[] suffix) { + byte[] idBytes = new byte[4 + suffix.length]; + ByteUtils.writeIntLE(routingHash, idBytes, 0); + System.arraycopy(suffix, 0, idBytes, 4, suffix.length); + return Base64.getUrlEncoder().withoutPadding().encodeToString(idBytes); + } + + private int hashSource(XContentType sourceType, BytesReference source) { List hashes = new ArrayList<>(); try { try (XContentParser parser = sourceType.xContent().createParser(parserConfig, source.streamInput())) { @@ -240,7 +284,7 @@ public int indexShard(String id, @Nullable String routing, XContentType sourceTy } catch (IOException | ParsingException e) { throw new IllegalArgumentException("Error extracting routing: " + e.getMessage(), e); } - return hashToShardId(hashesToHash(hashes)); + return hashesToHash(hashes); } private static void extractObject(List hashes, @Nullable String path, XContentParser source) throws IOException { @@ -276,6 +320,16 @@ private static void extractItem(List hashes, String path, XContentP } } + private int hashSource(Map flat) { + List hashes = new ArrayList<>(); + for (Map.Entry e : flat.entrySet()) { + if (Regex.simpleMatch(routingPaths, e.getKey())) { + hashes.add(new NameAndHash(new BytesRef(e.getKey()), hash(new BytesRef(e.getValue().toString())))); + } + } + return hashesToHash(hashes); + } + private static int hash(BytesRef ref) { return StringHelper.murmurhash3_x86_32(ref, 0); } @@ -307,12 +361,33 @@ public int updateShard(String id, @Nullable String routing) { @Override public int deleteShard(String id, @Nullable String routing) { - throw new IllegalArgumentException(error("delete")); + checkNoRouting(routing); + return idToHash(id); } @Override public int getShard(String id, @Nullable String routing) { - throw new IllegalArgumentException(error("get")); + checkNoRouting(routing); + return idToHash(id); + } + + private void checkNoRouting(@Nullable String routing) { + if (routing != null) { + throw new IllegalArgumentException(error("specifying routing")); + } + } + + private int idToHash(String id) { + byte[] idBytes; + try { + idBytes = Base64.getUrlDecoder().decode(id); + } catch (IllegalArgumentException e) { + throw new ResourceNotFoundException("invalid id [{}] for index [{}] in time series mode", id, indexName); + } + if (idBytes.length < 4) { + throw new ResourceNotFoundException("invalid id [{}] for index [{}] in time series mode", id, indexName); + } + return hashToShardId(ByteUtils.readIntLE(idBytes, 0)); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index de6633db7de15..3d45ff4834f29 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -18,18 +18,22 @@ import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentDimensions; +import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.NestedLookup; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; +import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.BooleanSupplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -86,6 +90,16 @@ public MetadataFieldMapper buildTimeSeriesIdFieldMapper() { return null; } + @Override + public IdFieldMapper buildNoFieldDataIdFieldMapper() { + return ProvidedIdFieldMapper.NO_FIELD_DATA; + } + + @Override + public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { + return new ProvidedIdFieldMapper(fieldDataEnabled); + } + @Override public DocumentDimensions buildDocumentDimensions() { return new DocumentDimensions.OnlySingleValueAllowed(); @@ -156,6 +170,17 @@ public MetadataFieldMapper buildTimeSeriesIdFieldMapper() { return TimeSeriesIdFieldMapper.INSTANCE; } + @Override + public IdFieldMapper buildNoFieldDataIdFieldMapper() { + return TsidExtractingIdFieldMapper.INSTANCE; + } + + @Override + public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { + // We don't support field data on TSDB's _id + return TsidExtractingIdFieldMapper.INSTANCE; + } + @Override public DocumentDimensions buildDocumentDimensions() { return new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); @@ -239,6 +264,10 @@ public String getName() { @Nullable public abstract CompressedXContent getDefaultMapping(); + public abstract IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled); + + public abstract IdFieldMapper buildNoFieldDataIdFieldMapper(); + /** * Get timebounds */ @@ -252,6 +281,9 @@ public String getName() { */ public abstract MetadataFieldMapper buildTimeSeriesIdFieldMapper(); + /** + * How {@code time_series_dimension} fields are handled by indices in this mode. + */ public abstract DocumentDimensions buildDocumentDimensions(); public static IndexMode fromString(String value) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index 1d13e64f23fe4..62b7b33281d58 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -581,7 +581,7 @@ public MapperService newIndexMapperService( new SimilarityService(indexSettings, scriptService, similarities), mapperRegistry, () -> { throw new UnsupportedOperationException("no index query shard context available"); }, - IdFieldMapper.NO_FIELD_DATA, + indexSettings.getMode().buildNoFieldDataIdFieldMapper(), scriptService ); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index b4c2a07164ba7..769cf26f2db05 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -13,6 +13,7 @@ import org.elasticsearch.Build; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; @@ -634,6 +635,8 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { */ private volatile int maxRegexLength; + private final IndexRouting indexRouting; + /** * Returns the default search fields for this index. */ @@ -745,6 +748,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mappingDepthLimit = scopedSettings.get(INDEX_MAPPING_DEPTH_LIMIT_SETTING); mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING); mappingDimensionFieldsLimit = scopedSettings.get(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING); + indexRouting = IndexRouting.fromIndexMetadata(indexMetadata); scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio); scopedSettings.addSettingsUpdateConsumer( @@ -1344,4 +1348,12 @@ private void setMappingDimensionFieldsLimit(long value) { public TimestampBounds getTimestampBounds() { return timestampBounds; } + + /** + * The way that documents are routed on the coordinating + * node when being sent to shards of this index. + */ + public IndexRouting getIndexRouting() { + return indexRouting; + } } diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 2b97508b5bbe6..5c6b66bbdb039 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -358,10 +358,11 @@ public abstract static class Result { private final Exception failure; private final SetOnce freeze = new SetOnce<>(); private final Mapping requiredMappingUpdate; + private final String id; private Translog.Location translogLocation; private long took; - protected Result(Operation.TYPE operationType, Exception failure, long version, long term, long seqNo) { + protected Result(Operation.TYPE operationType, Exception failure, long version, long term, long seqNo, String id) { this.operationType = operationType; this.failure = Objects.requireNonNull(failure); this.version = version; @@ -369,9 +370,10 @@ protected Result(Operation.TYPE operationType, Exception failure, long version, this.seqNo = seqNo; this.requiredMappingUpdate = null; this.resultType = Type.FAILURE; + this.id = id; } - protected Result(Operation.TYPE operationType, long version, long term, long seqNo) { + protected Result(Operation.TYPE operationType, long version, long term, long seqNo, String id) { this.operationType = operationType; this.version = version; this.seqNo = seqNo; @@ -379,9 +381,10 @@ protected Result(Operation.TYPE operationType, long version, long term, long seq this.failure = null; this.requiredMappingUpdate = null; this.resultType = Type.SUCCESS; + this.id = id; } - protected Result(Operation.TYPE operationType, Mapping requiredMappingUpdate) { + protected Result(Operation.TYPE operationType, Mapping requiredMappingUpdate, String id) { this.operationType = operationType; this.version = Versions.NOT_FOUND; this.seqNo = UNASSIGNED_SEQ_NO; @@ -389,6 +392,7 @@ protected Result(Operation.TYPE operationType, Mapping requiredMappingUpdate) { this.failure = null; this.requiredMappingUpdate = requiredMappingUpdate; this.resultType = Type.MAPPING_UPDATE_REQUIRED; + this.id = id; } /** whether the operation was successful, has failed or was aborted due to a mapping update */ @@ -441,6 +445,10 @@ public Operation.TYPE getOperationType() { return operationType; } + public String getId() { + return id; + } + void setTranslogLocation(Translog.Location translogLocation) { if (freeze.get() == null) { this.translogLocation = translogLocation; @@ -472,57 +480,56 @@ public static class IndexResult extends Result { private final boolean created; - public IndexResult(long version, long term, long seqNo, boolean created) { - super(Operation.TYPE.INDEX, version, term, seqNo); + public IndexResult(long version, long term, long seqNo, boolean created, String id) { + super(Operation.TYPE.INDEX, version, term, seqNo, id); this.created = created; } /** * use in case of the index operation failed before getting to internal engine **/ - public IndexResult(Exception failure, long version) { - this(failure, version, UNASSIGNED_PRIMARY_TERM, UNASSIGNED_SEQ_NO); + public IndexResult(Exception failure, long version, String id) { + this(failure, version, UNASSIGNED_PRIMARY_TERM, UNASSIGNED_SEQ_NO, id); } - public IndexResult(Exception failure, long version, long term, long seqNo) { - super(Operation.TYPE.INDEX, failure, version, term, seqNo); + public IndexResult(Exception failure, long version, long term, long seqNo, String id) { + super(Operation.TYPE.INDEX, failure, version, term, seqNo, id); this.created = false; } - public IndexResult(Mapping requiredMappingUpdate) { - super(Operation.TYPE.INDEX, requiredMappingUpdate); + public IndexResult(Mapping requiredMappingUpdate, String id) { + super(Operation.TYPE.INDEX, requiredMappingUpdate, id); this.created = false; } public boolean isCreated() { return created; } - } public static class DeleteResult extends Result { private final boolean found; - public DeleteResult(long version, long term, long seqNo, boolean found) { - super(Operation.TYPE.DELETE, version, term, seqNo); + public DeleteResult(long version, long term, long seqNo, boolean found, String id) { + super(Operation.TYPE.DELETE, version, term, seqNo, id); this.found = found; } /** * use in case of the delete operation failed before getting to internal engine **/ - public DeleteResult(Exception failure, long version, long term) { - this(failure, version, term, UNASSIGNED_SEQ_NO, false); + public DeleteResult(Exception failure, long version, long term, String id) { + this(failure, version, term, UNASSIGNED_SEQ_NO, false, id); } - public DeleteResult(Exception failure, long version, long term, long seqNo, boolean found) { - super(Operation.TYPE.DELETE, failure, version, term, seqNo); + public DeleteResult(Exception failure, long version, long term, long seqNo, boolean found, String id) { + super(Operation.TYPE.DELETE, failure, version, term, seqNo, id); this.found = found; } - public DeleteResult(Mapping requiredMappingUpdate) { - super(Operation.TYPE.DELETE, requiredMappingUpdate); + public DeleteResult(Mapping requiredMappingUpdate, String id) { + super(Operation.TYPE.DELETE, requiredMappingUpdate, id); this.found = false; } @@ -535,11 +542,11 @@ public boolean isFound() { public static class NoOpResult extends Result { NoOpResult(long term, long seqNo) { - super(Operation.TYPE.NO_OP, 0, term, seqNo); + super(Operation.TYPE.NO_OP, 0, term, seqNo, null); } NoOpResult(long term, long seqNo, Exception failure) { - super(Operation.TYPE.NO_OP, failure, 0, term, seqNo); + super(Operation.TYPE.NO_OP, failure, 0, term, seqNo, null); } } diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 4319529e92622..f87905393c05b 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -998,7 +998,8 @@ public IndexResult index(Index index) throws IOException { plan.versionForIndexing, index.primaryTerm(), index.seqNo(), - plan.currentNotFoundOrDeleted + plan.currentNotFoundOrDeleted, + index.id() ); } } @@ -1108,7 +1109,7 @@ private IndexingStrategy planIndexingAsPrimary(Index index) throws IOException { if (canOptimizeAddDocument && mayHaveBeenIndexedBefore(index) == false) { final Exception reserveError = tryAcquireInFlightDocs(index, reservingDocs); if (reserveError != null) { - plan = IndexingStrategy.failAsTooManyDocs(reserveError); + plan = IndexingStrategy.failAsTooManyDocs(reserveError, index.id()); } else { plan = IndexingStrategy.optimizedAppendOnly(1L, reservingDocs); } @@ -1134,7 +1135,7 @@ private IndexingStrategy planIndexingAsPrimary(Index index) throws IOException { SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM ); - plan = IndexingStrategy.skipDueToVersionConflict(e, true, currentVersion); + plan = IndexingStrategy.skipDueToVersionConflict(e, true, currentVersion, index.id()); } else if (index.getIfSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && (versionValue.seqNo != index.getIfSeqNo() || versionValue.term != index.getIfPrimaryTerm())) { final VersionConflictEngineException e = new VersionConflictEngineException( @@ -1145,7 +1146,7 @@ private IndexingStrategy planIndexingAsPrimary(Index index) throws IOException { versionValue.seqNo, versionValue.term ); - plan = IndexingStrategy.skipDueToVersionConflict(e, currentNotFoundOrDeleted, currentVersion); + plan = IndexingStrategy.skipDueToVersionConflict(e, currentNotFoundOrDeleted, currentVersion, index.id()); } else if (index.versionType().isVersionConflictForWrites(currentVersion, index.version(), currentNotFoundOrDeleted)) { final VersionConflictEngineException e = new VersionConflictEngineException( shardId, @@ -1153,11 +1154,11 @@ private IndexingStrategy planIndexingAsPrimary(Index index) throws IOException { currentVersion, currentNotFoundOrDeleted ); - plan = IndexingStrategy.skipDueToVersionConflict(e, currentNotFoundOrDeleted, currentVersion); + plan = IndexingStrategy.skipDueToVersionConflict(e, currentNotFoundOrDeleted, currentVersion, index.id()); } else { final Exception reserveError = tryAcquireInFlightDocs(index, reservingDocs); if (reserveError != null) { - plan = IndexingStrategy.failAsTooManyDocs(reserveError); + plan = IndexingStrategy.failAsTooManyDocs(reserveError, index.id()); } else { plan = IndexingStrategy.processNormally( currentNotFoundOrDeleted, @@ -1191,7 +1192,7 @@ private IndexResult indexIntoLucene(Index index, IndexingStrategy plan) throws I assert assertDocDoesNotExist(index, canOptimizeAddDocument(index) == false); addDocs(index.docs(), indexWriter); } - return new IndexResult(plan.versionForIndexing, index.primaryTerm(), index.seqNo(), plan.currentNotFoundOrDeleted); + return new IndexResult(plan.versionForIndexing, index.primaryTerm(), index.seqNo(), plan.currentNotFoundOrDeleted, index.id()); } catch (Exception ex) { if (ex instanceof AlreadyClosedException == false && indexWriter.getTragicException() == null @@ -1209,7 +1210,7 @@ && treatDocumentFailureAsTragicError(index) == false) { * we return a `MATCH_ANY` version to indicate no document was index. The value is * not used anyway */ - return new IndexResult(ex, Versions.MATCH_ANY, index.primaryTerm(), index.seqNo()); + return new IndexResult(ex, Versions.MATCH_ANY, index.primaryTerm(), index.seqNo(), index.id()); } else { throw ex; } @@ -1314,9 +1315,10 @@ static IndexingStrategy optimizedAppendOnly(long versionForIndexing, int reserve public static IndexingStrategy skipDueToVersionConflict( VersionConflictEngineException e, boolean currentNotFoundOrDeleted, - long currentVersion + long currentVersion, + String id ) { - final IndexResult result = new IndexResult(e, currentVersion); + final IndexResult result = new IndexResult(e, currentVersion, id); return new IndexingStrategy(currentNotFoundOrDeleted, false, false, false, Versions.NOT_FOUND, 0, result); } @@ -1340,8 +1342,8 @@ static IndexingStrategy processAsStaleOp(long versionForIndexing, int reservedDo return new IndexingStrategy(false, false, false, true, versionForIndexing, reservedDocs, null); } - static IndexingStrategy failAsTooManyDocs(Exception e) { - final IndexResult result = new IndexResult(e, Versions.NOT_FOUND); + static IndexingStrategy failAsTooManyDocs(Exception e, String id) { + final IndexResult result = new IndexResult(e, Versions.NOT_FOUND, id); return new IndexingStrategy(false, false, false, false, Versions.NOT_FOUND, 0, result); } } @@ -1424,7 +1426,8 @@ public DeleteResult delete(Delete delete) throws IOException { plan.versionOfDeletion, delete.primaryTerm(), delete.seqNo(), - plan.currentlyDeleted == false + plan.currentlyDeleted == false, + delete.id() ); } if (plan.deleteFromLucene) { @@ -1550,7 +1553,7 @@ private DeletionStrategy planDeletionAsPrimary(Delete delete) throws IOException SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM ); - plan = DeletionStrategy.skipDueToVersionConflict(e, currentVersion, true); + plan = DeletionStrategy.skipDueToVersionConflict(e, currentVersion, true, delete.id()); } else if (delete.getIfSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && (versionValue.seqNo != delete.getIfSeqNo() || versionValue.term != delete.getIfPrimaryTerm())) { final VersionConflictEngineException e = new VersionConflictEngineException( @@ -1561,7 +1564,7 @@ private DeletionStrategy planDeletionAsPrimary(Delete delete) throws IOException versionValue.seqNo, versionValue.term ); - plan = DeletionStrategy.skipDueToVersionConflict(e, currentVersion, currentlyDeleted); + plan = DeletionStrategy.skipDueToVersionConflict(e, currentVersion, currentlyDeleted, delete.id()); } else if (delete.versionType().isVersionConflictForWrites(currentVersion, delete.version(), currentlyDeleted)) { final VersionConflictEngineException e = new VersionConflictEngineException( shardId, @@ -1569,11 +1572,11 @@ private DeletionStrategy planDeletionAsPrimary(Delete delete) throws IOException currentVersion, currentlyDeleted ); - plan = DeletionStrategy.skipDueToVersionConflict(e, currentVersion, currentlyDeleted); + plan = DeletionStrategy.skipDueToVersionConflict(e, currentVersion, currentlyDeleted, delete.id()); } else { final Exception reserveError = tryAcquireInFlightDocs(delete, 1); if (reserveError != null) { - plan = DeletionStrategy.failAsTooManyDocs(reserveError); + plan = DeletionStrategy.failAsTooManyDocs(reserveError, delete.id()); } else { final long versionOfDeletion = delete.versionType().updateVersion(currentVersion, delete.version()); plan = DeletionStrategy.processNormally(currentlyDeleted, versionOfDeletion, 1); @@ -1598,7 +1601,13 @@ private DeleteResult deleteInLucene(Delete delete, DeletionStrategy plan) throws } else { indexWriter.softUpdateDocument(delete.uid(), doc, softDeletesField); } - return new DeleteResult(plan.versionOfDeletion, delete.primaryTerm(), delete.seqNo(), plan.currentlyDeleted == false); + return new DeleteResult( + plan.versionOfDeletion, + delete.primaryTerm(), + delete.seqNo(), + plan.currentlyDeleted == false, + delete.id() + ); } catch (final Exception ex) { /* * Document level failures when deleting are unexpected, we likely hit something fatal such as the Lucene index being corrupt, @@ -1655,14 +1664,16 @@ private DeletionStrategy( public static DeletionStrategy skipDueToVersionConflict( VersionConflictEngineException e, long currentVersion, - boolean currentlyDeleted + boolean currentlyDeleted, + String id ) { final DeleteResult deleteResult = new DeleteResult( e, currentVersion, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, SequenceNumbers.UNASSIGNED_SEQ_NO, - currentlyDeleted == false + currentlyDeleted == false, + id ); return new DeletionStrategy(false, false, currentlyDeleted, Versions.NOT_FOUND, 0, deleteResult); } @@ -1680,13 +1691,14 @@ static DeletionStrategy processAsStaleOp(long versionOfDeletion) { return new DeletionStrategy(false, true, false, versionOfDeletion, 0, null); } - static DeletionStrategy failAsTooManyDocs(Exception e) { + static DeletionStrategy failAsTooManyDocs(Exception e, String id) { final DeleteResult deleteResult = new DeleteResult( e, Versions.NOT_FOUND, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, SequenceNumbers.UNASSIGNED_SEQ_NO, - false + false, + id ); return new DeletionStrategy(false, false, false, Versions.NOT_FOUND, 0, deleteResult); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 5ccb43c099f91..c4e87edfeae94 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -60,10 +60,6 @@ public SourceFieldMapper sourceMapper() { return metadataMapper(SourceFieldMapper.class); } - public IdFieldMapper idFieldMapper() { - return metadataMapper(IdFieldMapper.class); - } - public RoutingFieldMapper routingFieldMapper() { return metadataMapper(RoutingFieldMapper.class); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index e6fa6bef98ae3..5f6571579fd17 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -90,7 +90,7 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL return new ParsedDocument( context.version(), context.seqID(), - context.sourceToParse().id(), + context.id(), source.routing(), context.reorderParentAndGetDocs(), context.sourceToParse().source(), @@ -339,7 +339,8 @@ private static DocumentParserContext nestedContext(DocumentParserContext context if (idField != null) { // We just need to store the id as indexed field, so that IndexWriter#deleteDocuments(term) can then // delete it when the root document is deleted too. - nestedDoc.add(new Field(IdFieldMapper.NAME, idField.binaryValue(), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + // NOTE: we don't support nested fields in tsdb so it's safe to assume the standard id mapper. + nestedDoc.add(new Field(IdFieldMapper.NAME, idField.binaryValue(), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); } else { throw new IllegalStateException("The root document of a nested document should have an _id field"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 9f234598f5e76..525103616fc2a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -90,6 +90,7 @@ protected void addDoc(LuceneDocument doc) { private final Map dynamicObjectMappers; private final List dynamicRuntimeFields; private final DocumentDimensions dimensions; + private String id; private Field version; private SeqNoFieldMapper.SequenceIDFields seqID; @@ -104,6 +105,7 @@ private DocumentParserContext(DocumentParserContext in) { this.newFieldsSeen = in.newFieldsSeen; this.dynamicObjectMappers = in.dynamicObjectMappers; this.dynamicRuntimeFields = in.dynamicRuntimeFields; + this.id = in.id; this.version = in.version; this.seqID = in.seqID; this.dimensions = in.dimensions; @@ -192,6 +194,18 @@ public final void version(Field version) { this.version = version; } + public final String id() { + if (id == null) { + assert false : "id field mapper has not set the id"; + throw new IllegalStateException("id field mapper has not set the id"); + } + return id; + } + + public final void id(String id) { + this.id = id; + } + public final SeqNoFieldMapper.SequenceIDFields seqID() { return this.seqID; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index af50a3fabff4a..d7a1d693b0571 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -9,277 +9,33 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TermInSetQuery; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.logging.DeprecationCategory; -import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.fielddata.FieldData; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.fielddata.LeafFieldData; -import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; -import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; -import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.script.field.DelegateDocValuesField; -import org.elasticsearch.script.field.DocValuesField; -import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.MultiValueMode; -import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.search.sort.BucketedSort; -import org.elasticsearch.search.sort.SortOrder; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.function.BooleanSupplier; -import java.util.function.Supplier; +import org.elasticsearch.index.analysis.NamedAnalyzer; /** - * A mapper for the _id field. It does nothing since _id is neither indexed nor - * stored, but we need to keep it so that its FieldType can be used to generate - * queries. + * A mapper for the _id field. */ -public class IdFieldMapper extends MetadataFieldMapper { - private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(IdFieldMapper.class); - static final String ID_FIELD_DATA_DEPRECATION_MESSAGE = - "Loading the fielddata on the _id field is deprecated and will be removed in future versions. " - + "If you require sorting or aggregating on this field you should also include the id in the " - + "body of your documents, and map this field as a keyword field that has [doc_values] enabled"; - +public abstract class IdFieldMapper extends MetadataFieldMapper { public static final String NAME = "_id"; public static final String CONTENT_TYPE = "_id"; - public static class Defaults { - - public static final FieldType FIELD_TYPE = new FieldType(); - public static final FieldType NESTED_FIELD_TYPE; - - static { - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - FIELD_TYPE.setStored(true); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.freeze(); - - NESTED_FIELD_TYPE = new FieldType(); - NESTED_FIELD_TYPE.setTokenized(false); - NESTED_FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - NESTED_FIELD_TYPE.setStored(false); - NESTED_FIELD_TYPE.setOmitNorms(true); - NESTED_FIELD_TYPE.freeze(); - } - } - - public static final IdFieldMapper NO_FIELD_DATA = new IdFieldMapper(() -> false); - public static final TypeParser PARSER = new FixedTypeParser(MappingParserContext::idFieldMapper); - static final class IdFieldType extends TermBasedFieldType { - - private final BooleanSupplier fieldDataEnabled; - - IdFieldType(BooleanSupplier fieldDataEnabled) { - super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); - this.fieldDataEnabled = fieldDataEnabled; - assert isSearchable(); - } - - @Override - public String typeName() { - return CONTENT_TYPE; - } - - @Override - public boolean isSearchable() { - // The _id field is always searchable. - return true; - } - - @Override - public boolean mayExistInIndex(SearchExecutionContext context) { - return true; - } - - @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return new StoredValueFetcher(context.lookup(), NAME); - } - - @Override - public Query termQuery(Object value, SearchExecutionContext context) { - return termsQuery(Arrays.asList(value), context); - } - - @Override - public Query existsQuery(SearchExecutionContext context) { - return new MatchAllDocsQuery(); - } - - @Override - public Query termsQuery(Collection values, SearchExecutionContext context) { - failIfNotIndexed(); - BytesRef[] bytesRefs = values.stream().map(v -> { - Object idObject = v; - if (idObject instanceof BytesRef) { - idObject = ((BytesRef) idObject).utf8ToString(); - } - return Uid.encodeId(idObject.toString()); - }).toArray(BytesRef[]::new); - return new TermInSetQuery(name(), bytesRefs); - } - - @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - if (fieldDataEnabled.getAsBoolean() == false) { - throw new IllegalArgumentException( - "Fielddata access on the _id field is disallowed, " - + "you can re-enable it by updating the dynamic cluster setting: " - + IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey() - ); - } - final IndexFieldData.Builder fieldDataBuilder = new PagedBytesIndexFieldData.Builder( - name(), - TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY, - TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY, - TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE, - CoreValuesSourceType.KEYWORD, - (dv, n) -> new DelegateDocValuesField( - new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))), - n - ) - ); - return new IndexFieldData.Builder() { - @Override - public IndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { - deprecationLogger.warn(DeprecationCategory.AGGREGATIONS, "id_field_data", ID_FIELD_DATA_DEPRECATION_MESSAGE); - final IndexFieldData fieldData = fieldDataBuilder.build(cache, breakerService); - return new IndexFieldData<>() { - @Override - public String getFieldName() { - return fieldData.getFieldName(); - } - - @Override - public ValuesSourceType getValuesSourceType() { - return fieldData.getValuesSourceType(); - } - - @Override - public LeafFieldData load(LeafReaderContext context) { - return wrap(fieldData.load(context)); - } - - @Override - public LeafFieldData loadDirect(LeafReaderContext context) throws Exception { - return wrap(fieldData.loadDirect(context)); - } - - @Override - public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) { - XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); - return new SortField(getFieldName(), source, reverse); - } - - @Override - public BucketedSort newBucketedSort( - BigArrays bigArrays, - Object missingValue, - MultiValueMode sortMode, - Nested nested, - SortOrder sortOrder, - DocValueFormat format, - int bucketSize, - BucketedSort.ExtraData extra - ) { - throw new UnsupportedOperationException("can't sort on the [" + CONTENT_TYPE + "] field"); - } - }; - } - }; - } - } - - private static LeafFieldData wrap(LeafFieldData in) { - return new LeafFieldData() { - - @Override - public void close() { - in.close(); - } - - @Override - public long ramBytesUsed() { - return in.ramBytesUsed(); - } - - @Override - public DocValuesField getScriptField(String name) { - return new DelegateDocValuesField(new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(getBytesValues())), name); - } - - @Override - public SortedBinaryDocValues getBytesValues() { - SortedBinaryDocValues inValues = in.getBytesValues(); - return new SortedBinaryDocValues() { - - @Override - public BytesRef nextValue() throws IOException { - BytesRef encoded = inValues.nextValue(); - return new BytesRef( - Uid.decodeId(Arrays.copyOfRange(encoded.bytes, encoded.offset, encoded.offset + encoded.length)) - ); - } - - @Override - public int docValueCount() { - final int count = inValues.docValueCount(); - // If the count is not 1 then the impl is not correct as the binary representation - // does not preserve order. But id fields only have one value per doc so we are good. - assert count == 1; - return inValues.docValueCount(); - } - - @Override - public boolean advanceExact(int doc) throws IOException { - return inValues.advanceExact(doc); - } - }; - } - }; - } - - public IdFieldMapper(BooleanSupplier fieldDataEnabled) { - super(new IdFieldType(fieldDataEnabled), Lucene.KEYWORD_ANALYZER); + protected IdFieldMapper(MappedFieldType mappedFieldType, NamedAnalyzer indexAnalyzer) { + super(mappedFieldType, indexAnalyzer); + assert mappedFieldType.isSearchable(); } @Override - public void preParse(DocumentParserContext context) { - context.doc().add(idField(context.sourceToParse().id())); - } - - public static Field idField(String id) { - return new Field(NAME, Uid.encodeId(id), Defaults.FIELD_TYPE); + protected final String contentType() { + return CONTENT_TYPE; } - @Override - protected String contentType() { - return CONTENT_TYPE; + /** + * Create a {@link Field} to store the provided {@code _id} that "stores" + * the {@code _id} so it can be fetched easily from the index. + */ + public static Field standardIdField(String id) { + return new Field(NAME, Uid.encodeId(id), ProvidedIdFieldMapper.Defaults.FIELD_TYPE); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LuceneDocument.java b/server/src/main/java/org/elasticsearch/index/mapper/LuceneDocument.java index 22b5d8bfc8ffa..e86f47febc6dc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LuceneDocument.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LuceneDocument.java @@ -144,5 +144,4 @@ public Number getNumericValue(String name) { } return null; } - } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java b/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java index 4c4d1a11552a8..ae8343b4f2ebd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java @@ -74,7 +74,7 @@ public static ParsedDocument deleteTombstone(String id) { seqIdFields.addFields(document); Field versionField = VersionFieldMapper.versionField(); document.add(versionField); - document.add(IdFieldMapper.idField(id)); + document.add(IdFieldMapper.standardIdField(id)); return new ParsedDocument( versionField, seqIdFields, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java new file mode 100644 index 0000000000000..dc4b4a84550b9 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java @@ -0,0 +1,272 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermInSetQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.LeafFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; +import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.script.field.DelegateDocValuesField; +import org.elasticsearch.script.field.DocValuesField; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.function.BooleanSupplier; +import java.util.function.Supplier; + +/** + * A mapper for the _id field. It does nothing since _id is neither indexed nor + * stored, but we need to keep it so that its FieldType can be used to generate + * queries. + */ +public class ProvidedIdFieldMapper extends IdFieldMapper { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ProvidedIdFieldMapper.class); + static final String ID_FIELD_DATA_DEPRECATION_MESSAGE = + "Loading the fielddata on the _id field is deprecated and will be removed in future versions. " + + "If you require sorting or aggregating on this field you should also include the id in the " + + "body of your documents, and map this field as a keyword field that has [doc_values] enabled"; + + public static class Defaults { + + public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType NESTED_FIELD_TYPE; + + static { + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.setStored(true); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.freeze(); + + NESTED_FIELD_TYPE = new FieldType(); + NESTED_FIELD_TYPE.setTokenized(false); + NESTED_FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + NESTED_FIELD_TYPE.setStored(false); + NESTED_FIELD_TYPE.setOmitNorms(true); + NESTED_FIELD_TYPE.freeze(); + } + } + + public static final ProvidedIdFieldMapper NO_FIELD_DATA = new ProvidedIdFieldMapper(() -> false); + + static final class IdFieldType extends TermBasedFieldType { + + private final BooleanSupplier fieldDataEnabled; + + IdFieldType(BooleanSupplier fieldDataEnabled) { + super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + this.fieldDataEnabled = fieldDataEnabled; + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public boolean isSearchable() { + // The _id field is always searchable. + return true; + } + + @Override + public boolean mayExistInIndex(SearchExecutionContext context) { + return true; + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + return new StoredValueFetcher(context.lookup(), NAME); + } + + @Override + public Query termQuery(Object value, SearchExecutionContext context) { + return termsQuery(Arrays.asList(value), context); + } + + @Override + public Query existsQuery(SearchExecutionContext context) { + return new MatchAllDocsQuery(); + } + + @Override + public Query termsQuery(Collection values, SearchExecutionContext context) { + failIfNotIndexed(); + BytesRef[] bytesRefs = values.stream().map(v -> { + Object idObject = v; + if (idObject instanceof BytesRef) { + idObject = ((BytesRef) idObject).utf8ToString(); + } + return Uid.encodeId(idObject.toString()); + }).toArray(BytesRef[]::new); + return new TermInSetQuery(name(), bytesRefs); + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + if (fieldDataEnabled.getAsBoolean() == false) { + throw new IllegalArgumentException( + "Fielddata access on the _id field is disallowed, " + + "you can re-enable it by updating the dynamic cluster setting: " + + IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey() + ); + } + final IndexFieldData.Builder fieldDataBuilder = new PagedBytesIndexFieldData.Builder( + name(), + TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY, + TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY, + TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE, + CoreValuesSourceType.KEYWORD, + (dv, n) -> new DelegateDocValuesField( + new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))), + n + ) + ); + return new IndexFieldData.Builder() { + @Override + public IndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { + deprecationLogger.warn(DeprecationCategory.AGGREGATIONS, "id_field_data", ID_FIELD_DATA_DEPRECATION_MESSAGE); + final IndexFieldData fieldData = fieldDataBuilder.build(cache, breakerService); + return new IndexFieldData<>() { + @Override + public String getFieldName() { + return fieldData.getFieldName(); + } + + @Override + public ValuesSourceType getValuesSourceType() { + return fieldData.getValuesSourceType(); + } + + @Override + public LeafFieldData load(LeafReaderContext context) { + return wrap(fieldData.load(context)); + } + + @Override + public LeafFieldData loadDirect(LeafReaderContext context) throws Exception { + return wrap(fieldData.loadDirect(context)); + } + + @Override + public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) { + XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); + return new SortField(getFieldName(), source, reverse); + } + + @Override + public BucketedSort newBucketedSort( + BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra + ) { + throw new UnsupportedOperationException("can't sort on the [" + CONTENT_TYPE + "] field"); + } + }; + } + }; + } + } + + private static LeafFieldData wrap(LeafFieldData in) { + return new LeafFieldData() { + + @Override + public void close() { + in.close(); + } + + @Override + public long ramBytesUsed() { + return in.ramBytesUsed(); + } + + @Override + public DocValuesField getScriptField(String name) { + return new DelegateDocValuesField(new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(getBytesValues())), name); + } + + @Override + public SortedBinaryDocValues getBytesValues() { + SortedBinaryDocValues inValues = in.getBytesValues(); + return new SortedBinaryDocValues() { + + @Override + public BytesRef nextValue() throws IOException { + BytesRef encoded = inValues.nextValue(); + return new BytesRef( + Uid.decodeId(Arrays.copyOfRange(encoded.bytes, encoded.offset, encoded.offset + encoded.length)) + ); + } + + @Override + public int docValueCount() { + final int count = inValues.docValueCount(); + // If the count is not 1 then the impl is not correct as the binary representation + // does not preserve order. But id fields only have one value per doc so we are good. + assert count == 1; + return inValues.docValueCount(); + } + + @Override + public boolean advanceExact(int doc) throws IOException { + return inValues.advanceExact(doc); + } + }; + } + }; + } + + public ProvidedIdFieldMapper(BooleanSupplier fieldDataEnabled) { + super(new IdFieldType(fieldDataEnabled), Lucene.KEYWORD_ANALYZER); + } + + @Override + public void preParse(DocumentParserContext context) { + if (context.sourceToParse().id() == null) { + throw new IllegalStateException("_id should have been set on the coordinating node"); + } + context.id(context.sourceToParse().id()); + context.doc().add(standardIdField(context.id())); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java index 0004c4e16dcdb..f3994f1ae7570 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java @@ -29,13 +29,13 @@ public class SourceToParse { private final Map dynamicTemplates; public SourceToParse( - String id, + @Nullable String id, BytesReference source, XContentType xContentType, @Nullable String routing, Map dynamicTemplates ) { - this.id = Objects.requireNonNull(id); + this.id = id; // we always convert back to byte array, since we store it and Field only supports bytes.. // so, we might as well do it here, and improve the performance of working with direct byte arrays this.source = new BytesArray(Objects.requireNonNull(source).toBytesRef()); @@ -52,7 +52,7 @@ public BytesReference source() { return this.source; } - public String id() { + public String id() { // TODO migrate callers that use this to describe the document to a new method return this.id; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 136a297d8b1f5..ae6bbd1a49718 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -144,7 +144,9 @@ public void postParse(DocumentParserContext context) throws IOException { assert fieldType().isIndexed() == false; TimeSeriesIdBuilder timeSeriesIdBuilder = (TimeSeriesIdBuilder) context.getDimensions(); - context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesIdBuilder.build().toBytesRef())); + BytesRef timeSeriesId = timeSeriesIdBuilder.build().toBytesRef(); + context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); + TsidExtractingIdFieldMapper.INSTANCE.createField(context, timeSeriesId); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java new file mode 100644 index 0000000000000..490e8f742c80f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermInSetQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.routing.IndexRouting; +import org.elasticsearch.common.hash.MurmurHash3; +import org.elasticsearch.common.hash.MurmurHash3.Hash128; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.util.ByteUtils; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Locale; +import java.util.function.Supplier; + +/** + * A mapper for the _id field. It does nothing since _id is neither indexed nor + * stored, but we need to keep it so that its FieldType can be used to generate + * queries. + */ +public class TsidExtractingIdFieldMapper extends IdFieldMapper { + private static final FieldType FIELD_TYPE = new FieldType(); + static { + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.setStored(true); // TODO reconstruct the id on fetch from tsid and timestamp + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.freeze(); + } + + public static final TsidExtractingIdFieldMapper INSTANCE = new TsidExtractingIdFieldMapper(); + + public static final TypeParser PARSER = new FixedTypeParser(MappingParserContext::idFieldMapper); + + static final class IdFieldType extends TermBasedFieldType { + IdFieldType() { + super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public boolean isSearchable() { + // The _id field is always searchable. + return true; + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + return new StoredValueFetcher(context.lookup(), NAME); + } + + @Override + public Query termQuery(Object value, SearchExecutionContext context) { + return termsQuery(Arrays.asList(value), context); + } + + @Override + public Query existsQuery(SearchExecutionContext context) { + return new MatchAllDocsQuery(); + } + + @Override + public Query termsQuery(Collection values, SearchExecutionContext context) { + failIfNotIndexed(); + BytesRef[] bytesRefs = values.stream().map(v -> { + Object idObject = v; + if (idObject instanceof BytesRef) { + idObject = ((BytesRef) idObject).utf8ToString(); + } + return Uid.encodeId(idObject.toString()); + }).toArray(BytesRef[]::new); + return new TermInSetQuery(name(), bytesRefs); + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + throw new IllegalArgumentException("Fielddata is not supported on [_id] field in [time_series] indices"); + } + } + + private TsidExtractingIdFieldMapper() { + super(new IdFieldType(), Lucene.KEYWORD_ANALYZER); + } + + private static final long SEED = 0; + + public void createField(DocumentParserContext context, BytesRef tsid) { + IndexableField[] timestampFields = context.rootDoc().getFields(DataStreamTimestampFieldMapper.DEFAULT_PATH); + if (timestampFields.length == 0) { + throw new IllegalArgumentException( + "data stream timestamp field [" + DataStreamTimestampFieldMapper.DEFAULT_PATH + "] is missing" + ); + } + long timestamp = timestampFields[0].numericValue().longValue(); + + Hash128 hash = new Hash128(); + MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, SEED, hash); + + byte[] suffix = new byte[16]; + ByteUtils.writeLongLE(hash.h1, suffix, 0); + ByteUtils.writeLongLE(timestamp, suffix, 8); // TODO compare disk usage for LE and BE on timestamp + + IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting(); + // TODO it'd be way faster to use the fields that we've extract here rather than the source or parse the tsid + String id = indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix); + assert Uid.isURLBase64WithoutPadding(id); // Make sure we get to use Uid's nice optimizations + /* + * Make sure that _id from extracting the tsid matches that _id + * from extracting the _source. This should be true for all valid + * documents with valid mappings. *But* some invalid mappings + * will not parse the field but be rejected later by the dynamic + * mappings machinery. So if there are any dynamic mappings + * at all we just skip the assertion because we can't be sure + * it always must pass. + */ + assert context.getDynamicMappers().isEmpty() == false + || context.getDynamicRuntimeFields().isEmpty() == false + || id.equals(indexRouting.createId(TimeSeriesIdFieldMapper.decodeTsid(tsid), suffix)); + + if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "_id must be unset or set to [%s] but was [%s] because [%s] is in time_series mode", + id, + context.sourceToParse().id(), + context.indexSettings().getIndexMetadata().getIndex().getName() + ) + ); + } + context.id(id); + + BytesRef uidEncoded = Uid.encodeId(context.id()); + context.doc().add(new Field(NAME, uidEncoded, FIELD_TYPE)); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 4913a6ba32965..12fbcd18e6f1b 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -959,7 +959,7 @@ private Engine.IndexResult applyIndexOperation( ); Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); if (update != null) { - return new Engine.IndexResult(update); + return new Engine.IndexResult(update, operation.parsedDoc().id()); } } catch (Exception e) { // We treat any exception during parsing and or mapping update as a document level failure @@ -967,7 +967,7 @@ private Engine.IndexResult applyIndexOperation( // can not raise an exception that may block any replication of previous operations to the // replicas verifyNotClosed(e); - return new Engine.IndexResult(e, version, opPrimaryTerm, seqNo); + return new Engine.IndexResult(e, version, opPrimaryTerm, seqNo, sourceToParse.id()); } return index(engine, operation); @@ -1097,12 +1097,12 @@ private Engine.NoOpResult noOp(Engine engine, Engine.NoOp noOp) throws IOExcepti return engine.noOp(noOp); } - public Engine.IndexResult getFailedIndexResult(Exception e, long version) { - return new Engine.IndexResult(e, version); + public Engine.IndexResult getFailedIndexResult(Exception e, long version, String id) { + return new Engine.IndexResult(e, version, id); } - public Engine.DeleteResult getFailedDeleteResult(Exception e, long version) { - return new Engine.DeleteResult(e, version, getOperationPrimaryTerm()); + public Engine.DeleteResult getFailedDeleteResult(Exception e, long version, String id) { + return new Engine.DeleteResult(e, version, getOperationPrimaryTerm(), id); } public Engine.DeleteResult applyDeleteOperationOnPrimary( diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 519dcdc580e15..3f9d2db350b52 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -73,6 +73,7 @@ import org.elasticsearch.gateway.MetaStateService; import org.elasticsearch.gateway.MetadataStateFormat; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexService; @@ -142,6 +143,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.EnumMap; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -237,7 +239,7 @@ public class IndicesService extends AbstractLifecycleComponent private volatile boolean idFieldDataEnabled; private volatile boolean allowExpensiveQueries; - private final IdFieldMapper idFieldMapper = new IdFieldMapper(() -> idFieldDataEnabled); + private final Function idFieldMappers; @Nullable private final EsThreadPoolExecutor danglingIndicesThreadPoolExecutor; @@ -359,6 +361,12 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon } }); + Map idFieldMappers = new EnumMap<>(IndexMode.class); + for (IndexMode mode : IndexMode.values()) { + idFieldMappers.put(mode, mode.buildIdFieldMapper(() -> idFieldDataEnabled)); + } + this.idFieldMappers = idFieldMappers::get; + final String nodeName = Objects.requireNonNull(Node.NODE_NAME_SETTING.get(settings)); nodeWriteDanglingIndicesInfo = WRITE_DANGLING_INDICES_INFO_SETTING.get(settings); danglingIndicesThreadPoolExecutor = nodeWriteDanglingIndicesInfo @@ -717,7 +725,7 @@ private synchronized IndexService createIndexService( mapperRegistry, indicesFieldDataCache, namedWriteableRegistry, - idFieldMapper, + idFieldMappers.apply(idxSettings.getMode()), valuesSourceRegistry, indexFoldersDeletionListeners, snapshotCommitSuppliers diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContextTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContextTests.java index bb3f1a628c0e3..630dd8d9aa7c0 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContextTests.java @@ -48,7 +48,9 @@ public void testAbortedSkipped() { visitedRequests.add(context.getCurrent()); context.setRequestToExecute(context.getCurrent()); // using failures prevents caring about types - context.markOperationAsExecuted(new Engine.IndexResult(new ElasticsearchException("bla"), 1)); + context.markOperationAsExecuted( + new Engine.IndexResult(new ElasticsearchException("bla"), 1, context.getRequestToExecute().id()) + ); context.markAsCompleted(context.getExecutionResult()); } @@ -97,25 +99,25 @@ public void testTranslogLocation() { case INDEX, CREATE -> { context.setRequestToExecute(current); if (failure) { - result = new Engine.IndexResult(new ElasticsearchException("bla"), 1); + result = new Engine.IndexResult(new ElasticsearchException("bla"), 1, current.id()); } else { - result = new FakeIndexResult(1, 1, randomLongBetween(0, 200), randomBoolean(), location); + result = new FakeIndexResult(1, 1, randomLongBetween(0, 200), randomBoolean(), location, "id"); } } case UPDATE -> { context.setRequestToExecute(new IndexRequest(current.index()).id(current.id())); if (failure) { - result = new Engine.IndexResult(new ElasticsearchException("bla"), 1, 1, 1); + result = new Engine.IndexResult(new ElasticsearchException("bla"), 1, 1, 1, current.id()); } else { - result = new FakeIndexResult(1, 1, randomLongBetween(0, 200), randomBoolean(), location); + result = new FakeIndexResult(1, 1, randomLongBetween(0, 200), randomBoolean(), location, "id"); } } case DELETE -> { context.setRequestToExecute(current); if (failure) { - result = new Engine.DeleteResult(new ElasticsearchException("bla"), 1, 1); + result = new Engine.DeleteResult(new ElasticsearchException("bla"), 1, 1, current.id()); } else { - result = new FakeDeleteResult(1, 1, randomLongBetween(0, 200), randomBoolean(), location); + result = new FakeDeleteResult(1, 1, randomLongBetween(0, 200), randomBoolean(), location, current.id()); } } default -> throw new AssertionError("unknown type:" + current.opType()); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index fa542758d63f1..44c15b9cbb61b 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -769,7 +769,7 @@ public void testIngestCallbackExceptionHandled() throws Exception { any(), eq(Names.WRITE) ); - indexRequest1.process(); + indexRequest1.autoGenerateId(); completionHandler.getValue().accept(Thread.currentThread(), null); // check failure passed through to the listener diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java index f2a8f0f107136..7a5b6079349a0 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java @@ -67,6 +67,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -254,10 +255,11 @@ public void testExecuteBulkIndexRequestWithMappingUpdates() throws Exception { BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items); Engine.IndexResult mappingUpdate = new Engine.IndexResult( - new Mapping(mock(RootObjectMapper.class), new MetadataFieldMapper[0], Collections.emptyMap()) + new Mapping(mock(RootObjectMapper.class), new MetadataFieldMapper[0], Collections.emptyMap()), + "id" ); Translog.Location resultLocation = new Translog.Location(42, 42, 42); - Engine.IndexResult success = new FakeIndexResult(1, 1, 13, true, resultLocation); + Engine.IndexResult success = new FakeIndexResult(1, 1, 13, true, resultLocation, "id"); IndexShard shard = mock(IndexShard.class); when(shard.shardId()).thenReturn(shardId); @@ -523,7 +525,7 @@ public void testUpdateRequestWithFailure() throws Exception { IndexRequest updateResponse = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"); Exception err = new ElasticsearchException("I'm dead <(x.x)>"); - Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0, 0); + Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0, 0, "id"); IndexShard shard = mock(IndexShard.class); when(shard.applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyLong(), anyLong(), anyBoolean())).thenReturn( indexResult @@ -580,7 +582,7 @@ public void testUpdateRequestWithConflictFailure() throws Exception { IndexRequest updateResponse = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"); Exception err = new VersionConflictEngineException(shardId, "id", "I'm conflicted <(;_;)>"); - Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0, 0); + Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0, 0, "id"); IndexShard shard = mock(IndexShard.class); when(shard.applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyLong(), anyLong(), anyBoolean())).thenReturn( indexResult @@ -636,7 +638,7 @@ public void testUpdateRequestWithSuccess() throws Exception { boolean created = randomBoolean(); Translog.Location resultLocation = new Translog.Location(42, 42, 42); - Engine.IndexResult indexResult = new FakeIndexResult(1, 1, 13, created, resultLocation); + Engine.IndexResult indexResult = new FakeIndexResult(1, 1, 13, created, resultLocation, "id"); IndexShard shard = mock(IndexShard.class); when(shard.applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyLong(), anyLong(), anyBoolean())).thenReturn( indexResult @@ -694,7 +696,7 @@ public void testUpdateWithDelete() throws Exception { boolean found = randomBoolean(); Translog.Location resultLocation = new Translog.Location(42, 42, 42); final long resultSeqNo = 13; - Engine.DeleteResult deleteResult = new FakeDeleteResult(1, 1, resultSeqNo, found, resultLocation); + Engine.DeleteResult deleteResult = new FakeDeleteResult(1, 1, resultSeqNo, found, resultLocation, "id"); IndexShard shard = mock(IndexShard.class); when(shard.applyDeleteOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyLong())).thenReturn(deleteResult); when(shard.indexSettings()).thenReturn(indexSettings); @@ -848,12 +850,13 @@ public void testRetries() throws Exception { IndexRequest updateResponse = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"); Exception err = new VersionConflictEngineException(shardId, "id", "I'm conflicted <(;_;)>"); - Engine.IndexResult conflictedResult = new Engine.IndexResult(err, 0); + Engine.IndexResult conflictedResult = new Engine.IndexResult(err, 0, "id"); Engine.IndexResult mappingUpdate = new Engine.IndexResult( - new Mapping(mock(RootObjectMapper.class), new MetadataFieldMapper[0], Collections.emptyMap()) + new Mapping(mock(RootObjectMapper.class), new MetadataFieldMapper[0], Collections.emptyMap()), + "id" ); Translog.Location resultLocation = new Translog.Location(42, 42, 42); - Engine.IndexResult success = new FakeIndexResult(1, 1, 13, true, resultLocation); + Engine.IndexResult success = new FakeIndexResult(1, 1, 13, true, resultLocation, "id"); IndexShard shard = mock(IndexShard.class); when(shard.applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyLong(), anyLong(), anyBoolean())).thenAnswer(ir -> { @@ -941,12 +944,13 @@ public void testForceExecutionOnRejectionAfterMappingUpdate() throws Exception { BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items); Engine.IndexResult mappingUpdate = new Engine.IndexResult( - new Mapping(mock(RootObjectMapper.class), new MetadataFieldMapper[0], Collections.emptyMap()) + new Mapping(mock(RootObjectMapper.class), new MetadataFieldMapper[0], Collections.emptyMap()), + "id" ); Translog.Location resultLocation1 = new Translog.Location(42, 36, 36); Translog.Location resultLocation2 = new Translog.Location(42, 42, 42); - Engine.IndexResult success1 = new FakeIndexResult(1, 1, 10, true, resultLocation1); - Engine.IndexResult success2 = new FakeIndexResult(1, 1, 13, true, resultLocation2); + Engine.IndexResult success1 = new FakeIndexResult(1, 1, 10, true, resultLocation1, "id"); + Engine.IndexResult success2 = new FakeIndexResult(1, 1, 13, true, resultLocation2, "id"); IndexShard shard = mock(IndexShard.class); when(shard.shardId()).thenReturn(shardId); @@ -955,7 +959,7 @@ public void testForceExecutionOnRejectionAfterMappingUpdate() throws Exception { mappingUpdate, success2 ); - when(shard.getFailedIndexResult(any(EsRejectedExecutionException.class), anyLong())).thenCallRealMethod(); + when(shard.getFailedIndexResult(any(EsRejectedExecutionException.class), anyLong(), anyString())).thenCallRealMethod(); when(shard.mapperService()).thenReturn(mock(MapperService.class)); randomlySetIgnoredPrimaryResponse(items[0]); @@ -1084,8 +1088,8 @@ static class FakeIndexResult extends Engine.IndexResult { private final Translog.Location location; - protected FakeIndexResult(long version, long term, long seqNo, boolean created, Translog.Location location) { - super(version, term, seqNo, created); + protected FakeIndexResult(long version, long term, long seqNo, boolean created, Translog.Location location, String id) { + super(version, term, seqNo, created, id); this.location = location; } @@ -1102,8 +1106,8 @@ static class FakeDeleteResult extends Engine.DeleteResult { private final Translog.Location location; - protected FakeDeleteResult(long version, long term, long seqNo, boolean found, Translog.Location location) { - super(version, term, seqNo, found); + protected FakeDeleteResult(long version, long term, long seqNo, boolean found, Translog.Location location, String id) { + super(version, term, seqNo, found, id); this.location = location; } diff --git a/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java index 787f8383e11e3..f43e9b9f12f09 100644 --- a/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java @@ -118,13 +118,10 @@ public void testWaitForActiveShards() { expectThrows(IllegalArgumentException.class, () -> request.waitForActiveShards(ActiveShardCount.from(randomIntBetween(-10, -1)))); } - public void testAutoGenIdTimestampIsSet() { + public void testAutoGenerateId() { IndexRequest request = new IndexRequest("index"); - request.process(); + request.autoGenerateId(); assertTrue("expected > 0 but got: " + request.getAutoGeneratedTimestamp(), request.getAutoGeneratedTimestamp() > 0); - request = new IndexRequest("index").id("1"); - request.process(); - assertEquals(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, request.getAutoGeneratedTimestamp()); } public void testIndexResponse() { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 5509ee5cc0d4d..c40283c128486 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -9,11 +9,14 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.RoutingMissingException; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -35,8 +38,41 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; public class IndexRoutingTests extends ESTestCase { + public void testSimpleRoutingRejectsEmptyId() { + IndexRouting indexRouting = IndexRouting.fromIndexMetadata( + IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1).build() + ); + IndexRequest req = new IndexRequest().id(""); + Exception e = expectThrows(IllegalArgumentException.class, () -> indexRouting.process(req)); + assertThat(e.getMessage(), equalTo("if _id is specified it must not be empty")); + } + + public void testSimpleRoutingAcceptsId() { + IndexRouting indexRouting = IndexRouting.fromIndexMetadata( + IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1).build() + ); + String id = randomAlphaOfLength(10); + IndexRequest req = new IndexRequest().id(id); + indexRouting.process(req); + assertThat(req.id(), equalTo(id)); + assertThat(req.getAutoGeneratedTimestamp(), equalTo(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP)); + } + + public void testSimpleRoutingAssignedRandomId() { + IndexRouting indexRouting = IndexRouting.fromIndexMetadata( + IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1).build() + ); + IndexRequest req = new IndexRequest(); + indexRouting.process(req); + req.checkAutoIdWithOpTypeCreateSupportedByVersion(null); + assertThat(req.id(), not(nullValue())); + assertThat(req.getAutoGeneratedTimestamp(), not(equalTo(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP))); + } + public void testGenerateShardId() { int[][] possibleValues = new int[][] { { 8, 4, 2 }, { 20, 10, 2 }, { 36, 12, 3 }, { 15, 5, 1 } }; for (int i = 0; i < 10; i++) { @@ -129,7 +165,10 @@ public void testCollectSearchShardsInStandardIndex() { } public void testPartitionedIndex() { - // make sure the same routing value always has each _id fall within the configured partition size + /* + * make sure the same routing value always has each _id fall within the + * configured partition size + */ for (int shards = 1; shards < 5; shards++) { for (int partitionSize = 1; partitionSize == 1 || partitionSize < shards; partitionSize++) { IndexRouting indexRouting = IndexRouting.fromIndexMetadata( @@ -414,8 +453,8 @@ public void testRequiredRouting() { /** * Extract a shardId from a "simple" {@link IndexRouting} using a randomly - * chosen method. All of the random methods should return - * the same results. + * chosen method. All of the random methods should return the + * same results. */ private int shardIdFromSimple(IndexRouting indexRouting, String id, @Nullable String routing) { return switch (between(0, 3)) { @@ -427,16 +466,25 @@ private int shardIdFromSimple(IndexRouting indexRouting, String id, @Nullable St }; } - public void testRoutingPathSpecifiedRouting() throws IOException { - IndexRouting routing = indexRoutingForPath(between(1, 5), randomAlphaOfLength(5)); - Exception e = expectThrows( - IllegalArgumentException.class, - () -> routing.indexShard(null, randomAlphaOfLength(5), XContentType.JSON, source(Map.of())) - ); - assertThat( - e.getMessage(), - equalTo("indexing with a specified routing is not supported because the destination index [test] is in time series mode") - ); + public void testRoutingAllowsId() { + IndexRouting indexRouting = indexRoutingForPath(between(1, 5), randomAlphaOfLength(5)); + String id = randomAlphaOfLength(5); + IndexRequest req = new IndexRequest().id(id); + indexRouting.process(req); + assertThat(req.id(), equalTo(id)); + } + + /** + * {@code routing_path} based {@link IndexRouting} implementations do + * not assign an {@code _id} on the coordinating node, instead they + * rely on the {@link IdFieldMapper} implementation to assign the + * {@code _id} as part of parsing the document. + */ + public void testRoutingPathLeavesIdNull() { + IndexRouting indexRouting = indexRoutingForPath(between(1, 5), randomAlphaOfLength(5)); + IndexRequest req = new IndexRequest(); + indexRouting.process(req); + assertThat(req.id(), nullValue()); } public void testRoutingPathEmptySource() throws IOException { @@ -466,22 +514,19 @@ public void testRoutingPathUpdate() throws IOException { assertThat(e.getMessage(), equalTo("update is not supported because the destination index [test] is in time series mode")); } - public void testRoutingPathDelete() throws IOException { - IndexRouting routing = indexRoutingForPath(between(1, 5), "foo"); + public void testRoutingIndexWithRouting() throws IOException { + IndexRouting indexRouting = indexRoutingForPath(5, "foo"); + String value = randomAlphaOfLength(5); + BytesReference source = source(Map.of("foo", value)); + String docRouting = randomAlphaOfLength(5); Exception e = expectThrows( IllegalArgumentException.class, - () -> routing.deleteShard(randomAlphaOfLength(5), randomBoolean() ? null : randomAlphaOfLength(5)) + () -> indexRouting.indexShard(randomAlphaOfLength(5), docRouting, XContentType.JSON, source) ); - assertThat(e.getMessage(), equalTo("delete is not supported because the destination index [test] is in time series mode")); - } - - public void testRoutingPathGet() throws IOException { - IndexRouting routing = indexRoutingForPath(between(1, 5), "foo"); - Exception e = expectThrows( - IllegalArgumentException.class, - () -> routing.getShard(randomAlphaOfLength(5), randomBoolean() ? null : randomAlphaOfLength(5)) + assertThat( + e.getMessage(), + equalTo("specifying routing is not supported because the destination index [test] is in time series mode") ); - assertThat(e.getMessage(), equalTo("get is not supported because the destination index [test] is in time series mode")); } public void testRoutingPathCollectSearchWithRouting() throws IOException { @@ -555,6 +600,29 @@ public void testRoutingPathBwc() throws IOException { assertIndexShard(routing, Map.of("dim.a", "a"), 4); } + public void testRoutingPathReadWithInvalidString() throws IOException { + int shards = between(2, 1000); + IndexRouting indexRouting = indexRoutingForPath(shards, "foo"); + Exception e = expectThrows(ResourceNotFoundException.class, () -> shardIdForReadFromSourceExtracting(indexRouting, "!@#")); + assertThat(e.getMessage(), equalTo("invalid id [!@#] for index [test] in time series mode")); + } + + public void testRoutingPathReadWithShortString() throws IOException { + int shards = between(2, 1000); + IndexRouting indexRouting = indexRoutingForPath(shards, "foo"); + Exception e = expectThrows(ResourceNotFoundException.class, () -> shardIdForReadFromSourceExtracting(indexRouting, "")); + assertThat(e.getMessage(), equalTo("invalid id [] for index [test] in time series mode")); + } + + /** + * Extract a shardId from an {@link IndexRouting} that extracts routingusing a randomly + * chosen method. All of the random methods should return the + * same results. + */ + private int shardIdForReadFromSourceExtracting(IndexRouting indexRouting, String id) { + return randomBoolean() ? indexRouting.deleteShard(id, null) : indexRouting.getShard(id, null); + } + private IndexRouting indexRoutingForPath(int shards, String path) { return indexRoutingForPath(Version.CURRENT, shards, path); } @@ -569,8 +637,23 @@ private IndexRouting indexRoutingForPath(Version createdVersion, int shards, Str ); } - private void assertIndexShard(IndexRouting routing, Map source, int expected) throws IOException { - assertThat(routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(source)), equalTo(expected)); + private void assertIndexShard(IndexRouting routing, Map source, int expectedShard) throws IOException { + byte[] suffix = randomSuffix(); + BytesReference sourceBytes = source(source); + assertThat(routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, sourceBytes), equalTo(expectedShard)); + IndexRouting.ExtractFromSource r = (IndexRouting.ExtractFromSource) routing; + String idFromSource = r.createId(XContentType.JSON, sourceBytes, suffix); + assertThat(shardIdForReadFromSourceExtracting(routing, idFromSource), equalTo(expectedShard)); + String idFromFlattened = r.createId(flatten(source), suffix); + assertThat(idFromFlattened, equalTo(idFromSource)); + } + + private byte[] randomSuffix() { + byte[] suffix = new byte[between(0, 10)]; + for (int i = 0; i < suffix.length; i++) { + suffix[i] = randomByte(); + } + return suffix; } private BytesReference source(Map doc) throws IOException { @@ -587,6 +670,23 @@ private BytesReference source(Map doc) throws IOException { ); } + private Map flatten(Map m) { + Map result = new HashMap<>(); + flatten(result, null, m); + return result; + } + + private void flatten(Map result, String path, Map m) { + for (Map.Entry e : m.entrySet()) { + String subPath = path == null ? e.getKey().toString() : path + "." + e.getKey(); + if (e.getValue()instanceof Map subM) { + flatten(result, subPath, subM); + } else { + result.put(subPath, e.getValue()); + } + } + } + /** * Build the hash we expect from the extracter. */ diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java index 8f723eced75ca..315712191e264 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.test.ESTestCase; @@ -43,7 +44,7 @@ public void testSimple() throws Exception { .setMergePolicy(NoMergePolicy.INSTANCE) ); Document doc = new Document(); - doc.add(new Field(IdFieldMapper.NAME, "6", IdFieldMapper.Defaults.FIELD_TYPE)); + doc.add(new Field(IdFieldMapper.NAME, "6", ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong())); doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE))); @@ -78,7 +79,7 @@ public void testTwoDocuments() throws Exception { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER).setMergePolicy(NoMergePolicy.INSTANCE)); Document doc = new Document(); - doc.add(new Field(IdFieldMapper.NAME, "6", IdFieldMapper.Defaults.FIELD_TYPE)); + doc.add(new Field(IdFieldMapper.NAME, "6", ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong())); doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE))); diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java index edb41bd5d10b1..d5ebac5e7bb43 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.shard.ShardId; @@ -56,7 +57,7 @@ public void testVersions() throws Exception { assertThat(loadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()), nullValue()); Document doc = new Document(); - doc.add(new Field(IdFieldMapper.NAME, "1", IdFieldMapper.Defaults.FIELD_TYPE)); + doc.add(new Field(IdFieldMapper.NAME, "1", ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 1)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong())); doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE))); @@ -65,7 +66,7 @@ public void testVersions() throws Exception { assertThat(loadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(1L)); doc = new Document(); - Field uid = new Field(IdFieldMapper.NAME, "1", IdFieldMapper.Defaults.FIELD_TYPE); + Field uid = new Field(IdFieldMapper.NAME, "1", ProvidedIdFieldMapper.Defaults.FIELD_TYPE); Field version = new NumericDocValuesField(VersionFieldMapper.NAME, 2); doc.add(uid); doc.add(version); @@ -103,12 +104,12 @@ public void testNestedDocuments() throws IOException { for (int i = 0; i < 4; ++i) { // Nested Document doc = new Document(); - doc.add(new Field(IdFieldMapper.NAME, "1", IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + doc.add(new Field(IdFieldMapper.NAME, "1", ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); docs.add(doc); } // Root Document doc = new Document(); - doc.add(new Field(IdFieldMapper.NAME, "1", IdFieldMapper.Defaults.FIELD_TYPE)); + doc.add(new Field(IdFieldMapper.NAME, "1", ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); NumericDocValuesField version = new NumericDocValuesField(VersionFieldMapper.NAME, 5L); doc.add(version); doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong())); @@ -141,7 +142,7 @@ public void testCache() throws Exception { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); Document doc = new Document(); - doc.add(new Field(IdFieldMapper.NAME, "6", IdFieldMapper.Defaults.FIELD_TYPE)); + doc.add(new Field(IdFieldMapper.NAME, "6", ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong())); doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE))); @@ -168,7 +169,7 @@ public void testCacheFilterReader() throws Exception { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); Document doc = new Document(); - doc.add(new Field(IdFieldMapper.NAME, "6", IdFieldMapper.Defaults.FIELD_TYPE)); + doc.add(new Field(IdFieldMapper.NAME, "6", ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong())); doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE))); diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 004191d374d20..bdf1af996544d 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -52,7 +52,6 @@ import org.elasticsearch.index.engine.InternalEngineFactory; import org.elasticsearch.index.engine.InternalEngineTests; import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.Uid; @@ -195,7 +194,7 @@ private IndexService newIndexService(IndexModule module) throws IOException { mapperRegistry, new IndicesFieldDataCache(settings, listener), writableRegistry(), - IdFieldMapper.NO_FIELD_DATA, + module.indexSettings().getMode().buildNoFieldDataIdFieldMapper(), null, indexDeletionListener, emptyMap() diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index bcee1d74b802a..879592ab31f79 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; -import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; @@ -91,7 +90,7 @@ private CodecService createCodecService() throws IOException { similarityService, mapperRegistry, () -> null, - IdFieldMapper.NO_FIELD_DATA, + settings.getMode().buildNoFieldDataIdFieldMapper(), ScriptCompiler.NONE ); return new CodecService(service); diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index b2ba370c8063f..d20cc142542fb 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -104,6 +104,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.Uid; @@ -1532,7 +1533,7 @@ public void testLookupVersionWithPrunedAwayIds() throws IOException { ) ) { org.apache.lucene.document.Document doc = new org.apache.lucene.document.Document(); - doc.add(new Field(IdFieldMapper.NAME, "1", IdFieldMapper.Defaults.FIELD_TYPE)); + doc.add(new Field(IdFieldMapper.NAME, "1", ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, -1)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, 1)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, 1)); @@ -5469,7 +5470,7 @@ public void testSeqNoGenerator() throws IOException { ) ) { final String id = "id"; - final Field uidField = new Field("_id", id, IdFieldMapper.Defaults.FIELD_TYPE); + final Field uidField = new Field("_id", id, ProvidedIdFieldMapper.Defaults.FIELD_TYPE); final Field versionField = new NumericDocValuesField("_version", 0); final SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); final LuceneDocument document = new LuceneDocument(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index 0a87ce065740d..0cb4b11f72b1a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -287,7 +287,7 @@ public void testEmptyDocumentMapper() { DocumentMapper documentMapper = DocumentMapper.createEmpty(mapperService); assertEquals("{\"_doc\":{}}", Strings.toString(documentMapper.mapping())); assertTrue(documentMapper.mappers().hasMappings()); - assertNotNull(documentMapper.idFieldMapper()); + assertNotNull(documentMapper.mappers().getMapper(IdFieldMapper.NAME)); assertNotNull(documentMapper.sourceMapper()); assertNotNull(documentMapper.IndexFieldMapper()); List> metadataMappers = new ArrayList<>(documentMapper.mappers().getMapping().getMetadataMappersMap().keySet()); @@ -297,10 +297,10 @@ public void testEmptyDocumentMapper() { matchesList().item(DataStreamTimestampFieldMapper.class) .item(DocCountFieldMapper.class) .item(FieldNamesFieldMapper.class) - .item(IdFieldMapper.class) .item(IgnoredFieldMapper.class) .item(IndexFieldMapper.class) .item(NestedPathFieldMapper.class) + .item(ProvidedIdFieldMapper.class) .item(RoutingFieldMapper.class) .item(SeqNoFieldMapper.class) .item(SourceFieldMapper.class) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index d4d6e155237c9..fd74a53ded127 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -468,14 +468,14 @@ public void testNestedHaveIdAndTypeFields() throws Exception { // Nested document: assertNotNull(result.docs().get(0).getField(IdFieldMapper.NAME)); assertEquals(Uid.encodeId("1"), result.docs().get(0).getField(IdFieldMapper.NAME).binaryValue()); - assertEquals(IdFieldMapper.Defaults.NESTED_FIELD_TYPE, result.docs().get(0).getField(IdFieldMapper.NAME).fieldType()); + assertEquals(ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE, result.docs().get(0).getField(IdFieldMapper.NAME).fieldType()); assertNotNull(result.docs().get(0).getField(NestedPathFieldMapper.NAME)); assertEquals("foo", result.docs().get(0).getField(NestedPathFieldMapper.NAME).stringValue()); assertEquals("value1", result.docs().get(0).getField("foo.bar").binaryValue().utf8ToString()); // Root document: assertNotNull(result.docs().get(1).getField(IdFieldMapper.NAME)); assertEquals(Uid.encodeId("1"), result.docs().get(1).getField(IdFieldMapper.NAME).binaryValue()); - assertEquals(IdFieldMapper.Defaults.FIELD_TYPE, result.docs().get(1).getField(IdFieldMapper.NAME).fieldType()); + assertEquals(ProvidedIdFieldMapper.Defaults.FIELD_TYPE, result.docs().get(1).getField(IdFieldMapper.NAME).fieldType()); assertNull(result.docs().get(1).getField(NestedPathFieldMapper.NAME)); assertEquals("value2", result.docs().get(1).getField("baz").binaryValue().utf8ToString()); } @@ -1536,7 +1536,7 @@ public void testParseToJsonAndParse() throws Exception { DocumentMapper builtDocMapper = createDocumentMapper(builtMapping); BytesReference json = new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/mapper/simple/test1.json")); LuceneDocument doc = builtDocMapper.parse(new SourceToParse("1", json, XContentType.JSON)).rootDoc(); - assertThat(doc.getBinaryValue(builtDocMapper.idFieldMapper().name()), equalTo(Uid.encodeId("1"))); + assertThat(doc.getBinaryValue(IdFieldMapper.NAME), equalTo(Uid.encodeId("1"))); assertThat(doc.get(builtDocMapper.mappers().getMapper("name.first").name()), equalTo("shay")); } @@ -1548,7 +1548,7 @@ public void testSimpleParser() throws Exception { BytesReference json = new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/mapper/simple/test1.json")); LuceneDocument doc = docMapper.parse(new SourceToParse("1", json, XContentType.JSON)).rootDoc(); - assertThat(doc.getBinaryValue(docMapper.idFieldMapper().name()), equalTo(Uid.encodeId("1"))); + assertThat(doc.getBinaryValue(IdFieldMapper.NAME), equalTo(Uid.encodeId("1"))); assertThat(doc.get(docMapper.mappers().getMapper("name.first").name()), equalTo("shay")); } @@ -1557,7 +1557,7 @@ public void testSimpleParserNoTypeNoId() throws Exception { DocumentMapper docMapper = createDocumentMapper(mapping); BytesReference json = new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/mapper/simple/test1-notype-noid.json")); LuceneDocument doc = docMapper.parse(new SourceToParse("1", json, XContentType.JSON)).rootDoc(); - assertThat(doc.getBinaryValue(docMapper.idFieldMapper().name()), equalTo(Uid.encodeId("1"))); + assertThat(doc.getBinaryValue(IdFieldMapper.NAME), equalTo(Uid.encodeId("1"))); assertThat(doc.get(docMapper.mappers().getMapper("name.first").name()), equalTo("shay")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java index 88a2de8ba0244..9320a6c5de2ea 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java @@ -21,7 +21,9 @@ public class IdFieldTypeTests extends ESTestCase { public void testRangeQuery() { - MappedFieldType ft = new IdFieldMapper.IdFieldType(() -> false); + MappedFieldType ft = randomBoolean() + ? new ProvidedIdFieldMapper.IdFieldType(() -> false) + : new TsidExtractingIdFieldMapper.IdFieldType(); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> ft.rangeQuery(null, null, randomBoolean(), randomBoolean(), null, null, null, null) @@ -31,26 +33,33 @@ public void testRangeQuery() { public void testTermsQuery() { SearchExecutionContext context = Mockito.mock(SearchExecutionContext.class); - Settings indexSettings = Settings.builder() + + Settings.Builder indexSettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) - .build(); + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); + if (randomBoolean()) { + indexSettings.put(IndexSettings.MODE.getKey(), "time_series"); + indexSettings.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo"); + } IndexMetadata indexMetadata = IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).settings(indexSettings).build(); IndexSettings mockSettings = new IndexSettings(indexMetadata, Settings.EMPTY); Mockito.when(context.getIndexSettings()).thenReturn(mockSettings); - Mockito.when(context.indexVersionCreated()).thenReturn(indexSettings.getAsVersion(IndexMetadata.SETTING_VERSION_CREATED, null)); - MappedFieldType ft = new IdFieldMapper.IdFieldType(() -> false); + Mockito.when(context.indexVersionCreated()).thenReturn(Version.CURRENT); + MappedFieldType ft = new ProvidedIdFieldMapper.IdFieldType(() -> false); Query query = ft.termQuery("id", context); assertEquals(new TermInSetQuery("_id", Uid.encodeId("id")), query); } public void testIsAggregatable() { - MappedFieldType ft = new IdFieldMapper.IdFieldType(() -> false); + MappedFieldType ft = new ProvidedIdFieldMapper.IdFieldType(() -> false); assertFalse(ft.isAggregatable()); - ft = new IdFieldMapper.IdFieldType(() -> true); + ft = new ProvidedIdFieldMapper.IdFieldType(() -> true); assertTrue(ft.isAggregatable()); + + ft = new TsidExtractingIdFieldMapper.IdFieldType(); + assertFalse(ft.isAggregatable()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java index 199d9df5aae7f..936af554cf15d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java @@ -44,7 +44,7 @@ private static MappingParser createMappingParser(Settings settings) { scriptService, indexAnalyzers, indexSettings, - IdFieldMapper.NO_FIELD_DATA + indexSettings.getMode().buildNoFieldDataIdFieldMapper() ); Map metadataMapperParsers = mapperRegistry.getMetadataMapperParsers( indexSettings.getIndexVersionCreated() diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index fa5b615f51a2c..9d2d67aef7b93 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -13,7 +13,9 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -251,6 +253,8 @@ private static TestMapper fromMapping(String mapping, Version version, boolean f Collections.emptyMap() ); when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); + IndexSettings indexSettings = createIndexSettings(version, Settings.EMPTY); + when(mapperService.getIndexSettings()).thenReturn(indexSettings); MappingParserContext pc = new MappingParserContext(s -> null, s -> { if (Objects.equals("keyword", s)) { return KeywordFieldMapper.PARSER; @@ -267,7 +271,7 @@ private static TestMapper fromMapping(String mapping, Version version, boolean f ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - IdFieldMapper.NO_FIELD_DATA + mapperService.getIndexSettings().getMode().buildNoFieldDataIdFieldMapper() ); if (fromDynamicTemplate) { pc = new MappingParserContext.DynamicTemplateParserContext(pc); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java similarity index 90% rename from server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java index 3df14e5c94ce5..386e093b57c0b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java @@ -20,12 +20,11 @@ import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.index.mapper.IdFieldMapper.ID_FIELD_DATA_DEPRECATION_MESSAGE; import static org.hamcrest.Matchers.containsString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class IdFieldMapperTests extends MapperServiceTestCase { +public class ProvidedIdFieldMapperTests extends MapperServiceTestCase { public void testIncludeInObjectNotAllowed() throws Exception { DocumentMapper docMapper = createDocumentMapper(mapping(b -> {})); @@ -50,7 +49,7 @@ public void testEnableFieldData() throws IOException { boolean[] enabled = new boolean[1]; MapperService mapperService = createMapperService(() -> enabled[0], mapping(b -> {})); - IdFieldMapper.IdFieldType ft = (IdFieldMapper.IdFieldType) mapperService.fieldType("_id"); + ProvidedIdFieldMapper.IdFieldType ft = (ProvidedIdFieldMapper.IdFieldType) mapperService.fieldType("_id"); IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, @@ -61,7 +60,7 @@ public void testEnableFieldData() throws IOException { enabled[0] = true; ft.fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }).build(null, null); - assertWarnings(ID_FIELD_DATA_DEPRECATION_MESSAGE); + assertWarnings(ProvidedIdFieldMapper.ID_FIELD_DATA_DEPRECATION_MESSAGE); assertTrue(ft.isAggregatable()); } @@ -75,7 +74,7 @@ public void testFetchIdFieldValue() throws IOException { SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.lookup()).thenReturn(lookup); - IdFieldMapper.IdFieldType ft = (IdFieldMapper.IdFieldType) mapperService.fieldType("_id"); + ProvidedIdFieldMapper.IdFieldType ft = (ProvidedIdFieldMapper.IdFieldType) mapperService.fieldType("_id"); ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); IndexSearcher searcher = newSearcher(iw); LeafReaderContext context = searcher.getIndexReader().leaves().get(0); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 633ba8ef30efe..7c33bb1eaf8bc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -10,8 +10,9 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.ByteArrayStreamInput; -import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xcontent.XContentBuilder; @@ -43,7 +44,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { private DocumentMapper createDocumentMapper(String routingPath, XContentBuilder mappings) throws IOException { return createMapperService( getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), 200) // Increase dimension limit + .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), 200) // Allow tests that use many dimensions .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") @@ -52,10 +53,16 @@ private DocumentMapper createDocumentMapper(String routingPath, XContentBuilder ).documentMapper(); } - private ParsedDocument parseDocument(DocumentMapper docMapper, CheckedFunction f) - throws IOException { + private ParsedDocument parseDocument(DocumentMapper docMapper, CheckedConsumer f) throws IOException { // Add the @timestamp field required by DataStreamTimestampFieldMapper for all time series indices - return docMapper.parse(source(b -> f.apply(b).field("@timestamp", "2021-10-01"))); + return docMapper.parse(source(null, b -> { + f.accept(b); + b.field("@timestamp", "2021-10-01"); + }, null)); + } + + private BytesRef parseAndGetTsid(DocumentMapper docMapper, CheckedConsumer f) throws IOException { + return parseDocument(docMapper, f).rootDoc().getBinaryValue(TimeSeriesIdFieldMapper.NAME); } public void testEnabledInTimeSeriesMode() throws Exception { @@ -84,7 +91,7 @@ public void testDisabledInStandardMode() throws Exception { ).documentMapper(); assertThat(docMapper.metadataMapper(TimeSeriesIdFieldMapper.class), is(nullValue())); - ParsedDocument doc = docMapper.parse(source(b -> b.field("field", "value"))); + ParsedDocument doc = docMapper.parse(source("id", b -> b.field("field", "value"), null)); assertThat(doc.rootDoc().getBinaryValue("_tsid"), is(nullValue())); assertThat(doc.rootDoc().get("field"), equalTo("value")); } @@ -115,12 +122,12 @@ public void testStrings() throws IOException { .endObject(); })); - ParsedDocument doc = parseDocument( + BytesRef tsid = parseAndGetTsid( docMapper, b -> b.field("a", "foo").field("b", "bar").field("c", "baz").startObject("o").field("e", "bort").endObject() ); assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), matchesMap().entry("a", "foo").entry("o.e", "bort") ); } @@ -128,7 +135,7 @@ public void testStrings() throws IOException { public void testUnicodeKeys() throws IOException { String fire = new String(new int[] { 0x1F525 }, 0, 1); String coffee = "\u2615"; - DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + DocumentMapper docMapper = createDocumentMapper(fire + "," + coffee, mapping(b -> { b.startObject(fire).field("type", "keyword").field("time_series_dimension", true).endObject(); b.startObject(coffee).field("type", "keyword").field("time_series_dimension", true).endObject(); })); @@ -170,13 +177,14 @@ public void testKeywordTooLongUtf8() throws IOException { } public void testKeywordNull() throws IOException { - DocumentMapper docMapper = createDocumentMapper( - "a", - mapping(b -> { b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); }) - ); + DocumentMapper docMapper = createDocumentMapper("r", mapping(b -> { + b.startObject("r").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); - Exception e = expectThrows(MapperParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", (String) null))); - assertThat(e.getCause().getMessage(), equalTo("Dimension fields are missing.")); + BytesRef withNull = parseAndGetTsid(docMapper, b -> b.field("r", "foo").field("a", (String) null)); + BytesRef withoutField = parseAndGetTsid(docMapper, b -> b.field("r", "foo")); + assertThat(withNull, equalTo(withoutField)); } /** @@ -196,13 +204,16 @@ public void testLong() throws IOException { .endObject(); })); - ParsedDocument doc = parseDocument( - docMapper, - b -> b.field("a", 1L).field("b", -1).field("c", "baz").startObject("o").field("e", 1234).endObject() - ); + BytesRef tsid = parseAndGetTsid(docMapper, b -> { + b.field("kw", "kw"); + b.field("a", 1L); + b.field("b", -1); + b.field("c", "baz"); + b.startObject("o").field("e", 1234).endObject(); + }); assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("a", 1L).entry("o.e", 1234L) + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), + matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", 1234L) ); } @@ -214,17 +225,20 @@ public void testLongInvalidString() throws IOException { Exception e = expectThrows(MapperParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_a_long"))); assertThat( e.getMessage(), - equalTo("failed to parse field [a] of type [long] in document with id '1'. Preview of field's value: 'not_a_long'") + // TODO describe the document instead of "null" + equalTo("failed to parse field [a] of type [long] in document with id 'null'. Preview of field's value: 'not_a_long'") ); } public void testLongNull() throws IOException { - DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { + DocumentMapper docMapper = createDocumentMapper("r", mapping(b -> { + b.startObject("r").field("type", "keyword").field("time_series_dimension", true).endObject(); b.startObject("a").field("type", "long").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); })); - Exception e = expectThrows(MapperParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", (Long) null))); - assertThat(e.getCause().getMessage(), equalTo("Dimension fields are missing.")); + + BytesRef withNull = parseAndGetTsid(docMapper, b -> b.field("r", "foo").field("a", (Long) null)); + BytesRef withoutField = parseAndGetTsid(docMapper, b -> b.field("r", "foo")); + assertThat(withNull, equalTo(withoutField)); } /** @@ -244,13 +258,16 @@ public void testInteger() throws IOException { .endObject(); })); - ParsedDocument doc = parseDocument( - docMapper, - b -> b.field("a", 1L).field("b", -1).field("c", "baz").startObject("o").field("e", Integer.MIN_VALUE).endObject() - ); + BytesRef tsid = parseAndGetTsid(docMapper, b -> { + b.field("kw", "kw"); + b.field("a", 1L); + b.field("b", -1); + b.field("c", "baz"); + b.startObject("o").field("e", Integer.MIN_VALUE).endObject(); + }); assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("a", 1L).entry("o.e", (long) Integer.MIN_VALUE) + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), + matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Integer.MIN_VALUE) ); } @@ -262,7 +279,7 @@ public void testIntegerInvalidString() throws IOException { Exception e = expectThrows(MapperParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_an_int"))); assertThat( e.getMessage(), - equalTo("failed to parse field [a] of type [integer] in document with id '1'. Preview of field's value: 'not_an_int'") + equalTo("failed to parse field [a] of type [integer] in document with id 'null'. Preview of field's value: 'not_an_int'") ); } @@ -275,7 +292,7 @@ public void testIntegerOutOfRange() throws IOException { assertThat( e.getMessage(), equalTo( - "failed to parse field [a] of type [integer] in document with id '1'. Preview of field's value: '" + Long.MAX_VALUE + "'" + "failed to parse field [a] of type [integer] in document with id 'null'. Preview of field's value: '" + Long.MAX_VALUE + "'" ) ); } @@ -297,13 +314,16 @@ public void testShort() throws IOException { .endObject(); })); - ParsedDocument doc = parseDocument( - docMapper, - b -> b.field("a", 1L).field("b", -1).field("c", "baz").startObject("o").field("e", Short.MIN_VALUE).endObject() - ); + BytesRef tsid = parseAndGetTsid(docMapper, b -> { + b.field("kw", "kw"); + b.field("a", 1L); + b.field("b", -1); + b.field("c", "baz"); + b.startObject("o").field("e", Short.MIN_VALUE).endObject(); + }); assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("a", 1L).entry("o.e", (long) Short.MIN_VALUE) + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), + matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Short.MIN_VALUE) ); } @@ -315,7 +335,7 @@ public void testShortInvalidString() throws IOException { Exception e = expectThrows(MapperParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_a_short"))); assertThat( e.getMessage(), - equalTo("failed to parse field [a] of type [short] in document with id '1'. Preview of field's value: 'not_a_short'") + equalTo("failed to parse field [a] of type [short] in document with id 'null'. Preview of field's value: 'not_a_short'") ); } @@ -327,7 +347,9 @@ public void testShortOutOfRange() throws IOException { Exception e = expectThrows(MapperParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", Long.MAX_VALUE))); assertThat( e.getMessage(), - equalTo("failed to parse field [a] of type [short] in document with id '1'. Preview of field's value: '" + Long.MAX_VALUE + "'") + equalTo( + "failed to parse field [a] of type [short] in document with id 'null'. Preview of field's value: '" + Long.MAX_VALUE + "'" + ) ); } @@ -348,13 +370,16 @@ public void testByte() throws IOException { .endObject(); })); - ParsedDocument doc = parseDocument( - docMapper, - b -> b.field("a", 1L).field("b", -1).field("c", "baz").startObject("o").field("e", (int) Byte.MIN_VALUE).endObject() - ); + BytesRef tsid = parseAndGetTsid(docMapper, b -> { + b.field("kw", "kw"); + b.field("a", 1L); + b.field("b", -1); + b.field("c", "baz"); + b.startObject("o").field("e", (int) Byte.MIN_VALUE).endObject(); + }); assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("a", 1L).entry("o.e", (long) Byte.MIN_VALUE) + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), + matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Byte.MIN_VALUE) ); } @@ -366,7 +391,7 @@ public void testByteInvalidString() throws IOException { Exception e = expectThrows(MapperParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_a_byte"))); assertThat( e.getMessage(), - equalTo("failed to parse field [a] of type [byte] in document with id '1'. Preview of field's value: 'not_a_byte'") + equalTo("failed to parse field [a] of type [byte] in document with id 'null'. Preview of field's value: 'not_a_byte'") ); } @@ -378,7 +403,9 @@ public void testByteOutOfRange() throws IOException { Exception e = expectThrows(MapperParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", Long.MAX_VALUE))); assertThat( e.getMessage(), - equalTo("failed to parse field [a] of type [byte] in document with id '1'. Preview of field's value: '" + Long.MAX_VALUE + "'") + equalTo( + "failed to parse field [a] of type [byte] in document with id 'null'. Preview of field's value: '" + Long.MAX_VALUE + "'" + ) ); } @@ -399,13 +426,16 @@ public void testIp() throws IOException { .endObject(); })); - ParsedDocument doc = parseDocument( - docMapper, - b -> b.field("a", "192.168.0.1").field("b", -1).field("c", "baz").startObject("o").field("e", "255.255.255.1").endObject() - ); + ParsedDocument doc = parseDocument(docMapper, b -> { + b.field("kw", "kw"); + b.field("a", "192.168.0.1"); + b.field("b", -1); + b.field("c", "baz"); + b.startObject("o").field("e", "255.255.255.1").endObject(); + }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("a", "192.168.0.1").entry("o.e", "255.255.255.1") + matchesMap().entry("kw", "kw").entry("a", "192.168.0.1").entry("o.e", "255.255.255.1") ); } @@ -417,7 +447,7 @@ public void testIpInvalidString() throws IOException { Exception e = expectThrows(MapperParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_an_ip"))); assertThat( e.getMessage(), - equalTo("failed to parse field [a] of type [ip] in document with id '1'. Preview of field's value: 'not_an_ip'") + equalTo("failed to parse field [a] of type [ip] in document with id 'null'. Preview of field's value: 'not_an_ip'") ); } @@ -425,8 +455,6 @@ public void testIpInvalidString() throws IOException { * Tests when the total of the tsid is more than 32k. */ public void testVeryLarge() throws IOException { - // By default, only 16 dimension fields are allowed. To support 100 dimension fields - // we must increase 'index.mapping.dimension_fields.limit' DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); for (int i = 0; i < 100; i++) { @@ -436,12 +464,12 @@ public void testVeryLarge() throws IOException { String large = "many words ".repeat(80); Exception e = expectThrows(MapperParsingException.class, () -> parseDocument(docMapper, b -> { + b.field("b", "foo"); for (int i = 0; i < 100; i++) { b.field("d" + i, large); } - return b; })); - assertThat(e.getCause().getMessage(), equalTo("_tsid longer than [32766] bytes [88691].")); + assertThat(e.getCause().getMessage(), equalTo("_tsid longer than [32766] bytes [88698].")); } /** @@ -457,7 +485,7 @@ public void testSameGenConsistentForSameDoc() throws IOException { String a = randomAlphaOfLength(10); int b = between(1, 100); int c = between(0, 2); - CheckedFunction fields = d -> d.field("a", a).field("b", b).field("c", (long) c); + CheckedConsumer fields = d -> d.field("a", a).field("b", b).field("c", (long) c); ParsedDocument doc1 = parseDocument(docMapper, fields); ParsedDocument doc2 = parseDocument(docMapper, fields); assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); @@ -517,7 +545,7 @@ public void testUnusedExtraDimensions() throws IOException { String a = randomAlphaOfLength(10); int b = between(1, 100); - CheckedFunction fields = d -> d.field("a", a).field("b", b); + CheckedConsumer fields = d -> d.field("a", a).field("b", b); ParsedDocument doc1 = parseDocument(docMapper, fields); ParsedDocument doc2 = parseDocument(docMapper, fields); assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); @@ -557,7 +585,7 @@ public void testDifferentDimensions() throws IOException { String a = randomAlphaOfLength(10); int b = between(1, 100); int c = between(5, 500); - CheckedFunction fields = d -> d.field("a", a).field("b", b).field("c", c); + CheckedConsumer fields = d -> d.field("a", a).field("b", b).field("c", c); ParsedDocument doc1 = parseDocument(docMapper1, fields); ParsedDocument doc2 = parseDocument(docMapper2, fields); assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); @@ -580,15 +608,4 @@ public void testFewerDimensions() throws IOException { ParsedDocument doc2 = parseDocument(docMapper, d -> d.field("a", a).field("b", b).field("c", c)); assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); } - - public void testEmpty() throws IOException { - DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); - b.startObject("c").field("type", "integer").field("time_series_dimension", true).endObject(); - })); - - Exception e = expectThrows(MapperParsingException.class, () -> parseDocument(docMapper, d -> d)); - assertThat(e.getCause().getMessage(), equalTo("Dimension fields are missing.")); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java new file mode 100644 index 0000000000000..062fa7c45d297 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -0,0 +1,491 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.index.mapper; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.routing.IndexRouting; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.name.Named; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class TsidExtractingIdFieldMapperTests extends MetadataMapperTestCase { + private static class TestCase { + private final String name; + private final String expectedId; + private final CheckedConsumer source; + private final List> equivalentSources = new ArrayList<>(); + + TestCase(String name, String expectedId, CheckedConsumer source) { + this.name = name; + this.expectedId = expectedId; + this.source = source; + } + + public TestCase and(CheckedConsumer equivalentSource) { + this.equivalentSources.add(equivalentSource); + return this; + } + + @Override + public String toString() { + return name; + } + } + + @ParametersFactory + public static Iterable params() { + List items = new ArrayList<>(); + /* + * If these values change then ids for individual samples will shift. You may + * modify them with a new index created version, but when you do you must copy + * this test and continue to support the versions here so Elasticsearch can + * continue to read older indices. + */ + + // Dates + items.add(new TestCase("2022-01-01T01:00:00Z", "XsFI2ezm5OViFixWgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + })); + items.add(new TestCase("2022-01-01T01:00:01Z", "XsFI2ezm5OViFixWaI4mE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:01Z"); + b.field("r1", "cat"); + })); + items.add(new TestCase("1970-01-01T00:00:00Z", "XsFI2ezm5OViFixWAAAAAAAAAAA", b -> { + b.field("@timestamp", "1970-01-01T00:00:00Z"); + b.field("r1", "cat"); + })); + items.add(new TestCase("-9998-01-01T00:00:00Z", "XsFI2ezm5OViFixWABhgBIKo_v8", b -> { + b.field("@timestamp", "-9998-01-01T00:00:00Z"); + b.field("r1", "cat"); + })); + items.add(new TestCase("9998-01-01T00:00:00Z", "XsFI2ezm5OViFixWAIS9ImnmAAA", b -> { + b.field("@timestamp", "9998-01-01T00:00:00Z"); + b.field("r1", "cat"); + })); + + // routing keywords + items.add(new TestCase("r1", "XsFI2ezm5OViFixWgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k1", (String) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("L1", (Long) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("i1", (Integer) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("s1", (Short) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("b1", (Byte) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("ip1", (String) null); + })); + items.add(new TestCase("r2", "1y-UzdYi98F0UVRigIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r2", "cat"); + })); + items.add(new TestCase("o.r3", "zh4dcftpIU55Ond-gIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o").field("r3", "cat").endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + })); + + // non-routing keyword + items.add(new TestCase("k1=dog", "XsFI2dL8sZeQhBgxgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k1", "dog"); + })); + items.add(new TestCase("k1=pumpkin", "XsFI2VlD6_SkSo4MgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k1", "pumpkin"); + })); + items.add(new TestCase("k1=empty string", "XsFI2aBA6UgrxLRqgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k1", ""); + })); + items.add(new TestCase("k2", "XsFI2W2e5Ycw0o5_gIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k2", "dog"); + })); + items.add(new TestCase("o.k3", "XsFI2ZAfOI6DMQhFgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.startObject("o").field("k3", "dog").endObject(); + })); + items.add(new TestCase("o.r3", "zh4dcbFtT1qHtjl8gIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o"); + { + b.field("r3", "cat"); + b.field("k3", "dog"); + } + b.endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + b.startObject("o").field("k3", "dog").endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o").field("r3", "cat").endObject(); + b.field("o.k3", "dog"); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + b.field("o.k3", "dog"); + })); + + // long + items.add(new TestCase("L1=1", "XsFI2eGMFOYjW7LLgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("L1", 1); + })); + items.add(new TestCase("L1=min", "XsFI2f9V0yuDfkRWgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("L1", Long.MIN_VALUE); + })); + items.add(new TestCase("L2=1234", "XsFI2S8PYEBSm6QYgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("L2", 1234); + })); + items.add(new TestCase("o.L3=max", "zh4dcaI-57LdG7-cgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o"); + { + b.field("r3", "cat"); + b.field("L3", Long.MAX_VALUE); + } + b.endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + b.startObject("o").field("L3", Long.MAX_VALUE).endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o").field("r3", "cat").endObject(); + b.field("o.L3", Long.MAX_VALUE); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + b.field("o.L3", Long.MAX_VALUE); + })); + + // int + items.add(new TestCase("i1=1", "XsFI2R3LiMZSeUGKgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("i1", 1); + })); + items.add(new TestCase("i1=min", "XsFI2fC7DMEVFaU9gIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("i1", Integer.MIN_VALUE); + })); + items.add(new TestCase("i2=1234", "XsFI2ZVte8HK90RJgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("i2", 1324); + })); + items.add(new TestCase("o.i3=max", "zh4dcQy_QJRCqIx7gIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o"); + { + b.field("r3", "cat"); + b.field("i3", Integer.MAX_VALUE); + } + b.endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + b.startObject("o").field("i3", Integer.MAX_VALUE).endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o").field("r3", "cat").endObject(); + b.field("o.i3", Integer.MAX_VALUE); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + b.field("o.i3", Integer.MAX_VALUE); + })); + + // short + items.add(new TestCase("s1=1", "XsFI2axCr11Q93m7gIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("s1", 1); + })); + items.add(new TestCase("s1=min", "XsFI2Rbs9Ua9BH1wgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("s1", Short.MIN_VALUE); + })); + items.add(new TestCase("s2=1234", "XsFI2SBKaLBqXMBYgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("s2", 1234); + })); + items.add(new TestCase("o.s3=max", "zh4dcYIFo98LQWs4gIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o"); + { + b.field("r3", "cat"); + b.field("s3", Short.MAX_VALUE); + } + b.endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + b.startObject("o").field("s3", Short.MAX_VALUE).endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o").field("r3", "cat").endObject(); + b.field("o.s3", Short.MAX_VALUE); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + b.field("o.s3", Short.MAX_VALUE); + })); + + // byte + items.add(new TestCase("b1=1", "XsFI2dDrcWaf3zDPgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("b1", 1); + })); + items.add(new TestCase("b1=min", "XsFI2cTzLrNqHtxngIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("b1", Byte.MIN_VALUE); + })); + items.add(new TestCase("b2=12", "XsFI2Sb77VB9AswjgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("b2", 12); + })); + items.add(new TestCase("o.s3=max", "zh4dcfFauKzj6lgxgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o"); + { + b.field("r3", "cat"); + b.field("b3", Byte.MAX_VALUE); + } + b.endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + b.startObject("o").field("b3", Byte.MAX_VALUE).endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o").field("r3", "cat").endObject(); + b.field("o.b3", Byte.MAX_VALUE); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + b.field("o.b3", Byte.MAX_VALUE); + })); + + // ip + items.add(new TestCase("ip1=192.168.0.1", "XsFI2dJ1cyrrjNa2gIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("ip1", "192.168.0.1"); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("ip1", "::ffff:c0a8:1"); + })); + items.add(new TestCase("ip1=12.12.45.254", "XsFI2ZUAcRxOwhHKgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("ip1", "12.12.45.254"); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("ip1", "::ffff:c0c:2dfe"); + })); + items.add(new TestCase("ip2=FE80:CD00:0000:0CDE:1257:0000:211E:729C", "XsFI2XTGWAekP_oGgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("ip2", "FE80:CD00:0000:0CDE:1257:0000:211E:729C"); + })); + items.add(new TestCase("o.ip3=2001:db8:85a3:8d3:1319:8a2e:370:7348", "zh4dcU_FSGP9GuHjgIomE34BAAA", b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o"); + { + b.field("r3", "cat"); + b.field("ip3", "2001:db8:85a3:8d3:1319:8a2e:370:7348"); + } + b.endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + b.startObject("o").field("ip3", "2001:db8:85a3:8d3:1319:8a2e:370:7348").endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o").field("r3", "cat").endObject(); + b.field("o.ip3", "2001:db8:85a3:8d3:1319:8a2e:370:7348"); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + b.field("o.ip3", "2001:db8:85a3:8d3:1319:8a2e:370:7348"); + })); + + return items.stream().map(td -> new Object[] { td }).toList(); + } + + private final TestCase testCase; + + public TsidExtractingIdFieldMapperTests(@Named("testCase") TestCase testCase) throws IOException { + this.testCase = testCase; + } + + public void testExpectedId() throws IOException { + assertThat(parse(null, mapperService(), testCase.source).id(), equalTo(testCase.expectedId)); + } + + public void testProvideExpectedId() throws IOException { + assertThat(parse(testCase.expectedId, mapperService(), testCase.source).id(), equalTo(testCase.expectedId)); + } + + public void testProvideWrongId() throws IOException { + String wrongId = testCase.expectedId + "wrong"; + Exception e = expectThrows(MapperParsingException.class, () -> parse(wrongId, mapperService(), testCase.source)); + assertThat( + e.getCause().getMessage(), + equalTo( + "_id must be unset or set to [" + + testCase.expectedId + + "] but was [" + + testCase.expectedId + + "wrong] because [index] is in time_series mode" + ) + ); + } + + public void testEquivalentSources() throws IOException { + MapperService mapperService = mapperService(); + for (CheckedConsumer equivalent : testCase.equivalentSources) { + assertThat(parse(null, mapperService, equivalent).id(), equalTo(testCase.expectedId)); + } + } + + private ParsedDocument parse(@Nullable String id, MapperService mapperService, CheckedConsumer source) + throws IOException { + try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) { + builder.startObject(); + source.accept(builder); + builder.endObject(); + SourceToParse sourceToParse = new SourceToParse(id, BytesReference.bytes(builder), builder.contentType()); + return mapperService.documentParser().parseDocument(sourceToParse, mapperService.mappingLookup()); + } + } + + public void testRoutingPathCompliant() throws IOException { + Version version = VersionUtils.randomIndexCompatibleVersion(random()); + IndexRouting indexRouting = createIndexSettings(version, indexSettings(version)).getIndexRouting(); + int indexShard = indexShard(indexRouting); + assertThat(indexRouting.getShard(testCase.expectedId, null), equalTo(indexShard)); + assertThat(indexRouting.deleteShard(testCase.expectedId, null), equalTo(indexShard)); + } + + private int indexShard(IndexRouting indexRouting) throws IOException { + try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) { + builder.startObject(); + testCase.source.accept(builder); + builder.endObject(); + return indexRouting.indexShard(null, null, builder.contentType(), BytesReference.bytes(builder)); + } + } + + private Settings indexSettings(Version version) { + return Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.SETTING_VERSION_CREATED, version) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 100)) + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "-9999-01-01T00:00:00Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "9999-01-01T00:00:00Z") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "r1,r2,o.r3") + .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), 100) + .build(); + } + + private MapperService mapperService() throws IOException { + Version version = VersionUtils.randomIndexCompatibleVersion(random()); + return createMapperService(indexSettings(version), mapping(b -> { + b.startObject("r1").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("r2").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("k1").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("k2").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("L1").field("type", "long").field("time_series_dimension", true).endObject(); + b.startObject("L2").field("type", "long").field("time_series_dimension", true).endObject(); + b.startObject("i1").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("i2").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("s1").field("type", "short").field("time_series_dimension", true).endObject(); + b.startObject("s2").field("type", "short").field("time_series_dimension", true).endObject(); + b.startObject("b1").field("type", "byte").field("time_series_dimension", true).endObject(); + b.startObject("b2").field("type", "byte").field("time_series_dimension", true).endObject(); + b.startObject("ip1").field("type", "ip").field("time_series_dimension", true).endObject(); + b.startObject("ip2").field("type", "ip").field("time_series_dimension", true).endObject(); + b.startObject("o").startObject("properties"); + { + b.startObject("r3").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("k3").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("L3").field("type", "long").field("time_series_dimension", true).endObject(); + b.startObject("i3").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("s3").field("type", "short").field("time_series_dimension", true).endObject(); + b.startObject("b3").field("type", "byte").field("time_series_dimension", true).endObject(); + b.startObject("ip3").field("type", "ip").field("time_series_dimension", true).endObject(); + } + b.endObject().endObject(); + })); + } + + @Override + protected String fieldName() { + return IdFieldMapper.NAME; + } + + @Override + protected void registerParameters(ParameterChecker checker) throws IOException {} +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java index 8c9dd3ac3f13c..98d0e96696d82 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -81,7 +81,7 @@ public void testMultiFieldWithinMultiField() throws IOException { ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - IdFieldMapper.NO_FIELD_DATA + ProvidedIdFieldMapper.NO_FIELD_DATA ); TextFieldMapper.PARSER.parse("some-field", fieldNode, olderContext); @@ -107,7 +107,7 @@ public void testMultiFieldWithinMultiField() throws IOException { ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - IdFieldMapper.NO_FIELD_DATA + ProvidedIdFieldMapper.NO_FIELD_DATA ); IllegalArgumentException e = expectThrows( diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 14234260c1dfb..a56eaff3910f5 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -38,7 +38,6 @@ import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.KeywordScriptFieldType; @@ -443,7 +442,7 @@ private static MapperService createMapperService(IndexSettings indexSettings) { ScriptCompiler.NONE, indexAnalyzers, indexSettings, - new IdFieldMapper(() -> true) + indexSettings.getMode().buildIdFieldMapper(() -> true) ) ); return mapperService; diff --git a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java index ca5dc38109431..66b130ca69bc8 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java @@ -148,7 +148,7 @@ public void testRetryAppendOnlyAfterRecovering() throws Exception { try (ReplicationGroup shards = createGroup(0)) { shards.startAll(); final IndexRequest originalRequest = new IndexRequest(index.getName()).source("{}", XContentType.JSON); - originalRequest.process(); + originalRequest.autoGenerateId(); final IndexRequest retryRequest = copyIndexRequest(originalRequest); retryRequest.onRetry(); shards.index(retryRequest); diff --git a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java index 88655e947c60e..da0e46a76d249 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java @@ -611,7 +611,7 @@ public void testTransferMaxSeenAutoIdTimestampOnResync() throws Exception { List replicationRequests = new ArrayList<>(); for (int numDocs = between(1, 10), i = 0; i < numDocs; i++) { final IndexRequest indexRequest = new IndexRequest(index.getName()).source("{}", XContentType.JSON); - indexRequest.process(); + indexRequest.autoGenerateId(); final IndexRequest copyRequest; if (randomBoolean()) { copyRequest = copyIndexRequest(indexRequest); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java index e517387c0ff2f..ce1e7f95d7d19 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java @@ -129,7 +129,11 @@ public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) { ParsedDocument doc = InternalEngineTests.createParsedDoc("1", null); Engine.Delete delete = new Engine.Delete("1", new Term("_id", Uid.encodeId(doc.id())), randomNonNegativeLong()); Engine.Index index = new Engine.Index(new Term("_id", Uid.encodeId(doc.id())), randomNonNegativeLong(), doc); - compositeListener.postDelete(randomShardId, delete, new Engine.DeleteResult(1, 0, SequenceNumbers.UNASSIGNED_SEQ_NO, true)); + compositeListener.postDelete( + randomShardId, + delete, + new Engine.DeleteResult(1, 0, SequenceNumbers.UNASSIGNED_SEQ_NO, true, delete.id()) + ); assertEquals(0, preIndex.get()); assertEquals(0, postIndex.get()); assertEquals(0, postIndexException.get()); @@ -153,7 +157,11 @@ public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) { assertEquals(2, postDelete.get()); assertEquals(2, postDeleteException.get()); - compositeListener.postIndex(randomShardId, index, new Engine.IndexResult(0, 0, SequenceNumbers.UNASSIGNED_SEQ_NO, false)); + compositeListener.postIndex( + randomShardId, + index, + new Engine.IndexResult(0, 0, SequenceNumbers.UNASSIGNED_SEQ_NO, false, index.id()) + ); assertEquals(0, preIndex.get()); assertEquals(2, postIndex.get()); assertEquals(0, postIndexException.get()); diff --git a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index 7ff97e84a9907..8cb09c3077949 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.seqno.RetentionLeases; @@ -537,7 +538,7 @@ private Engine.IndexResult index(String id, String testFieldValue) throws IOExce final Term uid = new Term(IdFieldMapper.NAME, Uid.encodeId(id)); LuceneDocument document = new LuceneDocument(); document.add(new TextField("test", testFieldValue, Field.Store.YES)); - Field idField = new Field(uid.field(), uid.bytes(), IdFieldMapper.Defaults.FIELD_TYPE); + Field idField = new Field(uid.field(), uid.bytes(), ProvidedIdFieldMapper.Defaults.FIELD_TYPE); Field versionField = new NumericDocValuesField("_version", Versions.MATCH_ANY); SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); document.add(idField); diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index d07a114bb7e82..578ec92fdd59c 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -943,10 +943,6 @@ private Term newUid(ParsedDocument doc) { return new Term("_id", Uid.encodeId(doc.id())); } - private Term newUid(String id) { - return new Term("_id", Uid.encodeId(id)); - } - public void testVerifyTranslogIsNotDeleted() throws IOException { assertFileIsPresent(translog, 1); translog.add(new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 })); @@ -3340,7 +3336,7 @@ public void testTranslogOpSerialization() throws Exception { seqID.seqNo.setLongValue(randomSeqNum); seqID.seqNoDocValue.setLongValue(randomSeqNum); seqID.primaryTerm.setLongValue(randomPrimaryTerm); - Field idField = new Field("_id", Uid.encodeId("1"), IdFieldMapper.Defaults.FIELD_TYPE); + Field idField = IdFieldMapper.standardIdField("1"); Field versionField = new NumericDocValuesField("_version", 1); LuceneDocument document = new LuceneDocument(); document.add(new TextField("value", "test", Field.Store.YES)); @@ -3365,7 +3361,7 @@ public void testTranslogOpSerialization() throws Exception { SequenceNumbers.UNASSIGNED_SEQ_NO, 0 ); - Engine.IndexResult eIndexResult = new Engine.IndexResult(1, randomPrimaryTerm, randomSeqNum, true); + Engine.IndexResult eIndexResult = new Engine.IndexResult(1, randomPrimaryTerm, randomSeqNum, true, eIndex.id()); Translog.Index index = new Translog.Index(eIndex, eIndexResult); Version wireVersion = VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT); @@ -3389,7 +3385,7 @@ public void testTranslogOpSerialization() throws Exception { SequenceNumbers.UNASSIGNED_SEQ_NO, 0 ); - Engine.DeleteResult eDeleteResult = new Engine.DeleteResult(2, randomPrimaryTerm, randomSeqNum, true); + Engine.DeleteResult eDeleteResult = new Engine.DeleteResult(2, randomPrimaryTerm, randomSeqNum, true, doc.id()); Translog.Delete delete = new Translog.Delete(eDelete, eDeleteResult); out = new BytesStreamOutput(); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index 6215575e7c359..c50e3d1e47018 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -256,12 +256,12 @@ public void testSendSnapshotSendsOps() throws IOException { final int initialNumberOfDocs = randomIntBetween(10, 1000); for (int i = 0; i < initialNumberOfDocs; i++) { final Engine.Index index = getIndex(Integer.toString(i)); - operations.add(new Translog.Index(index, new Engine.IndexResult(1, 1, SequenceNumbers.UNASSIGNED_SEQ_NO, true))); + operations.add(new Translog.Index(index, new Engine.IndexResult(1, 1, SequenceNumbers.UNASSIGNED_SEQ_NO, true, index.id()))); } final int numberOfDocsWithValidSequenceNumbers = randomIntBetween(10, 1000); for (int i = initialNumberOfDocs; i < initialNumberOfDocs + numberOfDocsWithValidSequenceNumbers; i++) { final Engine.Index index = getIndex(Integer.toString(i)); - operations.add(new Translog.Index(index, new Engine.IndexResult(1, 1, i - initialNumberOfDocs, true))); + operations.add(new Translog.Index(index, new Engine.IndexResult(1, 1, i - initialNumberOfDocs, true, index.id()))); } final long startingSeqNo = randomIntBetween(0, numberOfDocsWithValidSequenceNumbers - 1); final long endingSeqNo = randomLongBetween(startingSeqNo, numberOfDocsWithValidSequenceNumbers - 1); @@ -330,7 +330,7 @@ public void testSendSnapshotStopOnError() throws Exception { final List ops = new ArrayList<>(); for (int numOps = between(1, 256), i = 0; i < numOps; i++) { final Engine.Index index = getIndex(Integer.toString(i)); - ops.add(new Translog.Index(index, new Engine.IndexResult(1, 1, i, true))); + ops.add(new Translog.Index(index, new Engine.IndexResult(1, 1, i, true, index.id()))); } final AtomicBoolean wasFailed = new AtomicBoolean(); RecoveryTargetHandler recoveryTarget = new TestRecoveryTargetHandler() { @@ -464,10 +464,10 @@ public void indexTranslogOperations( assertThat(receivedSeqNos, equalTo(sentSeqNos)); } - private Engine.Index getIndex(final String id) { + private Engine.Index getIndex(String id) { final LuceneDocument document = new LuceneDocument(); document.add(new TextField("test", "test", Field.Store.YES)); - final Field idField = new Field("_id", Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE); + final Field idField = IdFieldMapper.standardIdField(id); // TODO tsdbid field could be different. final Field versionField = new NumericDocValuesField("_version", Versions.MATCH_ANY); final SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); document.add(idField); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index bf33a367b3315..d6df5f58535f6 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -54,6 +54,7 @@ import org.elasticsearch.index.mapper.NestedPathFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.ObjectMapper; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.Uid; @@ -660,13 +661,13 @@ public void testSubAggregationOfNested() throws Exception { // Root docs Document root; root = new Document(); - root.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.FIELD_TYPE)); + root.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); root.add(sequenceIDFields.primaryTerm); root.add(new StringField(rootNameField, new BytesRef("Ballpoint"), Field.Store.NO)); documents.add(root); root = new Document(); - root.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), IdFieldMapper.Defaults.FIELD_TYPE)); + root.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); root.add(new StringField(rootNameField, new BytesRef("Notebook"), Field.Store.NO)); root.add(sequenceIDFields.primaryTerm); documents.add(root); @@ -714,13 +715,13 @@ public void testSubAggregationOfNestedAggregateAfter() throws Exception { // Root docs Document root; root = new Document(); - root.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.FIELD_TYPE)); + root.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); root.add(sequenceIDFields.primaryTerm); root.add(new StringField(rootNameField, new BytesRef("Ballpoint"), Field.Store.NO)); documents.add(root); root = new Document(); - root.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), IdFieldMapper.Defaults.FIELD_TYPE)); + root.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); root.add(new StringField(rootNameField, new BytesRef("Notebook"), Field.Store.NO)); root.add(sequenceIDFields.primaryTerm); documents.add(root); @@ -3066,7 +3067,7 @@ private static Map> createDocument(Object... fields) { private Document createNestedDocument(String id, String nestedPath, Object... rawFields) { assert rawFields.length % 2 == 0; Document doc = new Document(); - doc.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + doc.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); doc.add(new Field(NestedPathFieldMapper.NAME, nestedPath, NestedPathFieldMapper.Defaults.FIELD_TYPE)); Object[] fields = new Object[rawFields.length]; for (int i = 0; i < fields.length; i += 2) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java index 306134fd773e9..d2ad2e5f10b8f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.index.mapper.NestedPathFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.ObjectMapper; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -178,7 +179,9 @@ public void testSingleNestingMax() throws IOException { expectedNestedDocs += numNestedDocs; Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add( + new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), ProvidedIdFieldMapper.Defaults.FIELD_TYPE) + ); document.add(new Field(NestedPathFieldMapper.NAME, "test", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); documents.add(document); @@ -232,7 +235,9 @@ public void testDoubleNestingMax() throws IOException { expectedNestedDocs += numNestedDocs; Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add( + new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), ProvidedIdFieldMapper.Defaults.FIELD_TYPE) + ); document.add(new Field(NestedPathFieldMapper.NAME, "test", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); documents.add(document); @@ -284,7 +289,9 @@ public void testOrphanedDocs() throws IOException { expectedNestedDocs += numNestedDocs; Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add( + new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), ProvidedIdFieldMapper.Defaults.FIELD_TYPE) + ); document.add(new Field(NestedPathFieldMapper.NAME, "test", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); documents.add(document); @@ -331,19 +338,19 @@ public void testResetRootDocId() throws Exception { // 1 segment with, 1 root document, with 3 nested sub docs Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "test", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); documents.add(document); @@ -354,11 +361,11 @@ public void testResetRootDocId() throws Exception { // 1 segment with: // 1 document, with 1 nested subdoc document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "test", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); documents.add(document); @@ -366,11 +373,11 @@ public void testResetRootDocId() throws Exception { documents.clear(); // and 1 document, with 1 nested subdoc document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("3"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("3"), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("3"), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("3"), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "test", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); documents.add(document); @@ -604,19 +611,19 @@ public void testPreGetChildLeafCollectors() throws IOException { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { List documents = new ArrayList<>(); Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedDocValuesField("key", new BytesRef("key1"))); document.add(new SortedDocValuesField("value", new BytesRef("a1"))); documents.add(document); document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedDocValuesField("key", new BytesRef("key2"))); document.add(new SortedDocValuesField("value", new BytesRef("b1"))); documents.add(document); document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "_doc", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); documents.add(document); @@ -625,19 +632,19 @@ public void testPreGetChildLeafCollectors() throws IOException { documents.clear(); document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedDocValuesField("key", new BytesRef("key1"))); document.add(new SortedDocValuesField("value", new BytesRef("a2"))); documents.add(document); document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedDocValuesField("key", new BytesRef("key2"))); document.add(new SortedDocValuesField("value", new BytesRef("b2"))); documents.add(document); document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "_doc", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); documents.add(document); @@ -646,19 +653,19 @@ public void testPreGetChildLeafCollectors() throws IOException { documents.clear(); document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("3"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("3"), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedDocValuesField("key", new BytesRef("key1"))); document.add(new SortedDocValuesField("value", new BytesRef("a3"))); documents.add(document); document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("3"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("3"), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedDocValuesField("key", new BytesRef("key2"))); document.add(new SortedDocValuesField("value", new BytesRef("b3"))); documents.add(document); document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("3"), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("3"), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "_doc", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); documents.add(document); @@ -728,7 +735,9 @@ public void testFieldAlias() throws IOException { generateDocuments(documents, numNestedDocs, i, NESTED_OBJECT, VALUE_FIELD_NAME); Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add( + new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), ProvidedIdFieldMapper.Defaults.FIELD_TYPE) + ); document.add(new Field(NestedPathFieldMapper.NAME, "test", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); documents.add(document); @@ -768,7 +777,9 @@ public void testNestedWithPipeline() throws IOException { expectedNestedDocs += 1; Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add( + new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), ProvidedIdFieldMapper.Defaults.FIELD_TYPE) + ); document.add(new Field(NestedPathFieldMapper.NAME, "test", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); documents.add(document); @@ -860,7 +871,7 @@ public static CheckedConsumer buildResellerData( documents.get(r).add(new SortedNumericDocValuesField("reseller_id", r)); } Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(p)), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(p)), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "test", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); document.add(new SortedNumericDocValuesField("product_id", p)); @@ -888,7 +899,9 @@ private static double[] generateDocuments(List documents, int numNeste double[] values = new double[numNestedDocs]; for (int nested = 0; nested < numNestedDocs; nested++) { Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(id)), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add( + new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(id)), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE) + ); document.add(new Field(NestedPathFieldMapper.NAME, path, NestedPathFieldMapper.Defaults.FIELD_TYPE)); long value = randomNonNegativeLong() % 10000; document.add(new SortedNumericDocValuesField(fieldName, value)); @@ -903,14 +916,14 @@ public static List generateBook(String id, String[] authors, int[] num for (int numPage : numPages) { Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_chapters", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedNumericDocValuesField("num_pages", numPage)); documents.add(document); } Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "book", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); for (String author : authors) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java index 09896b9c08425..be918d63a9916 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.mapper.NestedPathFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.ObjectMapper; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.search.aggregations.AggregationBuilder; @@ -96,14 +97,20 @@ public void testMaxFromParentDocs() throws IOException { for (int nested = 0; nested < numNestedDocs; nested++) { Document document = new Document(); document.add( - new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), IdFieldMapper.Defaults.NESTED_FIELD_TYPE) + new Field( + IdFieldMapper.NAME, + Uid.encodeId(Integer.toString(i)), + ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE + ) ); document.add(new Field(NestedPathFieldMapper.NAME, NESTED_OBJECT, NestedPathFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); expectedNestedDocs++; } Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add( + new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), ProvidedIdFieldMapper.Defaults.FIELD_TYPE) + ); document.add(new Field(NestedPathFieldMapper.NAME, "test", NestedPathFieldMapper.Defaults.FIELD_TYPE)); long value = randomNonNegativeLong() % 10000; document.add(new SortedNumericDocValuesField(VALUE_FIELD_NAME, value)); @@ -157,13 +164,19 @@ public void testFieldAlias() throws IOException { for (int nested = 0; nested < numNestedDocs; nested++) { Document document = new Document(); document.add( - new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), IdFieldMapper.Defaults.NESTED_FIELD_TYPE) + new Field( + IdFieldMapper.NAME, + Uid.encodeId(Integer.toString(i)), + ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE + ) ); document.add(new Field(NestedPathFieldMapper.NAME, NESTED_OBJECT, NestedPathFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); } Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add( + new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), ProvidedIdFieldMapper.Defaults.FIELD_TYPE) + ); document.add(new Field(NestedPathFieldMapper.NAME, "test", NestedPathFieldMapper.Defaults.FIELD_TYPE)); long value = randomNonNegativeLong() % 10000; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java index ed291c3ec7f09..0b8f4e1d25e4a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.mapper.NestedPathFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.ObjectMapper; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.mapper.SeqNoFieldMapper; @@ -490,14 +491,14 @@ private List generateDocsWithNested(String id, int value, int[] nested for (int nestedValue : nestedValues) { Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_object", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedNumericDocValuesField("nested_value", nestedValue)); documents.add(document); } Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "docs", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedNumericDocValuesField("value", value)); document.add(sequenceIDFields.primaryTerm); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index 82e144f5cd683..937f38a9a10cf 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -57,6 +57,7 @@ import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.ObjectMapper; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.mapper.SeqNoFieldMapper; @@ -2139,14 +2140,14 @@ private List generateDocsWithNested(String id, int value, int[] nested for (int nestedValue : nestedValues) { Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_object", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedNumericDocValuesField("nested_value", nestedValue)); documents.add(document); } Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "docs", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedNumericDocValuesField("value", value)); document.add(sequenceIDFields.primaryTerm); @@ -2166,7 +2167,7 @@ private List> generateAnimalDocsWithNested( for (int i = 0; i < tags.length; i++) { List document = new ArrayList<>(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(NestedPathFieldMapper.NAME, "nested_object", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedDocValuesField("tag", new BytesRef(tags[i]))); @@ -2175,7 +2176,7 @@ private List> generateAnimalDocsWithNested( } List document = new ArrayList<>(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); document.addAll(doc(animalFieldType, animal)); document.add(new Field(NestedPathFieldMapper.NAME, "docs", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorTests.java index aa79db97d9ca6..98c9d0ffe0472 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.Aggregation; @@ -137,7 +138,7 @@ private Aggregation testCase(Query query, AggregationBuilder builder) throws IOE private Document document(String id, String... stringValues) { Document document = new Document(); - document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); for (String stringValue : stringValues) { document.add(new Field("string", stringValue, KeywordFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedSetDocValuesField("string", new BytesRef(stringValue))); diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index 5524a4b41b646..51d1fa9239587 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.env.Environment; import org.elasticsearch.index.analysis.IndexAnalyzers; -import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; @@ -65,7 +64,7 @@ public static MapperService newMapperService( similarityService, mapperRegistry, () -> null, - IdFieldMapper.NO_FIELD_DATA, + indexSettings.getMode().buildNoFieldDataIdFieldMapper(), ScriptCompiler.NONE ); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 903617f1ae14c..ac4e8369855a6 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -79,6 +79,7 @@ import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceToParse; @@ -387,8 +388,8 @@ protected static ParsedDocument testParsedDocument( BytesReference source, Mapping mappingUpdate, boolean recoverySource - ) { - Field uidField = new Field("_id", Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE); + ) { // TODO try with TsdbIdFieldMapper + Field uidField = new Field("_id", Uid.encodeId(id), ProvidedIdFieldMapper.Defaults.FIELD_TYPE); Field versionField = new NumericDocValuesField("_version", 0); SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); document.add(uidField); diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index 584b4a8523d2e..7001d91ad889f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -17,7 +17,6 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SourceToParse; @@ -60,7 +59,7 @@ public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings ind similarityService, mapperRegistry, () -> null, - IdFieldMapper.NO_FIELD_DATA, + indexSettings.getMode().buildNoFieldDataIdFieldMapper(), null ); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 99fd964993858..ce8661a4a39ce 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -201,7 +201,7 @@ protected final MapperService createMapperService(Version version, Settings sett similarityService, mapperRegistry, () -> { throw new UnsupportedOperationException(); }, - new IdFieldMapper(idFieldDataEnabled), + indexSettings.getMode().buildIdFieldMapper(idFieldDataEnabled), this::compileScript ); } @@ -239,11 +239,14 @@ protected final void withLuceneIndex( } } + /** + * Build a {@link SourceToParse} with an id. + */ protected final SourceToParse source(CheckedConsumer build) throws IOException { return source("1", build, null); } - protected final SourceToParse source(String id, CheckedConsumer build, @Nullable String routing) + protected final SourceToParse source(@Nullable String id, CheckedConsumer build, @Nullable String routing) throws IOException { return source("test", id, build, routing, Map.of()); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index 2ac75c2c77b81..0d873b92af4b0 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -875,7 +875,7 @@ private void executeShardBulkOnPrimary( ) { for (BulkItemRequest itemRequest : request.items()) { if (itemRequest.request() instanceof IndexRequest) { - ((IndexRequest) itemRequest.request()).process(); + ((IndexRequest) itemRequest.request()).process(primary.indexSettings().getIndexRouting()); } } final PlainActionFuture permitAcquiredFuture = new PlainActionFuture<>(); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 5d97c6e6900f5..c98d8ced8042c 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -410,6 +410,7 @@ public boolean shouldCache(Query query) { IndexShard indexShard = mock(IndexShard.class); when(indexShard.shardId()).thenReturn(new ShardId("test", "test", 0)); + when(indexShard.indexSettings()).thenReturn(indexSettings); when(ctx.indexShard()).thenReturn(indexShard); return new SubSearchContext(ctx); } @@ -991,7 +992,8 @@ public void testSupportedFieldTypes() throws IOException { source.put("doc_values", "true"); } - Mapper.Builder builder = mappedType.getValue().parse(fieldName, source, new MockParserContext()); + IndexSettings indexSettings = createIndexSettings(); + Mapper.Builder builder = mappedType.getValue().parse(fieldName, source, new MockParserContext(indexSettings)); FieldMapper mapper = (FieldMapper) builder.build(MapperBuilderContext.ROOT); MappedFieldType fieldType = mapper.fieldType(); @@ -1174,8 +1176,8 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd } private static class MockParserContext extends MappingParserContext { - MockParserContext() { - super(null, null, null, Version.CURRENT, null, null, ScriptCompiler.NONE, null, null, null); + MockParserContext(IndexSettings indexSettings) { + super(null, null, null, Version.CURRENT, null, null, ScriptCompiler.NONE, null, indexSettings, null); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 99d53776af696..f1f0debc265b7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -41,7 +41,6 @@ import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexFieldDataService; -import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.SearchExecutionContext; @@ -403,7 +402,7 @@ private static class ServiceHolder implements Closeable { similarityService, mapperRegistry, () -> createShardContext(null), - IdFieldMapper.NO_FIELD_DATA, + idxSettings.getMode().buildNoFieldDataIdFieldMapper(), ScriptCompiler.NONE ); IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 96421186cb969..b49a603c20ea2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -2314,7 +2314,7 @@ synchronized String routingKeyForShard(Index index, int shard, Random random) { IndexRouting indexRouting = IndexRouting.fromIndexMetadata(clusterState.metadata().getIndexSafe(index)); while (true) { String routing = RandomStrings.randomAsciiLettersOfLength(random, 10); - if (shard == indexRouting.indexShard(null, routing, null, null)) { + if (shard == indexRouting.indexShard("id", routing, null, null)) { return routing; } } diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java index b6952dc70dbed..c34351020547f 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java +++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java @@ -261,15 +261,7 @@ public void testFollowTsdbIndex() throws Exception { ); for (int i = 0; i < numDocs; i++) { logger.info("Indexing doc [{}]", i); - index( - client(), - leaderIndexName, - Integer.toString(i), - "@timestamp", - basetime + TimeUnit.SECONDS.toMillis(i * 10), - "dim", - "foobar" - ); + index(client(), leaderIndexName, null, "@timestamp", basetime + TimeUnit.SECONDS.toMillis(i * 10), "dim", "foobar"); } refresh(leaderIndexName); verifyDocuments(client(), leaderIndexName, numDocs); @@ -306,31 +298,30 @@ public void testFollowTsdbIndex() throws Exception { pauseFollow(followIndexName); resumeFollow(followIndexName); try (RestClient leaderClient = buildLeaderClient()) { - int id = numDocs; index( leaderClient, leaderIndexName, - Integer.toString(id), + null, "@timestamp", - basetime + TimeUnit.SECONDS.toMillis(id * 10), + basetime + TimeUnit.SECONDS.toMillis(numDocs * 10), "dim", "foobar" ); index( leaderClient, leaderIndexName, - Integer.toString(id + 1), + null, "@timestamp", - basetime + TimeUnit.SECONDS.toMillis(id * 10 + 10), + basetime + TimeUnit.SECONDS.toMillis(numDocs * 10 + 10), "dim", "foobar" ); index( leaderClient, leaderIndexName, - Integer.toString(id + 2), + null, "@timestamp", - basetime + TimeUnit.SECONDS.toMillis(id * 10 + 20), + basetime + TimeUnit.SECONDS.toMillis(numDocs * 10 + 20), "dim", "foobar" ); diff --git a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java index ed84f36c669e8..097b536fed445 100644 --- a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java +++ b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java @@ -57,7 +57,7 @@ protected static void index(RestClient client, String index, String id, Object.. document.field((String) fields[i], fields[i + 1]); } document.endObject(); - final Request request = new Request("POST", "/" + index + "/_doc/" + id); + final Request request = new Request("POST", "/" + index + "/_doc" + (id == null ? "" : "/" + id)); request.setJsonEntity(Strings.toString(document)); assertOK(client.performRequest(request)); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngine.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngine.java index 9987d16bae5a6..f503d2d0e6094 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngine.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngine.java @@ -83,7 +83,7 @@ protected InternalEngine.IndexingStrategy indexingStrategyForOperation(final Ind index.seqNo(), lookupPrimaryTerm(index.seqNo()) ); - return IndexingStrategy.skipDueToVersionConflict(error, false, index.version()); + return IndexingStrategy.skipDueToVersionConflict(error, false, index.version(), index.id()); } else { return planIndexingAsNonPrimary(index); } @@ -99,7 +99,7 @@ protected InternalEngine.DeletionStrategy deletionStrategyForOperation(final Del delete.seqNo(), lookupPrimaryTerm(delete.seqNo()) ); - return DeletionStrategy.skipDueToVersionConflict(error, delete.version(), false); + return DeletionStrategy.skipDueToVersionConflict(error, delete.version(), false, delete.id()); } else { return planDeletionAsNonPrimary(delete); }