Skip to content

Commit 5c98098

Browse files
quux00quux00
authored
Revert "Search coordinator uses event.ingested in cluster state to do rewrites (#110352)" (#110881)
This reverts commit d45d164.
1 parent 91fb6e9 commit 5c98098

12 files changed

Lines changed: 173 additions & 1034 deletions

File tree

‎docs/changelog/110352.yaml‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/TimestampFieldMapperServiceTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public void testGetTimestampFieldTypeForTsdbDataStream() throws IOException {
6161
DocWriteResponse indexResponse = indexDoc();
6262

6363
var indicesService = getInstanceFromNode(IndicesService.class);
64-
var result = indicesService.getTimestampFieldTypeInfo(indexResponse.getShardId().getIndex());
64+
var result = indicesService.getTimestampFieldType(indexResponse.getShardId().getIndex());
6565
assertThat(result, notNullValue());
6666
}
6767

@@ -70,7 +70,7 @@ public void testGetTimestampFieldTypeForDataStream() throws IOException {
7070
DocWriteResponse indexResponse = indexDoc();
7171

7272
var indicesService = getInstanceFromNode(IndicesService.class);
73-
var result = indicesService.getTimestampFieldTypeInfo(indexResponse.getShardId().getIndex());
73+
var result = indicesService.getTimestampFieldType(indexResponse.getShardId().getIndex());
7474
assertThat(result, nullValue());
7575
}
7676

‎server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java‎

Lines changed: 18 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,11 @@
99
package org.elasticsearch.index.query;
1010

1111
import org.elasticsearch.client.internal.Client;
12-
import org.elasticsearch.cluster.metadata.DataStream;
13-
import org.elasticsearch.cluster.metadata.IndexMetadata;
14-
import org.elasticsearch.common.Strings;
1512
import org.elasticsearch.core.Nullable;
13+
import org.elasticsearch.index.mapper.DateFieldMapper;
1614
import org.elasticsearch.index.mapper.MappedFieldType;
1715
import org.elasticsearch.index.mapper.MappingLookup;
1816
import org.elasticsearch.index.shard.IndexLongFieldRange;
19-
import org.elasticsearch.indices.DateFieldRangeInfo;
2017
import org.elasticsearch.xcontent.XContentParserConfiguration;
2118

2219
import java.util.Collections;
@@ -26,24 +23,19 @@
2623
* Context object used to rewrite {@link QueryBuilder} instances into simplified version in the coordinator.
2724
* Instances of this object rely on information stored in the {@code IndexMetadata} for certain indices.
2825
* Right now this context object is able to rewrite range queries that include a known timestamp field
29-
* (i.e. the timestamp field for DataStreams or the 'event.ingested' field in ECS) into a MatchNoneQueryBuilder
30-
* and skip the shards that don't hold queried data. See IndexMetadata for more details.
26+
* (i.e. the timestamp field for DataStreams) into a MatchNoneQueryBuilder and skip the shards that
27+
* don't hold queried data. See IndexMetadata#getTimestampRange() for more details
3128
*/
3229
public class CoordinatorRewriteContext extends QueryRewriteContext {
33-
private final DateFieldRangeInfo dateFieldRangeInfo;
30+
private final IndexLongFieldRange indexLongFieldRange;
31+
private final DateFieldMapper.DateFieldType timestampFieldType;
3432

35-
/**
36-
* Context for coordinator search rewrites based on time ranges for the @timestamp field and/or 'event.ingested' field
37-
* @param parserConfig
38-
* @param client
39-
* @param nowInMillis
40-
* @param dateFieldRangeInfo range and field type info for @timestamp and 'event.ingested'
41-
*/
4233
public CoordinatorRewriteContext(
4334
XContentParserConfiguration parserConfig,
4435
Client client,
4536
LongSupplier nowInMillis,
46-
DateFieldRangeInfo dateFieldRangeInfo
37+
IndexLongFieldRange indexLongFieldRange,
38+
DateFieldMapper.DateFieldType timestampFieldType
4739
) {
4840
super(
4941
parserConfig,
@@ -61,98 +53,29 @@ public CoordinatorRewriteContext(
6153
null,
6254
null
6355
);
64-
this.dateFieldRangeInfo = dateFieldRangeInfo;
56+
this.indexLongFieldRange = indexLongFieldRange;
57+
this.timestampFieldType = timestampFieldType;
6558
}
6659

67-
/**
68-
* Get min timestamp for either '@timestamp' or 'event.ingested' fields. Any other field
69-
* passed in will cause an {@link IllegalArgumentException} to be thrown, as these are the only
70-
* two fields supported for coordinator rewrites (based on time range).
71-
* @param fieldName Must be DataStream.TIMESTAMP_FIELD_NAME or IndexMetadata.EVENT_INGESTED_FIELD_NAME
72-
* @return min timestamp for the field from IndexMetadata in cluster state.
73-
*/
74-
long getMinTimestamp(String fieldName) {
75-
if (DataStream.TIMESTAMP_FIELD_NAME.equals(fieldName)) {
76-
return dateFieldRangeInfo.getTimestampRange().getMin();
77-
} else if (IndexMetadata.EVENT_INGESTED_FIELD_NAME.equals(fieldName)) {
78-
return dateFieldRangeInfo.getEventIngestedRange().getMin();
79-
} else {
80-
throw new IllegalArgumentException(
81-
Strings.format(
82-
"Only [%s] or [%s] fields are supported for min timestamp coordinator rewrites, but got: [%s]",
83-
DataStream.TIMESTAMP_FIELD_NAME,
84-
IndexMetadata.EVENT_INGESTED_FIELD_NAME,
85-
fieldName
86-
)
87-
);
88-
}
60+
long getMinTimestamp() {
61+
return indexLongFieldRange.getMin();
8962
}
9063

91-
/**
92-
* Get max timestamp for either '@timestamp' or 'event.ingested' fields. Any other field
93-
* passed in will cause an {@link IllegalArgumentException} to be thrown, as these are the only
94-
* two fields supported for coordinator rewrites (based on time range).
95-
* @param fieldName Must be DataStream.TIMESTAMP_FIELD_NAME or IndexMetadata.EVENT_INGESTED_FIELD_NAME
96-
* @return max timestamp for the field from IndexMetadata in cluster state.
97-
*/
98-
long getMaxTimestamp(String fieldName) {
99-
if (DataStream.TIMESTAMP_FIELD_NAME.equals(fieldName)) {
100-
return dateFieldRangeInfo.getTimestampRange().getMax();
101-
} else if (IndexMetadata.EVENT_INGESTED_FIELD_NAME.equals(fieldName)) {
102-
return dateFieldRangeInfo.getEventIngestedRange().getMax();
103-
} else {
104-
throw new IllegalArgumentException(
105-
Strings.format(
106-
"Only [%s] or [%s] fields are supported for max timestamp coordinator rewrites, but got: [%s]",
107-
DataStream.TIMESTAMP_FIELD_NAME,
108-
IndexMetadata.EVENT_INGESTED_FIELD_NAME,
109-
fieldName
110-
)
111-
);
112-
}
64+
long getMaxTimestamp() {
65+
return indexLongFieldRange.getMax();
11366
}
11467

115-
/**
116-
* Determine whether either '@timestamp' or 'event.ingested' fields has useful timestamp ranges
117-
* stored in cluster state for this context.
118-
* Any other fieldname will cause an {@link IllegalArgumentException} to be thrown, as these are the only
119-
* two fields supported for coordinator rewrites (based on time range).
120-
* @param fieldName Must be DataStream.TIMESTAMP_FIELD_NAME or IndexMetadata.EVENT_INGESTED_FIELD_NAME
121-
* @return min timestamp for the field from IndexMetadata in cluster state.
122-
*/
123-
boolean hasTimestampData(String fieldName) {
124-
if (DataStream.TIMESTAMP_FIELD_NAME.equals(fieldName)) {
125-
return dateFieldRangeInfo.getTimestampRange().isComplete()
126-
&& dateFieldRangeInfo.getTimestampRange() != IndexLongFieldRange.EMPTY;
127-
} else if (IndexMetadata.EVENT_INGESTED_FIELD_NAME.equals(fieldName)) {
128-
return dateFieldRangeInfo.getEventIngestedRange().isComplete()
129-
&& dateFieldRangeInfo.getEventIngestedRange() != IndexLongFieldRange.EMPTY;
130-
} else {
131-
throw new IllegalArgumentException(
132-
Strings.format(
133-
"Only [%s] or [%s] fields are supported for min/max timestamp coordinator rewrites, but got: [%s]",
134-
DataStream.TIMESTAMP_FIELD_NAME,
135-
IndexMetadata.EVENT_INGESTED_FIELD_NAME,
136-
fieldName
137-
)
138-
);
139-
}
68+
boolean hasTimestampData() {
69+
return indexLongFieldRange.isComplete() && indexLongFieldRange != IndexLongFieldRange.EMPTY;
14070
}
14171

142-
/**
143-
* @param fieldName Get MappedFieldType for either '@timestamp' or 'event.ingested' fields.
144-
* @return min timestamp for the field from IndexMetadata in cluster state or null if fieldName was not
145-
* DataStream.TIMESTAMP_FIELD_NAME or IndexMetadata.EVENT_INGESTED_FIELD_NAME.
146-
*/
14772
@Nullable
14873
public MappedFieldType getFieldType(String fieldName) {
149-
if (DataStream.TIMESTAMP_FIELD_NAME.equals(fieldName)) {
150-
return dateFieldRangeInfo.getTimestampFieldType();
151-
} else if (IndexMetadata.EVENT_INGESTED_FIELD_NAME.equals(fieldName)) {
152-
return dateFieldRangeInfo.getEventIngestedFieldType();
153-
} else {
74+
if (fieldName.equals(timestampFieldType.name()) == false) {
15475
return null;
15576
}
77+
78+
return timestampFieldType;
15679
}
15780

15881
@Override

‎server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContextProvider.java‎

Lines changed: 7 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import org.elasticsearch.index.Index;
1515
import org.elasticsearch.index.mapper.DateFieldMapper;
1616
import org.elasticsearch.index.shard.IndexLongFieldRange;
17-
import org.elasticsearch.indices.DateFieldRangeInfo;
1817
import org.elasticsearch.xcontent.XContentParserConfiguration;
1918

2019
import java.util.function.Function;
@@ -26,14 +25,14 @@ public class CoordinatorRewriteContextProvider {
2625
private final Client client;
2726
private final LongSupplier nowInMillis;
2827
private final Supplier<ClusterState> clusterStateSupplier;
29-
private final Function<Index, DateFieldRangeInfo> mappingSupplier;
28+
private final Function<Index, DateFieldMapper.DateFieldType> mappingSupplier;
3029

3130
public CoordinatorRewriteContextProvider(
3231
XContentParserConfiguration parserConfig,
3332
Client client,
3433
LongSupplier nowInMillis,
3534
Supplier<ClusterState> clusterStateSupplier,
36-
Function<Index, DateFieldRangeInfo> mappingSupplier
35+
Function<Index, DateFieldMapper.DateFieldType> mappingSupplier
3736
) {
3837
this.parserConfig = parserConfig;
3938
this.client = client;
@@ -50,33 +49,18 @@ public CoordinatorRewriteContext getCoordinatorRewriteContext(Index index) {
5049
if (indexMetadata == null) {
5150
return null;
5251
}
53-
54-
DateFieldRangeInfo dateFieldRangeInfo = mappingSupplier.apply(index);
55-
if (dateFieldRangeInfo == null) {
52+
DateFieldMapper.DateFieldType dateFieldType = mappingSupplier.apply(index);
53+
if (dateFieldType == null) {
5654
return null;
5755
}
58-
59-
DateFieldMapper.DateFieldType timestampFieldType = dateFieldRangeInfo.getTimestampFieldType();
60-
DateFieldMapper.DateFieldType eventIngestedFieldType = dateFieldRangeInfo.getEventIngestedFieldType();
6156
IndexLongFieldRange timestampRange = indexMetadata.getTimestampRange();
62-
IndexLongFieldRange eventIngestedRange = indexMetadata.getEventIngestedRange();
63-
6457
if (timestampRange.containsAllShardRanges() == false) {
65-
// if @timestamp range is not present or not ready in cluster state, fallback to using time series range (if present)
66-
timestampRange = indexMetadata.getTimeSeriesTimestampRange(timestampFieldType);
67-
// if timestampRange in the time series is null AND the eventIngestedRange is not ready for use, return null (no coord rewrite)
68-
if (timestampRange == null && eventIngestedRange.containsAllShardRanges() == false) {
58+
timestampRange = indexMetadata.getTimeSeriesTimestampRange(dateFieldType);
59+
if (timestampRange == null) {
6960
return null;
7061
}
7162
}
7263

73-
// the DateFieldRangeInfo from the mappingSupplier only has field types, but not ranges
74-
// so create a new object with ranges pulled from cluster state
75-
return new CoordinatorRewriteContext(
76-
parserConfig,
77-
client,
78-
nowInMillis,
79-
new DateFieldRangeInfo(timestampFieldType, timestampRange, eventIngestedFieldType, eventIngestedRange)
80-
);
64+
return new CoordinatorRewriteContext(parserConfig, client, nowInMillis, timestampRange, dateFieldType);
8165
}
8266
}

‎server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -436,11 +436,11 @@ public String getWriteableName() {
436436
protected MappedFieldType.Relation getRelation(final CoordinatorRewriteContext coordinatorRewriteContext) {
437437
final MappedFieldType fieldType = coordinatorRewriteContext.getFieldType(fieldName);
438438
if (fieldType instanceof final DateFieldMapper.DateFieldType dateFieldType) {
439-
if (coordinatorRewriteContext.hasTimestampData(fieldName) == false) {
439+
if (coordinatorRewriteContext.hasTimestampData() == false) {
440440
return MappedFieldType.Relation.DISJOINT;
441441
}
442-
long minTimestamp = coordinatorRewriteContext.getMinTimestamp(fieldName);
443-
long maxTimestamp = coordinatorRewriteContext.getMaxTimestamp(fieldName);
442+
long minTimestamp = coordinatorRewriteContext.getMinTimestamp();
443+
long maxTimestamp = coordinatorRewriteContext.getMaxTimestamp();
444444
DateMathParser dateMathParser = getForceDateParser();
445445
return dateFieldType.isFieldWithinQuery(
446446
minTimestamp,

‎server/src/main/java/org/elasticsearch/indices/DateFieldRangeInfo.java‎

Lines changed: 0 additions & 51 deletions
This file was deleted.

‎server/src/main/java/org/elasticsearch/indices/IndicesService.java‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@
9898
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
9999
import org.elasticsearch.index.flush.FlushStats;
100100
import org.elasticsearch.index.get.GetStats;
101+
import org.elasticsearch.index.mapper.DateFieldMapper;
101102
import org.elasticsearch.index.mapper.IdFieldMapper;
102103
import org.elasticsearch.index.mapper.MapperMetrics;
103104
import org.elasticsearch.index.mapper.MapperRegistry;
@@ -1763,13 +1764,7 @@ public DataRewriteContext getDataRewriteContext(LongSupplier nowInMillis) {
17631764
}
17641765

17651766
public CoordinatorRewriteContextProvider getCoordinatorRewriteContextProvider(LongSupplier nowInMillis) {
1766-
return new CoordinatorRewriteContextProvider(
1767-
parserConfig,
1768-
client,
1769-
nowInMillis,
1770-
clusterService::state,
1771-
this::getTimestampFieldTypeInfo
1772-
);
1767+
return new CoordinatorRewriteContextProvider(parserConfig, client, nowInMillis, clusterService::state, this::getTimestampFieldType);
17731768
}
17741769

17751770
/**
@@ -1859,16 +1854,14 @@ public boolean allPendingDanglingIndicesWritten() {
18591854
}
18601855

18611856
/**
1862-
* @return DateFieldRangeInfo holding the field types of the {@code @timestamp} and {@code event.ingested} fields of the index.
1863-
* or {@code null} if:
1857+
* @return the field type of the {@code @timestamp} field of the given index, or {@code null} if:
18641858
* - the index is not found,
18651859
* - the field is not found, or
1866-
* - the mapping is not known yet, or
1867-
* - the index does not have a useful timestamp field.
1860+
* - the field is not a timestamp field.
18681861
*/
18691862
@Nullable
1870-
public DateFieldRangeInfo getTimestampFieldTypeInfo(Index index) {
1871-
return timestampFieldMapperService.getTimestampFieldTypeMap(index);
1863+
public DateFieldMapper.DateFieldType getTimestampFieldType(Index index) {
1864+
return timestampFieldMapperService.getTimestampFieldType(index);
18721865
}
18731866

18741867
public IndexScopedSettings getIndexScopedSettings() {

0 commit comments

Comments
 (0)