Skip to content

Remove default mapping #44945

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
romseygeek merged 7 commits into from
Sep 20, 2019
Merged

Remove default mapping #44945

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
510f41b
Remove default mapping
romseygeek Jul 29, 2019
4edc640
precommit
romseygeek Jul 29, 2019
34a89f9
Merge remote-tracking branch 'origin/master' into default-mappings
romseygeek Sep 11, 2019
00e1e8f
feedback + merge master
romseygeek Sep 11, 2019
506e531
Remove final _default_ reference
romseygeek Sep 12, 2019
7b3d0f2
Merge branch 'master' into default-mappings
elasticmachine Sep 14, 2019
f44f9d5
Merge branch 'master' into default-mappings
elasticmachine Sep 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 2 additions & 6 deletions server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
package org.elasticsearch.action.admin.indices.get;

import com.carrotsearch.hppc.cursors.ObjectObjectCursor;

import org.apache.lucene.util.CollectionUtil;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.cluster.metadata.AliasMetaData;
Expand All @@ -34,7 +33,6 @@
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.index.mapper.MapperService;

import java.io.IOException;
import java.util.ArrayList;
Expand Down Expand Up @@ -258,10 +256,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
} else {
MappingMetaData mappings = null;
for (final ObjectObjectCursor<String, MappingMetaData> typeEntry : indexMappings) {
if (typeEntry.key.equals(MapperService.DEFAULT_MAPPING) == false) {
assert mappings == null;
mappings = typeEntry.value;
}
assert mappings == null;
mappings = typeEntry.value;
}
if (mappings == null) {
// no mappings yet
Expand Down
7 changes: 2 additions & 5 deletions ...ain/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.rest.BaseRestHandler;

import java.io.IOException;
Expand Down Expand Up @@ -150,10 +149,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (includeTypeName == false) {
Map<String, FieldMappingMetaData> mappings = null;
for (Map.Entry<String, Map<String, FieldMappingMetaData>> typeEntry : indexEntry.getValue().entrySet()) {
if (typeEntry.getKey().equals(MapperService.DEFAULT_MAPPING) == false) {
assert mappings == null;
mappings = typeEntry.getValue();
}
assert mappings == null;
mappings = typeEntry.getValue();
}
if (mappings != null) {
addFieldMappingsToBuilder(builder, params, mappings);
Expand Down
6 changes: 2 additions & 4 deletions ...src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (includeTypeName == false) {
MappingMetaData mappings = null;
for (final ObjectObjectCursor<String, MappingMetaData> typeEntry : indexEntry.value) {
if (typeEntry.key.equals("_default_") == false) {
assert mappings == null;
mappings = typeEntry.value;
}
assert mappings == null;
mappings = typeEntry.value;
}
if (mappings == null) {
// no mappings yet
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ protected void doRun() {
case INDEX:
IndexRequest indexRequest = (IndexRequest) docWriteRequest;
final IndexMetaData indexMetaData = metaData.index(concreteIndex);
MappingMetaData mappingMd = indexMetaData.mappingOrDefault();
MappingMetaData mappingMd = indexMetaData.mapping();
Version indexCreated = indexMetaData.getCreationVersion();
indexRequest.resolveRouting(metaData);
indexRequest.process(indexCreated, mappingMd, concreteIndex.getName());
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ static boolean executeBulkItemRequest(BulkPrimaryExecutionContext context, Updat
case UPDATED:
IndexRequest indexRequest = updateResult.action();
IndexMetaData metaData = context.getPrimary().indexSettings().getIndexMetaData();
MappingMetaData mappingMd = metaData.mappingOrDefault();
MappingMetaData mappingMd = metaData.mapping();
indexRequest.process(metaData.getCreationVersion(), mappingMd, updateRequest.concreteIndex());
context.setRequestToExecute(indexRequest);
break;
Expand Down
4 changes: 0 additions & 4 deletions server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
import org.elasticsearch.common.util.concurrent.FutureUtils;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.Mapping;

/**
Expand Down Expand Up @@ -71,9 +70,6 @@ public void setClient(Client client) {
* potentially waiting for a master node to be available.
*/
public void updateMappingOnMaster(Index index, String type, Mapping mappingUpdate, ActionListener<Void> listener) {
if (type.equals(MapperService.DEFAULT_MAPPING)) {
throw new IllegalArgumentException("_default_ mapping should not be updated");
}
client.preparePutMapping().setConcreteIndex(index).setType(type).setSource(mappingUpdate.toString(), XContentType.JSON)
.setMasterNodeTimeout(dynamicMappingUpdateTimeout).setTimeout(TimeValue.ZERO)
.execute(new ActionListener<AcknowledgedResponse>() {
Expand Down
41 changes: 1 addition & 40 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.gateway.MetaDataStateFormat;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.seqno.SequenceNumbers;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.rest.RestStatus;
Expand Down Expand Up @@ -496,22 +495,11 @@ public ImmutableOpenMap<String, MappingMetaData> getMappings() {
@Nullable
public MappingMetaData mapping() {
for (ObjectObjectCursor<String, MappingMetaData> cursor : mappings) {
if (cursor.key.equals(MapperService.DEFAULT_MAPPING) == false) {
return cursor.value;
}
return cursor.value;
}
return null;
}

/**
* Get the default mapping.
* NOTE: this is always {@code null} for 7.x indices which are disallowed to have a default mapping.
*/
@Nullable
public MappingMetaData defaultMapping() {
return mappings.get(MapperService.DEFAULT_MAPPING);
}

public static final String INDEX_RESIZE_SOURCE_UUID_KEY = "index.resize.source.uuid";
public static final String INDEX_RESIZE_SOURCE_NAME_KEY = "index.resize.source.name";
public static final Setting<String> INDEX_RESIZE_SOURCE_UUID = Setting.simpleString(INDEX_RESIZE_SOURCE_UUID_KEY);
Expand All @@ -522,25 +510,6 @@ public Index getResizeSourceIndex() {
INDEX_RESIZE_SOURCE_UUID.get(settings)) : null;
}

/**
* Sometimes, the default mapping exists and an actual mapping is not created yet (introduced),
* in this case, we want to return the default mapping in case it has some default mapping definitions.
* <p>
* Note, once the mapping type is introduced, the default mapping is applied on the actual typed MappingMetaData,
* setting its routing, timestamp, and so on if needed.
*/
@Nullable
public MappingMetaData mappingOrDefault() {
MappingMetaData mapping = null;
for (ObjectCursor<MappingMetaData> m : mappings.values()) {
if (mapping == null || mapping.type().equals(MapperService.DEFAULT_MAPPING)) {
mapping = m.value;
}
}

return mapping;
}

ImmutableOpenMap<String, DiffableStringMap> getCustomData() {
return this.customData;
}
Expand Down Expand Up @@ -1102,14 +1071,6 @@ public IndexMetaData build() {
ImmutableOpenMap.Builder<String, AliasMetaData> tmpAliases = aliases;
Settings tmpSettings = settings;

// update default mapping on the MappingMetaData
if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) {
MappingMetaData defaultMapping = mappings.get(MapperService.DEFAULT_MAPPING);
for (ObjectCursor<MappingMetaData> cursor : mappings.values()) {
cursor.value.updateDefaultMapping(defaultMapping);
Copy link
Copy Markdown
Contributor

@jtibshirani jtibshirani Jul 30, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we can nuke the updateDefaultMapping method now that this block is deleted.

}
}

Integer maybeNumberOfShards = settings.getAsInt(SETTING_NUMBER_OF_SHARDS, null);
if (maybeNumberOfShards == null) {
throw new IllegalArgumentException("must specify numberOfShards for index [" + index + "]");
Expand Down
11 changes: 4 additions & 7 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.MapperService;

import java.io.IOException;
import java.util.ArrayList;
Expand Down Expand Up @@ -378,12 +377,10 @@ private static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData,
if (includeTypeName == false) {
Map<String, Object> documentMapping = null;
for (ObjectObjectCursor<String, CompressedXContent> cursor : indexTemplateMetaData.mappings()) {
if (!cursor.key.equals(MapperService.DEFAULT_MAPPING)) {
assert documentMapping == null;
byte[] mappingSource = cursor.value.uncompressed();
Map<String, Object> mapping = XContentHelper.convertToMap(new BytesArray(mappingSource), true).v2();
documentMapping = reduceMapping(cursor.key, mapping);
}
assert documentMapping == null;
byte[] mappingSource = cursor.value.uncompressed();
Map<String, Object> mapping = XContentHelper.convertToMap(new BytesArray(mappingSource), true).v2();
documentMapping = reduceMapping(cursor.key, mapping);
}

if (documentMapping != null) {
Expand Down
6 changes: 0 additions & 6 deletions server/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,12 +126,6 @@ private void initMappers(Map<String, Object> withoutType) {
}
}

void updateDefaultMapping(MappingMetaData defaultMapping) {
if (routing == Routing.EMPTY) {
routing = defaultMapping.routing();
}
}

public String type() {
return this.type;
}
Expand Down
14 changes: 5 additions & 9 deletions server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@
import java.nio.file.Path;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -519,12 +518,10 @@ public ClusterState execute(ClusterState currentState) throws Exception {

// now, update the mappings with the actual source
Map<String, MappingMetaData> mappingsMetaData = new HashMap<>();
for (DocumentMapper mapper : Arrays.asList(mapperService.documentMapper(),
mapperService.documentMapper(MapperService.DEFAULT_MAPPING))) {
if (mapper != null) {
MappingMetaData mappingMd = new MappingMetaData(mapper);
mappingsMetaData.put(mapper.type(), mappingMd);
}
DocumentMapper mapper = mapperService.documentMapper();
if (mapper != null) {
MappingMetaData mappingMd = new MappingMetaData(mapper);
mappingsMetaData.put(mapper.type(), mappingMd);
}

final IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(request.index())
Expand Down Expand Up @@ -737,8 +734,7 @@ static IndexMetaData validateResize(ClusterState state, String sourceIndex,
throw new IllegalStateException("index " + sourceIndex + " must be read-only to resize index. use \"index.blocks.write=true\"");
}

if ((targetIndexMappingsTypes.size() > 1 ||
(targetIndexMappingsTypes.isEmpty() || targetIndexMappingsTypes.contains(MapperService.DEFAULT_MAPPING)) == false)) {
if (targetIndexMappingsTypes.size() > 0) {
throw new IllegalArgumentException("mappings are not allowed when resizing indices" +
", all mappings are copied from the source index");
}
Expand Down
48 changes: 18 additions & 30 deletions server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,26 +177,22 @@ private boolean refreshIndexMapping(IndexService indexService, IndexMetaData.Bui
try {
List<String> updatedTypes = new ArrayList<>();
MapperService mapperService = indexService.mapperService();
for (DocumentMapper mapper : Arrays.asList(mapperService.documentMapper(),
mapperService.documentMapper(MapperService.DEFAULT_MAPPING))) {
if (mapper != null) {
final String type = mapper.type();
if (!mapper.mappingSource().equals(builder.mapping(type).source())) {
updatedTypes.add(type);
}
DocumentMapper mapper = mapperService.documentMapper();
if (mapper != null) {
final String type = mapper.type();
if (!mapper.mappingSource().equals(builder.mapping(type).source())) {
updatedTypes.add(type);
}
}

// if a single type is not up-to-date, re-send everything
if (updatedTypes.isEmpty() == false) {
logger.warn("[{}] re-syncing mappings with cluster state because of types [{}]", index, updatedTypes);
dirty = true;
for (DocumentMapper mapper : Arrays.asList(mapperService.documentMapper(),
mapperService.documentMapper(MapperService.DEFAULT_MAPPING))) {
if (mapper != null) {
builder.putMapping(new MappingMetaData(mapper));
}
if (mapper != null) {
builder.putMapping(new MappingMetaData(mapper));
}

}
} catch (Exception e) {
logger.warn(() -> new ParameterizedMessage("[{}] failed to refresh-mapping in cluster state", index), e);
Expand Down Expand Up @@ -263,25 +259,20 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt
// we used for the validation, it makes this mechanism little less scary (a little)
updateList.add(indexMetaData);
// try and parse it (no need to add it here) so we can bail early in case of parsing exception
DocumentMapper newMapper;
DocumentMapper existingMapper = mapperService.documentMapper();

String typeForUpdate = mapperService.getTypeForUpdate(mappingType, mappingUpdateSource);
if (existingMapper != null && existingMapper.type().equals(typeForUpdate) == false) {
throw new IllegalArgumentException("Rejecting mapping update to [" + mapperService.index().getName() +
"] as the final mapping would have more than 1 type: " + Arrays.asList(existingMapper.type(), typeForUpdate));
}

if (MapperService.DEFAULT_MAPPING.equals(request.type())) {
// _default_ types do not go through merging, but we do test the new settings. Also don't apply the old default
newMapper = mapperService.parse(request.type(), mappingUpdateSource, false);
} else {
newMapper = mapperService.parse(request.type(), mappingUpdateSource, existingMapper == null);
if (existingMapper != null) {
// first, simulate: just call merge and ignore the result
existingMapper.merge(newMapper.mapping());
}
DocumentMapper newMapper = mapperService.parse(request.type(), mappingUpdateSource);
if (existingMapper != null) {
// first, simulate: just call merge and ignore the result
existingMapper.merge(newMapper.mapping());
}


if (mappingType == null) {
mappingType = newMapper.type();
} else if (mappingType.equals(newMapper.type()) == false
Expand All @@ -292,8 +283,7 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt
}
assert mappingType != null;

if (MapperService.DEFAULT_MAPPING.equals(mappingType) == false
&& MapperService.SINGLE_MAPPING_NAME.equals(mappingType) == false
if (MapperService.SINGLE_MAPPING_NAME.equals(mappingType) == false
&& mappingType.charAt(0) == '_') {
throw new InvalidTypeNameException("Document mapping type name can't start with '_', found: [" + mappingType + "]");
}
Expand Down Expand Up @@ -340,11 +330,9 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt
IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(indexMetaData);
// Mapping updates on a single type may have side-effects on other types so we need to
// update mapping metadata on all types
for (DocumentMapper mapper : Arrays.asList(mapperService.documentMapper(),
mapperService.documentMapper(MapperService.DEFAULT_MAPPING))) {
if (mapper != null) {
indexMetaDataBuilder.putMapping(new MappingMetaData(mapper.mappingSource()));
}
DocumentMapper mapper = mapperService.documentMapper();
if (mapper != null) {
indexMetaDataBuilder.putMapping(new MappingMetaData(mapper.mappingSource()));
}
if (updatedMapping) {
indexMetaDataBuilder.mappingVersion(1 + indexMetaDataBuilder.mappingVersion());
Expand Down
16 changes: 2 additions & 14 deletions server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,6 @@ public Mapper.TypeParser.ParserContext parserContext() {
}

public DocumentMapper parse(@Nullable String type, CompressedXContent source) throws MapperParsingException {
return parse(type, source, null);
}

public DocumentMapper parse(@Nullable String type, CompressedXContent source, String defaultSource) throws MapperParsingException {
Map<String, Object> mapping = null;
if (source != null) {
Map<String, Object> root = XContentHelper.convertToMap(source.compressedReference(), true, XContentType.JSON).v2();
Expand All @@ -84,23 +80,15 @@ public DocumentMapper parse(@Nullable String type, CompressedXContent source, St
if (mapping == null) {
mapping = new HashMap<>();
}
return parse(type, mapping, defaultSource);
return parse(type, mapping);
}

@SuppressWarnings({"unchecked"})
private DocumentMapper parse(String type, Map<String, Object> mapping, String defaultSource) throws MapperParsingException {
private DocumentMapper parse(String type, Map<String, Object> mapping) throws MapperParsingException {
if (type == null) {
throw new MapperParsingException("Failed to derive type");
}

if (defaultSource != null) {
Tuple<String, Map<String, Object>> t = extractMapping(MapperService.DEFAULT_MAPPING, defaultSource);
if (t.v2() != null) {
XContentHelper.mergeDefaults(mapping, t.v2());
}
}


Mapper.TypeParser.ParserContext parserContext = parserContext();
// parse RootObjectMapper
DocumentMapper.Builder docBuilder = new DocumentMapper.Builder(
Expand Down
Loading