diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml index 3b7ea15164e9f..bbd282d805964 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml @@ -176,3 +176,106 @@ # This should failed with 400 as half_float is not supported for index sort - match: { status: 400 } - match: { error.type: illegal_argument_exception } + +--- +"Index sort with nested fields": + - skip: + version: " - 3.1.0" + reason: "Index sort on nested field is only supported after 3.1.0" + - do: + indices.create: + index: test_nested_sort + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + index.sort.field: foo + index.sort.order: desc + mappings: + properties: + foo: + type: integer + foo1: + type: keyword + contacts: + type: nested + properties: + name: + type: keyword + age: + type: integer + + - do: + index: + index: test_nested_sort + id: "1" + body: + foo: 100 + foo1: "A" + contacts: + - name: "Alice" + age: 30 + + - do: + index: + index: test_nested_sort + id: "2" + body: + foo: 200 + foo1: "B" + contacts: + - name: "Bob" + age: 40 + + - do: + index: + index: test_nested_sort + id: "3" + body: + foo: 150 + foo1: "C" + contacts: + - name: "Charlie" + age: 25 + + - do: + indices.refresh: + index: test_nested_sort + + - do: + search: + index: test_nested_sort + body: + sort: + - foo: desc + size: 3 + + - match: { hits.total.value: 3 } + - match: { hits.hits.0._id: "2" } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.2._id: "1" } + +--- +"Index sort with nested field as sort field validation": + - skip: + version: " - 3.1.0" + reason: "Index sort on nested field is only supported after 3.1.0" + - do: + catch: bad_request + indices.create: + index: test_nested_sort + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + index.sort.field: contacts.age + mappings: + properties: + contacts: + type: nested + properties: + age: + type: integer + - match: { status: 400 } + - match: { error.type: illegal_argument_exception } + - match: { error.reason: "index sorting on nested fields is not supported: found nested sort field [contacts.age] in [test_nested_sort]" } diff --git a/server/src/internalClusterTest/java/org/opensearch/index/IndexSortIT.java b/server/src/internalClusterTest/java/org/opensearch/index/IndexSortIT.java index 369c9f9b1a653..fe39d87ca24ef 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/IndexSortIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/IndexSortIT.java @@ -36,15 +36,26 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSelector; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSortField; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.sort.SortOrder; import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; @@ -52,6 +63,7 @@ public class IndexSortIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { private static final XContentBuilder TEST_MAPPING = createTestMapping(); + private static final XContentBuilder NESTED_TEST_MAPPING = createNestedTestMapping(); public IndexSortIT(Settings staticSettings) { super(staticSettings); @@ -95,6 +107,49 @@ private static XContentBuilder createTestMapping() { } } + private static XContentBuilder createNestedTestMapping() { + try { + return jsonBuilder().startObject() + .startObject("properties") + .startObject("foo") + .field("type", "integer") + .endObject() + .startObject("foo1") + .field("type", "keyword") + .endObject() + .startObject("contacts") + .field("type", "nested") + .startObject("properties") + .startObject("name") + .field("type", "keyword") + .endObject() + .startObject("age") + .field("type", "integer") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private static void addNestedDocuments(String id, int foo, String foo1, String name, int age) throws IOException { + XContentBuilder sourceBuilder = jsonBuilder().startObject() + .field("foo", foo) + .field("foo1", foo1) + .startArray("contacts") + .startObject() + .field("name", name) + .field("age", age) + .endObject() + .endArray() + .endObject(); + + client().prepareIndex("nested-test-index").setId(id).setSource(sourceBuilder).get(); + } + public void testIndexSort() { SortField dateSort = new SortedNumericSortField("date", SortField.Type.LONG, false); dateSort.setMissingValue(Long.MAX_VALUE); @@ -146,4 +201,141 @@ public void testInvalidIndexSort() { ); assertThat(exc.getMessage(), containsString("docvalues not found for index sort field:[keyword]")); } + + public void testIndexSortOnNestedField() throws IOException { + boolean ascending = randomBoolean(); + SortedNumericSelector.Type selector = ascending ? SortedNumericSelector.Type.MIN : SortedNumericSelector.Type.MAX; + SortField regularSort = new SortedNumericSortField("foo", SortField.Type.INT, !ascending, selector); + regularSort.setMissingValue(ascending ? Integer.MAX_VALUE : Integer.MIN_VALUE); + + Sort indexSort = new Sort(regularSort); + + prepareCreate("nested-test-index").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "0") + .putList("index.sort.field", "foo") + .putList("index.sort.order", ascending ? "asc" : "desc") + ).setMapping(NESTED_TEST_MAPPING).get(); + + int numDocs = randomIntBetween(10, 30); + List fooValues = new ArrayList<>(numDocs); + List ids = new ArrayList<>(numDocs); + + for (int i = 0; i < numDocs; i++) { + String id = String.valueOf(i); + int fooValue = randomIntBetween(1, 100); + String name = UUID.randomUUID().toString().replace("-", "").substring(0, 5); + + addNestedDocuments(id, fooValue, "", name, fooValue); + fooValues.add(fooValue); + ids.add(id); + } + + flushAndRefresh("nested-test-index"); + ensureGreen("nested-test-index"); + + assertSortedSegments("nested-test-index", indexSort); + + SearchResponse response = client().prepareSearch("nested-test-index") + .addSort("foo", ascending ? SortOrder.ASC : SortOrder.DESC) + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(numDocs) + .get(); + + assertEquals(numDocs, response.getHits().getTotalHits().value()); + + Map valueToId = new HashMap<>(); + for (int i = 0; i < numDocs; i++) { + valueToId.put(fooValues.get(i), ids.get(i)); + } + + List sortedValues = new ArrayList<>(fooValues); + if (ascending) { + Collections.sort(sortedValues); + } else { + sortedValues.sort(Collections.reverseOrder()); + } + + for (int i = 0; i < numDocs; i++) { + int expectedValue = sortedValues.get(i); + assertEquals(expectedValue, response.getHits().getAt(i).getSourceAsMap().get("foo")); + } + } + + public void testIndexSortWithNestedField_MultiField() throws IOException { + boolean ascendingPrimary = randomBoolean(); + boolean ascendingSecondary = randomBoolean(); + prepareCreate("nested-test-index").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "0") + .putList("index.sort.field", "foo", "foo1") + .putList("index.sort.order", ascendingPrimary ? "asc" : "desc", ascendingSecondary ? "asc" : "desc") + ).setMapping(NESTED_TEST_MAPPING).get(); + + int numDocs = randomIntBetween(10, 30); + List> docValues = new ArrayList<>(numDocs); + List ids = new ArrayList<>(numDocs); + + int duplicateValue = randomIntBetween(30, 50); + int numDuplicates = randomIntBetween(3, 5); + + for (int i = 0; i < numDocs; i++) { + String id = String.valueOf(i); + int fooValue; + if (i < numDuplicates) { + fooValue = duplicateValue; + } else { + fooValue = randomIntBetween(1, 100); + } + String name = UUID.randomUUID().toString().replace("-", "").substring(0, 5); + addNestedDocuments(id, fooValue, name, name, fooValue); + docValues.add(new Tuple<>(fooValue, name)); + ids.add(id); + } + + flushAndRefresh("nested-test-index"); + ensureGreen("nested-test-index"); + SearchResponse response = client().prepareSearch("nested-test-index") + .addSort("foo", ascendingPrimary ? SortOrder.ASC : SortOrder.DESC) + .addSort("foo1", ascendingSecondary ? SortOrder.ASC : SortOrder.DESC) + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(numDocs) + .get(); + + assertEquals(numDocs, response.getHits().getTotalHits().value()); + + List> sortedValues = new ArrayList<>(docValues); + sortedValues.sort((a, b) -> { + int primaryCompare = ascendingPrimary ? Integer.compare(a.v1(), b.v1()) : Integer.compare(b.v1(), a.v1()); + if (primaryCompare != 0) { + return primaryCompare; + } + return ascendingSecondary ? a.v2().compareTo(b.v2()) : b.v2().compareTo(a.v2()); + }); + + for (int i = 0; i < numDocs; i++) { + assertEquals(sortedValues.get(i).v1(), response.getHits().getAt(i).getSourceAsMap().get("foo")); + assertEquals(sortedValues.get(i).v2(), response.getHits().getAt(i).getSourceAsMap().get("foo1")); + } + } + + public void testIndexSortWithSortFieldInsideDocBlock() { + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("nested-sort-test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "0") + .putList("index.sort.field", "contacts.age") + .putList("index.sort.order", "desc") + ).setMapping(NESTED_TEST_MAPPING).get() + ); + + assertThat(exception.getMessage(), containsString("index sorting on nested fields is not supported")); + } } diff --git a/server/src/main/java/org/opensearch/common/lucene/Lucene.java b/server/src/main/java/org/opensearch/common/lucene/Lucene.java index 47427b08b6207..0dcb95ed9a9de 100644 --- a/server/src/main/java/org/opensearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/opensearch/common/lucene/Lucene.java @@ -113,6 +113,7 @@ public class Lucene { public static final String LATEST_CODEC = "Lucene101"; public static final String SOFT_DELETES_FIELD = "__soft_deletes"; + public static final String PARENT_FIELD = "__nested_parent"; public static final NamedAnalyzer STANDARD_ANALYZER = new NamedAnalyzer("_standard", AnalyzerScope.GLOBAL, new StandardAnalyzer()); public static final NamedAnalyzer KEYWORD_ANALYZER = new NamedAnalyzer("_keyword", AnalyzerScope.GLOBAL, new KeywordAnalyzer()); diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index 62bfd27516964..816d850c42d8d 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -66,6 +66,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.InfoStream; import org.opensearch.ExceptionsHelper; +import org.opensearch.Version; import org.opensearch.action.index.IndexRequest; import org.opensearch.common.Booleans; import org.opensearch.common.Nullable; @@ -2379,6 +2380,9 @@ private IndexWriterConfig getIndexWriterConfig() { iwc.setUseCompoundFile(engineConfig.useCompoundFile()); if (config().getIndexSort() != null) { iwc.setIndexSort(config().getIndexSort()); + if (config().getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_3_2_0)) { + iwc.setParentField(Lucene.PARENT_FIELD); + } } if (config().getLeafSorter() != null) { iwc.setLeafSorter(config().getLeafSorter()); // The default segment search order diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java index 039d8c7044a0b..cb7e08f062d6d 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java @@ -38,6 +38,7 @@ import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BytesRef; import org.opensearch.OpenSearchGenerationException; +import org.opensearch.Version; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; @@ -48,6 +49,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; +import org.opensearch.index.IndexSortConfig; import org.opensearch.index.analysis.IndexAnalyzers; import org.opensearch.index.mapper.MapperService.MergeReason; import org.opensearch.index.mapper.MetadataFieldMapper.TypeParser; @@ -58,6 +60,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Stream; @@ -328,9 +331,39 @@ public void validate(IndexSettings settings, boolean checkLimits) { ); } } + + // Indexing Sort with Nested Fields is only supported on & after Version 3.2.0 if (settings.getIndexSortConfig().hasIndexSort() && hasNestedObjects()) { - throw new IllegalArgumentException("cannot have nested fields when index sort is activated"); + if (settings.getIndexVersionCreated().before(Version.V_3_2_0)) { + throw new IllegalArgumentException("cannot have nested fields when index sort is activated"); + } + + /* + * Index sorting works for regular fields across documents that may contain nested objects, + * but sorting on fields inside nested objects is not supported. This validation checks + * the index sort configuration and throws an exception if any sort field is inside + * a nested object. + */ + List sortFields = settings.getValue(IndexSortConfig.INDEX_SORT_FIELD_SETTING); + for (String sortField : sortFields) { + Mapper mapper = this.fieldMappers.getMapper(sortField); + if (mapper != null && mapper.name().contains(".")) { + String parentPath = mapper.name().substring(0, mapper.name().lastIndexOf('.')); + ObjectMapper nestedParent = objectMappers().get(parentPath); + if (nestedParent != null && nestedParent.nested().isNested()) { + throw new IllegalArgumentException( + "index sorting on nested fields is not supported: " + + "found nested sort field [" + + sortField + + "] in [" + + settings.getIndex().getName() + + "]" + ); + } + } + } } + if (checkLimits) { this.fieldMappers.checkLimits(settings); } diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index 8d572f95c384e..8f99efe502858 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -43,6 +43,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.opensearch.ExceptionsHelper; +import org.opensearch.Version; import org.opensearch.action.StepListener; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; @@ -179,6 +180,9 @@ void recoverFromLocalShards( final Directory directory = indexShard.store().directory(); // don't close this directory!! final Directory[] sources = shards.stream().map(LocalShardSnapshot::getSnapshotDirectory).toArray(Directory[]::new); final long maxSeqNo = shards.stream().mapToLong(LocalShardSnapshot::maxSeqNo).max().getAsLong(); + final boolean isParentFieldEnabledVersion = indexShard.indexSettings() + .getIndexVersionCreated() + .onOrAfter(Version.V_3_2_0); final long maxUnsafeAutoIdTimestamp = shards.stream() .mapToLong(LocalShardSnapshot::maxUnsafeAutoIdTimestamp) .max() @@ -191,6 +195,7 @@ void recoverFromLocalShards( maxSeqNo, maxUnsafeAutoIdTimestamp, indexShard.indexSettings().getIndexMetadata(), + isParentFieldEnabledVersion, indexShard.shardId().id(), isSplit, hasNested @@ -220,6 +225,7 @@ void addIndices( final long maxSeqNo, final long maxUnsafeAutoIdTimestamp, IndexMetadata indexMetadata, + final boolean isParentFieldEnabledVersion, int shardId, boolean split, boolean hasNested @@ -240,6 +246,9 @@ void addIndices( .setIndexCreatedVersionMajor(luceneIndexCreatedVersionMajor); if (indexSort != null) { iwc.setIndexSort(indexSort); + if (isParentFieldEnabledVersion) { + iwc.setParentField(Lucene.PARENT_FIELD); + } } try (IndexWriter writer = new IndexWriter(new StatsDirectoryWrapper(hardLinkOrCopyTarget, indexRecoveryStats), iwc)) { diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java index 7d8525b0ff400..53bbb6e298991 100644 --- a/server/src/main/java/org/opensearch/index/store/Store.java +++ b/server/src/main/java/org/opensearch/index/store/Store.java @@ -177,6 +177,8 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref private final ShardLock shardLock; private final OnClose onClose; private final ShardPath shardPath; + private final boolean isParentFieldEnabledVersion; + private final boolean isIndexSortEnabled; // used to ref count files when a new Reader is opened for PIT/Scroll queries // prevents segment files deletion until the PIT/Scroll expires or is discarded @@ -209,6 +211,8 @@ public Store( this.shardLock = shardLock; this.onClose = onClose; this.shardPath = shardPath; + this.isIndexSortEnabled = indexSettings.getIndexSortConfig().hasIndexSort(); + this.isParentFieldEnabledVersion = indexSettings.getIndexVersionCreated().onOrAfter(org.opensearch.Version.V_3_2_0); assert onClose != null; assert shardLock != null; assert shardLock.getShardId().equals(shardId); @@ -1919,23 +1923,27 @@ private static Map getUserData(IndexWriter writer) { return userData; } - private static IndexWriter newAppendingIndexWriter(final Directory dir, final IndexCommit commit) throws IOException { + private IndexWriter newAppendingIndexWriter(final Directory dir, final IndexCommit commit) throws IOException { IndexWriterConfig iwc = newIndexWriterConfig().setIndexCommit(commit).setOpenMode(IndexWriterConfig.OpenMode.APPEND); return new IndexWriter(dir, iwc); } - private static IndexWriter newEmptyIndexWriter(final Directory dir, final Version luceneVersion) throws IOException { + private IndexWriter newEmptyIndexWriter(final Directory dir, final Version luceneVersion) throws IOException { IndexWriterConfig iwc = newIndexWriterConfig().setOpenMode(IndexWriterConfig.OpenMode.CREATE) .setIndexCreatedVersionMajor(luceneVersion.major); return new IndexWriter(dir, iwc); } - private static IndexWriterConfig newIndexWriterConfig() { - return new IndexWriterConfig(null).setSoftDeletesField(Lucene.SOFT_DELETES_FIELD) + private IndexWriterConfig newIndexWriterConfig() { + final IndexWriterConfig iwc = new IndexWriterConfig(null).setSoftDeletesField(Lucene.SOFT_DELETES_FIELD) .setCommitOnClose(false) // we don't want merges to happen here - we call maybe merge on the engine // later once we stared it up otherwise we would need to wait for it here // we also don't specify a codec here and merges should use the engines for this index .setMergePolicy(NoMergePolicy.INSTANCE); + if (this.isIndexSortEnabled && this.isParentFieldEnabledVersion) { + iwc.setParentField(Lucene.PARENT_FIELD); + } + return iwc; } } diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index fa501b155e96b..517e3412958a1 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -115,6 +115,7 @@ import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.concurrent.ReleasableLock; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesArray; @@ -123,6 +124,7 @@ import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.VersionType; import org.opensearch.index.codec.CodecService; @@ -477,6 +479,71 @@ public void testSegmentsWithIndexSort() throws Exception { } } + public void testSegmentsWithNestedFieldIndexSort() throws Exception { + Sort indexSort = new Sort(new SortedSetSortField("foo1", false)); + try ( + Store store = createStore(); + Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null, null, null, indexSort, null) + ) { + List segments = engine.segments(true); + assertThat(segments.isEmpty(), equalTo(true)); + + List docs = List.of( + createDocumentWithNestedField("1", "Alice", 30), + createDocumentWithNestedField("2", "Bob", 25), + createDocumentWithNestedField("3", "Charlie", 35) + ); + for (ParsedDocument doc : docs) { + engine.index(indexForDoc(doc)); + } + engine.refresh("test"); + segments = engine.segments(true); + + assertThat(segments.size(), equalTo(1)); + assertThat(segments.get(0).getSegmentSort(), equalTo(indexSort)); + + } + } + + public void testSegmentsWithNestedFieldIndexSortWithMerge() throws Exception { + Sort indexSort = new Sort(new SortedSetSortField("foo1", false)); + try ( + Store store = createStore(); + Engine engine = createEngine( + defaultSettings, + store, + createTempDir(), + new TieredMergePolicy(), + null, + null, + null, + indexSort, + null + ) + ) { + List docs = List.of( + createDocumentWithNestedField("1", "Alice", 30), + createDocumentWithNestedField("2", "Bob", 25), + createDocumentWithNestedField("3", "Charlie", 35) + ); + for (ParsedDocument doc : docs) { + engine.index(indexForDoc(doc)); + engine.refresh("test"); + } + List preMergeSegments = engine.segments(true); + assertThat(preMergeSegments.size(), equalTo(3)); + for (Segment segment : preMergeSegments) { + assertThat(segment.getSegmentSort(), equalTo(indexSort)); + } + engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID()); + engine.refresh("test"); + List mergedSegments = engine.segments(true); + + assertThat(mergedSegments.size(), equalTo(1)); + assertThat(mergedSegments.get(0).getSegmentSort(), equalTo(indexSort)); + } + } + public void testSegmentsStatsIncludingFileSizes() throws Exception { try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) { assertThat(engine.segmentsStats(true, false).getFileSizes().size(), equalTo(0)); @@ -8224,4 +8291,27 @@ public void testGetMaxSeqNoFromSegmentInfosConcurrentWrites() throws IOException store.close(); engine.close(); } + + private ParsedDocument createDocumentWithNestedField(String id, String contactName, int contactAge) { + BytesReference source = null; + try { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field("foo", 123) + .field("foo1", contactAge) + .startArray("contacts") + .startObject() + .field("name", contactName) + .field("age", contactAge) + .endObject() + .endArray() + .endObject(); + source = BytesReference.bytes(builder); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return testParsedDocument(id, null, testDocumentWithTextField(), source, null); + } + } diff --git a/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java index b9c8866e69cbb..bc0fa4e96d011 100644 --- a/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java @@ -33,6 +33,8 @@ package org.opensearch.index.mapper; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.search.Sort; +import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; @@ -68,6 +70,7 @@ import java.util.Map; import java.util.function.Function; +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.instanceOf; @@ -81,6 +84,11 @@ protected Collection> getPlugins() { return Arrays.asList(InternalSettingsPlugin.class, ReloadableFilterPlugin.class); } + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + public void testTypeValidation() { InvalidTypeNameException e = expectThrows(InvalidTypeNameException.class, () -> MapperService.validateTypeName("_type")); assertEquals("mapping type name [_type] can't start with '_' unless it is called [_doc]", e.getMessage()); @@ -223,8 +231,8 @@ public void testPartitionedConstraints() { ); } - public void testIndexSortWithNestedFields() throws IOException { - Settings settings = Settings.builder().put("index.sort.field", "foo").build(); + public void testIndexSortWithNestedFieldsWithOlderVersion() throws IOException { + Settings settings = settings(Version.V_3_0_0).put("index.sort.field", "foo").build(); IllegalArgumentException invalidNestedException = expectThrows( IllegalArgumentException.class, () -> createIndex("test", settings, "t", "nested_field", "type=nested", "foo", "type=keyword") @@ -250,6 +258,64 @@ public void testIndexSortWithNestedFields() throws IOException { assertThat(invalidNestedException.getMessage(), containsString("cannot have nested fields when index sort is activated")); } + public void testIndexSortWithNestedFieldsWithNewVersion() throws IOException { + Settings settings = settings(Version.CURRENT).put("index.sort.field", "foo").build(); + IndexService indexService = createIndex("test", settings, "t", "nested_field", "type=nested", "foo", "type=keyword"); + assertTrue(indexService.getIndexSortSupplier() != null); + assertEquals("foo", indexService.getIndexSortSupplier().get().getSort()[0].getField()); + assertTrue(indexService.mapperService().getIndexSettings().getIndexSortConfig().hasIndexSort()); + final Sort oldSort = indexService.getIndexSortSupplier().get(); + + // adding new nested Field to existing index, index sort should remain same + CompressedXContent nestedFieldMapping = new CompressedXContent( + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("nested_field2") + .field("type", "nested") + .endObject() + .endObject() + .endObject() + ) + ); + indexService.mapperService().merge("t", nestedFieldMapping, updateOrPreflight()); + final Sort newSort = indexService.getIndexSortSupplier().get(); + + assertTrue(indexService.getIndexSortSupplier() != null); + assertEquals(oldSort, newSort); + assertEquals(1, indexService.getIndexSortSupplier().get().getSort().length); + assertEquals("foo", indexService.getIndexSortSupplier().get().getSort()[0].getField()); + } + + public void testIndexSortWithNestedFieldsValidation() throws IOException { + Settings settings = Settings.builder().putList("index.sort.field", "contacts.age").put("index.sort.order", "desc").build(); + + XContentBuilder mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("contacts") + .field("type", "nested") + .startObject("properties") + .startObject("name") + .field("type", "keyword") + .endObject() + .startObject("age") + .field("type", "integer") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> createIndex("test_nested", settings, "t", mapping) + ); + + assertThat(exception.getMessage(), containsString("index sorting on nested fields is not supported")); + + } + public void testFieldAliasWithMismatchedNestedScope() throws Throwable { IndexService indexService = createIndex("test"); MapperService mapperService = indexService.mapperService(); diff --git a/server/src/test/java/org/opensearch/index/shard/StoreRecoveryTests.java b/server/src/test/java/org/opensearch/index/shard/StoreRecoveryTests.java index 846b975a9520e..0d628388d2e2e 100644 --- a/server/src/test/java/org/opensearch/index/shard/StoreRecoveryTests.java +++ b/server/src/test/java/org/opensearch/index/shard/StoreRecoveryTests.java @@ -113,7 +113,7 @@ public void testAddIndices() throws IOException { Directory target = newFSDirectory(createTempDir()); final long maxSeqNo = randomNonNegativeLong(); final long maxUnsafeAutoIdTimestamp = randomNonNegativeLong(); - storeRecovery.addIndices(indexStats, target, indexSort, dirs, maxSeqNo, maxUnsafeAutoIdTimestamp, null, 0, false, false); + storeRecovery.addIndices(indexStats, target, indexSort, dirs, maxSeqNo, maxUnsafeAutoIdTimestamp, null, false, 0, false, false); int numFiles = 0; Predicate filesFilter = (f) -> f.startsWith("segments") == false && f.equals("write.lock") == false @@ -195,6 +195,7 @@ public void testSplitShard() throws IOException { maxSeqNo, maxUnsafeAutoIdTimestamp, metadata, + false, targetShardId, true, false