[Derived Source] Support ignore_above/normalizer for keyword/wildcard field and optimise text field lookup

CHANGELOG.md

@@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Service does not start on Windows with OpenJDK ([#20615](https://github.com/opensearch-project/OpenSearch/pull/20615))
 - Update RemoteClusterStateCleanupManager to performed batched deletions of stale ClusterMetadataManifests and address deletion timeout issues ([#20566](https://github.com/opensearch-project/OpenSearch/pull/20566))
 - Fix the regression of terms agg optimization at high cardinality ([#20623](https://github.com/opensearch-project/OpenSearch/pull/20623))
+- Support ignore_above for keyword/wildcard field and optimise text field under derived source ([#20113](https://github.com/opensearch-project/OpenSearch/pull/20113))
 
 ### Dependencies
 - Bump `ch.qos.logback:logback-core` and `ch.qos.logback:logback-classic` from 1.5.24 to 1.5.27 ([#20525](https://github.com/opensearch-project/OpenSearch/pull/20525))

server/src/internalClusterTest/java/org/opensearch/get/GetActionIT.java

@@ -803,6 +803,15 @@ public void testDerivedSourceSimple() throws IOException {
                             "enabled": true
                         },
                         "refresh_interval": -1
+                    },
+                    "analysis": {
+                        "normalizer": {
+                                "normalized_keyword": {
+                                "type": "custom",
+                                "char_filter": [],
+                                "filter": [ "lowercase" ]
+                            }
+                        }
                     }
                 },
                 "mappings": {
@@ -812,7 +821,8 @@ public void testDerivedSourceSimple() throws IOException {
                                 "type": "geo_point"
                             },
                             "keyword_field": {
-                                "type": "keyword"
+                                "type": "keyword",
+                                "normalizer": "normalized_keyword"
                             },
                             "numeric_field": {
                                 "type": "long"
@@ -824,7 +834,17 @@ public void testDerivedSourceSimple() throws IOException {
                                 "type": "boolean"
                             },
                             "text_field": {
-                                "type": "text"
+                                "type": "text",
+                                "fields": {
+                                    "keyword_field_1": {
+                                        "type": "keyword",
+                                        "normalizer": "normalized_keyword"
+                                    },
+                                    "keyword_field_2": {
+                                        "type": "keyword",
+                                        "ignore_above": 20
+                                    }
+                                }
                             },
                             "ip_field": {
                                 "type": "ip"
@@ -843,11 +863,11 @@ public void testDerivedSourceSimple() throws IOException {
             .setSource(
                 jsonBuilder().startObject()
                     .field("geopoint_field", Geohash.stringEncode(40.33, 75.98))
-                    .field("keyword_field", "test_keyword")
+                    .field("keyword_field", "TEST_KEYWORD")
                     .field("numeric_field", 123)
                     .field("date_field", "2023-01-01")
                     .field("bool_field", true)
-                    .field("text_field", "test text")
+                    .field("text_field", "ALL CAPS TEXT FIELD")
                     .field("ip_field", "1.2.3.4")
                     .endObject()
             )
@@ -881,7 +901,7 @@ public void testDerivedSourceSimple() throws IOException {
         assertTrue(getResponse.isExists());
         source = getResponse.getSourceAsMap();
         assertEquals(2, source.size());
-        assertEquals("test_keyword", source.get("keyword_field"));
+        assertEquals("TEST_KEYWORD", source.get("keyword_field"));
         assertEquals(123, source.get("numeric_field"));
 
         // Test get with field exclusion
@@ -1082,11 +1102,11 @@ void validateDeriveSource(Map<String, Object> source) {
         Map<String, Object> latLon = (Map<String, Object>) source.get("geopoint_field");
         assertEquals(75.98, (Double) latLon.get("lat"), 0.001);
         assertEquals(40.33, (Double) latLon.get("lon"), 0.001);
-        assertEquals("test_keyword", source.get("keyword_field"));
+        assertEquals("TEST_KEYWORD", source.get("keyword_field"));
         assertEquals(123, source.get("numeric_field"));
         assertEquals("2023-01-01T00:00:00.000Z", source.get("date_field"));
         assertEquals(true, source.get("bool_field"));
-        assertEquals("test text", source.get("text_field"));
+        assertEquals("ALL CAPS TEXT FIELD", source.get("text_field"));
         assertEquals("1.2.3.4", source.get("ip_field"));
     }
 

server/src/internalClusterTest/java/org/opensearch/recovery/FullRollingRestartIT.java

@@ -416,8 +416,7 @@ public void testDerivedSourceRollingRestart() throws Exception {
             {
               "properties": {
                 "text_field": {
-                  "type": "text",
-                  "store": true
+                  "type": "text"
                 },
                 "keyword_field": {
                   "type": "keyword"
@@ -459,8 +458,7 @@ public void testDerivedSourceWithMultiFieldsRollingRestart() throws Exception {
             {
               "properties": {
                 "text_field": {
-                  "type": "text",
-                  "store": true
+                  "type": "text"
                 },
                 "multi_field": {
                   "properties": {
@@ -545,8 +543,7 @@ public void testDerivedSourceWithConcurrentUpdatesRollingRestart() throws Except
             {
               "properties": {
                 "text_field": {
-                  "type": "text",
-                  "store": true
+                  "type": "text"
                 },
                 "counter": {
                   "type": "long"

server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java

@@ -777,12 +777,10 @@ public void testDerivedSourceSearch() throws Exception {
                                 "type": "ip"
                             },
                             "text_field": {
-                                "type": "text",
-                                "store": true
+                                "type": "text"
                             },
                             "wildcard_field": {
-                                "type": "wildcard",
-                                "doc_values": true
+                                "type": "wildcard"
                             },
                             "constant_keyword": {
                                 "type": "constant_keyword",

server/src/internalClusterTest/java/org/opensearch/update/UpdateIT.java

@@ -932,8 +932,7 @@ public void testDerivedSourceWithUpdates() throws Exception {
                                 "type": "boolean"
                             },
                             "text_field": {
-                                "type": "text",
-                                "store": true
+                                "type": "text"
                             }
                         }
                     }

server/src/main/java/org/opensearch/index/mapper/CompositeFieldValueFetcher.java

@@ -0,0 +1,55 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.index.LeafReader;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Composite fetcher that tries multiple sources and returns already-converted values from the available source
+ * with the highest priority
+ *
+ * @opensearch.internal
+ */
+public class CompositeFieldValueFetcher extends FieldValueFetcher {
+
+    private final List<FieldValueFetcher> fieldValueFetchers;
+
+    public CompositeFieldValueFetcher(String simpleName, List<FieldValueFetcher> fieldValueFetchers) {
+        super(simpleName);
+        this.fieldValueFetchers = fieldValueFetchers;
+    }
+
+    @Override
+    public List<Object> fetch(LeafReader reader, int docId) throws IOException {
+        // Try fetching values from various fetchers as per priority
+        for (final FieldValueFetcher fieldValueFetcher : fieldValueFetchers) {
+            List<Object> values = fieldValueFetcher.fetch(reader, docId);
+
+            // Convert values immediately after fetching
+            if (values != null && !values.isEmpty()) {
+                List<Object> convertedValues = new ArrayList<>(values.size());
+                for (Object value : values) {
+                    convertedValues.add(fieldValueFetcher.convert(value));
+                }
+                return convertedValues;
+            }
+        }
+        return null;
+    }
+
+    @Override
+    Object convert(Object value) {
+        // Values are already converted, return as-is
+        return value;
+    }
+}

server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java

@@ -367,6 +367,27 @@ public void validate(IndexSettings settings, boolean checkLimits) {
         if (checkLimits) {
             this.fieldMappers.checkLimits(settings);
         }
+
+        if (settings.isDerivedSourceEnabled() && settings.getIndexVersionCreated().onOrAfter(Version.V_3_6_0)) {
+            validateDerivedSourceFieldConflicts();
+        }
+    }
+
+    private void validateDerivedSourceFieldConflicts() {
+        // Check if any user-defined field collides with a reserved name
+        for (MappedFieldType fieldType : fieldTypes()) {
+            Mapper mapper = fieldMappers.getMapper(fieldType.name());
+            if (mapper.hasDerivedSourceIgnoredField() && fieldMappers.getMapper(fieldType.derivedSourceIgnoreFieldName()) != null) {
+                throw new MapperParsingException(
+                    "Field ["
+                        + fieldType.derivedSourceIgnoreFieldName()
+                        + "] conflicts with an internal field "
+                        + "for field ["
+                        + mapper.name()
+                        + "]. Please rename this field."
+                );
+            }
+        }
     }
 
     @Override

server/src/main/java/org/opensearch/index/mapper/FieldMapper.java

@@ -633,14 +633,6 @@ protected DerivedFieldGenerator derivedFieldGenerator() {
         return null;
     }
 
-    protected void setDerivedFieldGenerator(DerivedFieldGenerator derivedFieldGenerator) {
-        this.derivedFieldGenerator = derivedFieldGenerator;
-    }
-
-    protected DerivedFieldGenerator getDerivedFieldGenerator() {
-        return this.derivedFieldGenerator;
-    }
-
     /**
      * Method to determine, if it is possible to derive source for this field using field mapping parameters.
      * DerivedFieldGenerator should be set for which derived source feature is supported, this behaviour can be
@@ -652,7 +644,7 @@ public void canDeriveSource() {
             throw new UnsupportedOperationException("Unable to derive source for fields with copy_to parameter set");
         }
         canDeriveSourceInternal();
-        if (getDerivedFieldGenerator() == null) {
+        if (derivedFieldGenerator == null) {
             throw new UnsupportedOperationException(
                 "Derive source is not supported for field [" + name() + "] with field " + "type [" + fieldType().typeName() + "]"
             );
@@ -668,24 +660,6 @@ protected void canDeriveSourceInternal() {
         );
     }
 
-    /**
-     * Validates if doc values is enabled for a field or not
-     */
-    protected void checkDocValuesForDerivedSource() {
-        if (!mappedFieldType.hasDocValues()) {
-            throw new UnsupportedOperationException("Unable to derive source for [" + name() + "] with doc values disabled");
-        }
-    }
-
-    /**
-     * Validates if stored field is enabled for a field or not
-     */
-    protected void checkStoredForDerivedSource() {
-        if (!mappedFieldType.isStored()) {
-            throw new UnsupportedOperationException("Unable to derive source for [" + name() + "] with store disabled");
-        }
-    }
-
     /**
      * Validates if doc_values or stored field is enabled for a field or not
      */

server/src/main/java/org/opensearch/index/mapper/FieldValueFetcher.java

@@ -51,7 +51,7 @@ Object convert(Object value) {
      * @param builder - builder to store the field value(s) in
      */
     void write(XContentBuilder builder, List<Object> values) throws IOException {
-        if (values.isEmpty()) {
+        if (values == null || values.isEmpty()) {
             return;
         }
         if (values.size() == 1) {