Updating the FieldMappers to insert call addField of DocumentInput

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java

@@ -498,18 +498,22 @@ protected void parseCreateField(ParseContext context) throws IOException {
         }
         long scaledValue = Math.round(doubleValue * scalingFactor);
 
-        List<Field> fields = NumberFieldMapper.NumberType.LONG.createFields(
-            fieldType().name(),
-            scaledValue,
-            indexed,
-            hasDocValues,
-            skiplist,
-            stored
-        );
-        context.doc().addAll(fields);
+        if (isPluggableDataFormatFeatureEnabled(context)) {
+            context.documentInput().addField(fieldType(), scaledValue);
+        } else {
+            List<Field> fields = NumberFieldMapper.NumberType.LONG.createFields(
+                fieldType().name(),
+                scaledValue,
+                indexed,
+                hasDocValues,
+                skiplist,
+                stored
+            );
+            context.doc().addAll(fields);
 
-        if (hasDocValues == false && (indexed || stored)) {
-            createFieldNamesField(context);
+            if (hasDocValues == false && (indexed || stored)) {
+                createFieldNamesField(context);
+            }
         }
     }
 

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java

@@ -655,13 +655,17 @@ protected void parseCreateField(ParseContext context) throws IOException {
             return;
         }
 
-        context.doc().add(new Field(fieldType().name(), value, fieldType().fieldType));
-        for (ShingleFieldMapper subFieldMapper : shingleFields) {
-            context.doc().add(new Field(subFieldMapper.fieldType().name(), value, subFieldMapper.getLuceneFieldType()));
-        }
-        context.doc().add(new Field(prefixField.fieldType().name(), value, prefixField.getLuceneFieldType()));
-        if (fieldType().fieldType.omitNorms()) {
-            createFieldNamesField(context);
+        if (isPluggableDataFormatFeatureEnabled(context)) {
+            context.documentInput().addField(fieldType(), value);
+        } else {
+            context.doc().add(new Field(fieldType().name(), value, fieldType().fieldType));
+            for (ShingleFieldMapper subFieldMapper : shingleFields) {
+                context.doc().add(new Field(subFieldMapper.fieldType().name(), value, subFieldMapper.getLuceneFieldType()));
+            }
+            context.doc().add(new Field(prefixField.fieldType().name(), value, prefixField.getLuceneFieldType()));
+            if (fieldType().fieldType.omitNorms()) {
+                createFieldNamesField(context);
+            }
         }
     }
 

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java

@@ -185,10 +185,14 @@ protected void parseCreateField(ParseContext context) throws IOException {
             tokenCount = countPositions(analyzer, name(), value, enablePositionIncrements);
         }
 
-        context.doc()
-            .addAll(
-                NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, skiplist, store)
-            );
+        if (isPluggableDataFormatFeatureEnabled(context)) {
+            context.documentInput().addField(fieldType(), tokenCount);
+        } else {
+            context.doc()
+                .addAll(
+                    NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, skiplist, store)
+                );
+        }
     }
 
     /**

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java

@@ -45,6 +45,7 @@
 import org.apache.lucene.store.Directory;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
@@ -524,4 +525,32 @@ public void testSkiplistParameter() throws IOException {
         );
         assertThat(e.getMessage(), containsString("Failed to parse value [invalid] as only [true] or [false] are allowed"));
     }
+
+    @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testPluggableDataFormatScaledFloatValue() throws Exception {
+        Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build();
+        DocumentMapper mapper = createDocumentMapper(
+            pluggableSettings,
+            mapping(b -> b.startObject("field").field("type", "scaled_float").field("scaling_factor", 100).endObject())
+        );
+        CapturingDocumentInput docInput = new CapturingDocumentInput();
+        mapper.parse(source(b -> b.field("field", 3.14)), docInput);
+
+        boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field"));
+        assertTrue("Expected scaled float field to be captured", found);
+    }
+
+    @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testPluggableDataFormatScaledFloatNullSkipped() throws Exception {
+        Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build();
+        DocumentMapper mapper = createDocumentMapper(
+            pluggableSettings,
+            mapping(b -> b.startObject("field").field("type", "scaled_float").field("scaling_factor", 100).endObject())
+        );
+        CapturingDocumentInput docInput = new CapturingDocumentInput();
+        mapper.parse(source(b -> b.nullField("field")), docInput);
+
+        boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field"));
+        assertFalse("Expected no field entry for null value", found);
+    }
 }

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java

@@ -52,6 +52,8 @@
 import org.apache.lucene.search.SynonymQuery;
 import org.apache.lucene.search.TermQuery;
 import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.XContentBuilder;
@@ -769,4 +771,34 @@ private static PrefixFieldMapper getPrefixFieldMapper(DocumentMapper defaultMapp
         assertThat(mapper, instanceOf(PrefixFieldMapper.class));
         return (PrefixFieldMapper) mapper;
     }
+
+    @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testPluggableDataFormatSearchAsYouTypeValue() throws Exception {
+        Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build();
+        DocumentMapper mapper = createDocumentMapper(
+            pluggableSettings,
+            mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject())
+        );
+        CapturingDocumentInput docInput = new CapturingDocumentInput();
+        mapper.parse(source(b -> b.field("field", "hello world")), docInput);
+
+        boolean found = docInput.getCapturedFields()
+            .stream()
+            .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("hello world"));
+        assertTrue("Expected search_as_you_type value", found);
+    }
+
+    @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testPluggableDataFormatSearchAsYouTypeNullSkipped() throws Exception {
+        Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build();
+        DocumentMapper mapper = createDocumentMapper(
+            pluggableSettings,
+            mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject())
+        );
+        CapturingDocumentInput docInput = new CapturingDocumentInput();
+        mapper.parse(source(b -> b.nullField("field")), docInput);
+
+        boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field"));
+        assertFalse("Expected no field entry for null value", found);
+    }
 }

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/TokenCountFieldMapperTests.java

@@ -42,6 +42,8 @@
 import org.apache.lucene.tests.analysis.CannedTokenStream;
 import org.apache.lucene.tests.analysis.MockTokenizer;
 import org.apache.lucene.tests.analysis.Token;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.analysis.AnalyzerScope;
@@ -219,4 +221,45 @@ private SourceToParse createDocument(String fieldValue) throws Exception {
     private ParseContext.Document parseDocument(DocumentMapper mapper, SourceToParse request) {
         return mapper.parse(request).docs().stream().findFirst().orElseThrow(() -> new IllegalStateException("Test object not parsed"));
     }
+
+    private DocumentMapper createIndexWithTokenCountFieldPluggableDataFormat() throws IOException {
+        Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build();
+        return createDocumentMapper(pluggableSettings, mapping(b -> {
+            b.startObject("test");
+            {
+                b.field("type", "text");
+                b.startObject("fields");
+                {
+                    b.startObject("tc");
+                    {
+                        b.field("type", "token_count");
+                        b.field("analyzer", "standard");
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        }));
+    }
+
+    @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testPluggableDataFormatTokenCountValue() throws Exception {
+        DocumentMapper mapper = createIndexWithTokenCountFieldPluggableDataFormat();
+        CapturingDocumentInput docInput = new CapturingDocumentInput();
+        mapper.parse(createDocument("three tokens string"), docInput);
+
+        boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("test.tc") && e.getValue().equals(3));
+        assertTrue("Expected token count of 3 for field test.tc", found);
+    }
+
+    @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testPluggableDataFormatTokenCountNullSkipped() throws Exception {
+        DocumentMapper mapper = createIndexWithTokenCountFieldPluggableDataFormat();
+        CapturingDocumentInput docInput = new CapturingDocumentInput();
+        mapper.parse(createDocument(null), docInput);
+
+        boolean hasTokenCountField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("test.tc"));
+        assertFalse("Expected no token count field for null value", hasTokenCountField);
+    }
 }

modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentIdFieldMapper.java

@@ -203,9 +203,13 @@ protected void parseCreateField(ParseContext context) throws IOException {
         }
         String refId = (String) context.externalValue();
         BytesRef binaryValue = new BytesRef(refId);
-        Field field = new Field(fieldType().name(), binaryValue, fieldType);
-        context.doc().add(field);
-        context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue));
+        if (isPluggableDataFormatFeatureEnabled(context)) {
+            context.documentInput().addField(fieldType(), binaryValue);
+        } else {
+            Field field = new Field(fieldType().name(), binaryValue, fieldType);
+            context.doc().add(field);
+            context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue));
+        }
     }
 
     @Override

modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentJoinFieldMapper.java

@@ -430,9 +430,14 @@ public void parse(ParseContext context) throws IOException {
         }
 
         BytesRef binaryValue = new BytesRef(name);
-        Field field = new Field(fieldType().name(), binaryValue, fieldType);
-        context.doc().add(field);
-        context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue));
+        if (isPluggableDataFormatFeatureEnabled(context)) {
+            context.documentInput().addField(fieldType(), binaryValue);
+        } else {
+            Field field = new Field(fieldType().name(), binaryValue, fieldType);
+            context.doc().add(field);
+            context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue));
+        }
+
         context.path().remove();
     }
 

modules/percolator/src/test/java/org/opensearch/percolator/QueryBuilderStoreTests.java

@@ -100,6 +100,12 @@ public void testStoringQueryBuilders() throws IOException {
                     ParseContext parseContext = mock(ParseContext.class);
                     ParseContext.Document document = new ParseContext.Document();
                     when(parseContext.doc()).thenReturn(document);
+                    when(parseContext.indexSettings()).thenReturn(
+                        new org.opensearch.index.IndexSettings(
+                            IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build(),
+                            settings
+                        )
+                    );
                     PercolatorFieldMapper.createQueryBuilderField(version, fieldMapper, queryBuilders[i], parseContext);
                     indexWriter.addDocument(document);
                 }

plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java

@@ -812,15 +812,20 @@ protected void parseCreateField(ParseContext context) throws IOException {
         RawCollationKey key = collator.getRawCollationKey(value, null);
         final BytesRef binaryValue = new BytesRef(key.bytes, 0, key.size);
 
-        if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
-            Field field = new Field(mappedFieldType.name(), binaryValue, fieldType);
-            context.doc().add(field);
-        }
+        if (isPluggableDataFormatFeatureEnabled(context)) {
+            context.documentInput().addField(fieldType(), binaryValue);
+        } else {
 
-        if (fieldType().hasDocValues()) {
-            context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue));
-        } else if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
-            createFieldNamesField(context);
+            if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
+                Field field = new Field(mappedFieldType.name(), binaryValue, fieldType);
+                context.doc().add(field);
+            }
+
+            if (fieldType().hasDocValues()) {
+                context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue));
+            } else if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
+                createFieldNamesField(context);
+            }
         }
     }
 

plugins/mapper-size/src/main/java/org/opensearch/index/mapper/size/SizeFieldMapper.java

@@ -99,7 +99,11 @@ public void postParse(ParseContext context) throws IOException {
             return;
         }
         final int value = context.sourceToParse().source().length();
-        context.doc().addAll(NumberType.INTEGER.createFields(name(), value, true, true, false, true));
+        if (isPluggableDataFormatFeatureEnabled(context)) {
+            context.documentInput().addField(fieldType(), value);
+        } else {
+            context.doc().addAll(NumberType.INTEGER.createFields(name(), value, true, true, false, true));
+        }
     }
 
     @Override

server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java

@@ -37,6 +37,7 @@ protected FeatureFlagSettings(
         FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
         FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING,
         FeatureFlags.TERM_VERSION_PRECOMMIT_ENABLE_SETTING,
-        FeatureFlags.STREAM_TRANSPORT_SETTING
+        FeatureFlags.STREAM_TRANSPORT_SETTING,
+        FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_SETTING
     );
 }

server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java

@@ -304,6 +304,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 // Setting for derived source feature
                 IndexSettings.INDEX_DERIVED_SOURCE_SETTING,
                 IndexSettings.INDEX_DERIVED_SOURCE_TRANSLOG_ENABLED_SETTING,
+                IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING,
 
                 // validate that built-in similarities don't get redefined
                 Setting.groupSetting("index.similarity.", (s) -> {

server/src/main/java/org/opensearch/common/util/FeatureFlags.java

@@ -74,6 +74,17 @@ public class FeatureFlags {
         Property.NodeScope
     );
 
+    /**
+     * Gates the functionality of pluggable dataformat feature.
+     */
+    public static final String PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG = FEATURE_FLAG_PREFIX + "pluggable.dataformat.enabled";
+
+    public static final Setting<Boolean> PLUGGABLE_DATAFORMAT_EXPERIMENTAL_SETTING = Setting.boolSetting(
+        PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG,
+        false,
+        Property.NodeScope
+    );
+
     public static final Setting<Boolean> CONTEXT_AWARE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting(
         CONTEXT_AWARE_MIGRATION_EXPERIMENTAL_FLAG,
         false,
@@ -141,6 +152,7 @@ static class FeatureFlagsImpl {
                 put(TERM_VERSION_PRECOMMIT_ENABLE_SETTING, TERM_VERSION_PRECOMMIT_ENABLE_SETTING.getDefault(Settings.EMPTY));
                 put(STREAM_TRANSPORT_SETTING, STREAM_TRANSPORT_SETTING.getDefault(Settings.EMPTY));
                 put(CONTEXT_AWARE_MIGRATION_EXPERIMENTAL_SETTING, CONTEXT_AWARE_MIGRATION_EXPERIMENTAL_SETTING.getDefault(Settings.EMPTY));
+                put(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_SETTING, PLUGGABLE_DATAFORMAT_EXPERIMENTAL_SETTING.getDefault(Settings.EMPTY));
             }
         };
 

server/src/main/java/org/opensearch/index/IndexSettings.java

@@ -917,6 +917,13 @@ public static IndexMergePolicy fromString(String text) {
         Property.Dynamic
     );
 
+    public static final Setting<Boolean> PLUGGABLE_DATAFORMAT_ENABLED_SETTING = Setting.boolSetting(
+        "index.pluggable.dataformat.enabled",
+        false,
+        Property.IndexScope,
+        Property.Final
+    );
+
     private final Index index;
     private final Version version;
     private final Logger logger;