add semantic_field_search_analyzer field to rank_feature type

CHANGELOG.md

@@ -32,10 +32,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add support for `matched_fields` with the unified highlighter ([#18164](https://github.com/opensearch-project/OpenSearch/issues/18164))
 - [repository-s3] Add support for SSE-KMS and S3 bucket owner verification ([#18312](https://github.com/opensearch-project/OpenSearch/pull/18312))
 - Added File Cache Stats - Involves Block level as well as full file level stats ([#17538](https://github.com/opensearch-project/OpenSearch/issues/17479))
+- [Semantic Field] add semantic_field_search_analyzer field to rank_feature type ([#18372](https://github.com/opensearch-project/OpenSearch/pull/18372))
 
 ### Changed
 - Create generic DocRequest to better categorize ActionRequests ([#18269](https://github.com/opensearch-project/OpenSearch/pull/18269)))
-    
+
 ### Dependencies
 - Update Apache Lucene from 10.1.0 to 10.2.1 ([#17961](https://github.com/opensearch-project/OpenSearch/pull/17961))
 - Bump `com.google.code.gson:gson` from 2.12.1 to 2.13.1 ([#17923](https://github.com/opensearch-project/OpenSearch/pull/17923), [#18266](https://github.com/opensearch-project/OpenSearch/pull/18266))

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeatureFieldMapper.java

@@ -80,6 +80,12 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
             m -> ft(m).positiveScoreImpact,
             true
         );
+        private final Parameter<String> semanticFieldSearchAnalyzer = Parameter.stringParam(
+            "semantic_field_search_analyzer",
+            false,
+            m -> ft(m).semanticFieldSearchAnalyzer,
+            ""
+        );
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
         public Builder(String name) {
@@ -88,17 +94,23 @@ public Builder(String name) {
 
         @Override
         protected List<Parameter<?>> getParameters() {
-            return Arrays.asList(positiveScoreImpact, meta);
+            return Arrays.asList(positiveScoreImpact, meta, semanticFieldSearchAnalyzer);
         }
 
         @Override
         public RankFeatureFieldMapper build(BuilderContext context) {
             return new RankFeatureFieldMapper(
                 name,
-                new RankFeatureFieldType(buildFullName(context), meta.getValue(), positiveScoreImpact.getValue()),
+                new RankFeatureFieldType(
+                    buildFullName(context),
+                    meta.getValue(),
+                    positiveScoreImpact.getValue(),
+                    semanticFieldSearchAnalyzer.getValue()
+                ),
                 multiFieldsBuilder.build(this, context),
                 copyTo.build(),
-                positiveScoreImpact.getValue()
+                positiveScoreImpact.getValue(),
+                semanticFieldSearchAnalyzer.getValue()
             );
         }
     }
@@ -108,10 +120,17 @@ public RankFeatureFieldMapper build(BuilderContext context) {
     public static final class RankFeatureFieldType extends MappedFieldType {
 
         private final boolean positiveScoreImpact;
-
-        public RankFeatureFieldType(String name, Map<String, String> meta, boolean positiveScoreImpact) {
+        private final String semanticFieldSearchAnalyzer;
+
+        public RankFeatureFieldType(
+            String name,
+            Map<String, String> meta,
+            boolean positiveScoreImpact,
+            String semanticFieldSearchAnalyzer
+        ) {
             super(name, true, false, false, TextSearchInfo.NONE, meta);
             this.positiveScoreImpact = positiveScoreImpact;
+            this.semanticFieldSearchAnalyzer = semanticFieldSearchAnalyzer;
             setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
         }
 
@@ -154,16 +173,19 @@ public Query termQuery(Object value, QueryShardContext context) {
     }
 
     private final boolean positiveScoreImpact;
+    private final String semanticFieldSearchAnalyzer;
 
     private RankFeatureFieldMapper(
         String simpleName,
         MappedFieldType mappedFieldType,
         MultiFields multiFields,
         CopyTo copyTo,
-        boolean positiveScoreImpact
+        boolean positiveScoreImpact,
+        String semanticFieldSearchAnalyzer
     ) {
         super(simpleName, mappedFieldType, multiFields, copyTo);
         this.positiveScoreImpact = positiveScoreImpact;
+        this.semanticFieldSearchAnalyzer = semanticFieldSearchAnalyzer;
     }
 
     @Override

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeaturesFieldMapper.java

@@ -66,6 +66,12 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
             m -> ft(m).positiveScoreImpact,
             true
         );
+        private final Parameter<String> semanticFieldSearchAnalyzer = Parameter.stringParam(
+            "semantic_field_search_analyzer",
+            false,
+            m -> ft(m).semanticFieldSearchAnalyzer,
+            ""
+        );
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
         public Builder(String name) {
@@ -75,17 +81,23 @@ public Builder(String name) {
 
         @Override
         protected List<Parameter<?>> getParameters() {
-            return List.of(meta, positiveScoreImpact);
+            return List.of(meta, positiveScoreImpact, semanticFieldSearchAnalyzer);
         }
 
         @Override
         public RankFeaturesFieldMapper build(BuilderContext context) {
             return new RankFeaturesFieldMapper(
                 name,
-                new RankFeaturesFieldType(buildFullName(context), meta.getValue(), positiveScoreImpact.getValue()),
+                new RankFeaturesFieldType(
+                    buildFullName(context),
+                    meta.getValue(),
+                    positiveScoreImpact.getValue(),
+                    semanticFieldSearchAnalyzer.getValue()
+                ),
                 multiFieldsBuilder.build(this, context),
                 copyTo.build(),
-                positiveScoreImpact.getValue()
+                positiveScoreImpact.getValue(),
+                semanticFieldSearchAnalyzer.getValue()
             );
         }
     }
@@ -95,11 +107,18 @@ public RankFeaturesFieldMapper build(BuilderContext context) {
     public static final class RankFeaturesFieldType extends MappedFieldType {
 
         private final boolean positiveScoreImpact;
-
-        public RankFeaturesFieldType(String name, Map<String, String> meta, boolean positiveScoreImpact) {
+        private final String semanticFieldSearchAnalyzer;
+
+        public RankFeaturesFieldType(
+            String name,
+            Map<String, String> meta,
+            boolean positiveScoreImpact,
+            String semanticFieldSearchAnalyzer
+        ) {
             super(name, false, false, false, TextSearchInfo.NONE, meta);
             setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
             this.positiveScoreImpact = positiveScoreImpact;
+            this.semanticFieldSearchAnalyzer = semanticFieldSearchAnalyzer;
         }
 
         @Override
@@ -111,6 +130,10 @@ public boolean positiveScoreImpact() {
             return positiveScoreImpact;
         }
 
+        public String semanticFieldSearchAnalyzer() {
+            return semanticFieldSearchAnalyzer;
+        }
+
         @Override
         public Query existsQuery(QueryShardContext context) {
             throw new IllegalArgumentException("[rank_features] fields do not support [exists] queries");
@@ -133,17 +156,20 @@ public Query termQuery(Object value, QueryShardContext context) {
     }
 
     private final boolean positiveScoreImpact;
+    private final String semanticFieldSearchAnalyzer;
 
     private RankFeaturesFieldMapper(
         String simpleName,
         MappedFieldType mappedFieldType,
         MultiFields multiFields,
         CopyTo copyTo,
-        boolean positiveScoreImpact
+        boolean positiveScoreImpact,
+        String semanticFieldSearchAnalyzer
     ) {
         super(simpleName, mappedFieldType, multiFields, copyTo);
         assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0;
         this.positiveScoreImpact = positiveScoreImpact;
+        this.semanticFieldSearchAnalyzer = semanticFieldSearchAnalyzer;
     }
 
     @Override

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldMapperTests.java

@@ -58,6 +58,7 @@ protected void writeFieldValue(XContentBuilder builder) throws IOException {
     @Override
     protected void registerParameters(ParameterChecker checker) throws IOException {
         checker.registerConflictCheck("positive_score_impact", b -> b.field("positive_score_impact", false));
+        checker.registerConflictCheck("semantic_field_search_analyzer", b -> b.field("semantic_field_search_analyzer", "standard"));
     }
 
     @Override
@@ -106,6 +107,25 @@ public void testDefaults() throws Exception {
         assertTrue(freq1 < freq2);
     }
 
+    public void testSemanticFieldSearchAnalyzer() throws Exception {
+        DocumentMapper mapper = createDocumentMapper(
+            fieldMapping(b -> b.field("type", "rank_feature").field("semantic_field_search_analyzer", "standard"))
+        );
+
+        ParsedDocument doc1 = mapper.parse(source(b -> b.field("field", 10)));
+        IndexableField[] fields = doc1.rootDoc().getFields("_feature");
+        assertEquals(1, fields.length);
+        assertThat(fields[0], instanceOf(FeatureField.class));
+        FeatureField featureField1 = (FeatureField) fields[0];
+
+        ParsedDocument doc2 = mapper.parse(source(b -> b.field("field", 12)));
+        FeatureField featureField2 = (FeatureField) doc2.rootDoc().getFields("_feature")[0];
+
+        int freq1 = getFrequency(featureField1.tokenStream(null, null));
+        int freq2 = getFrequency(featureField2.tokenStream(null, null));
+        assertTrue(freq1 < freq2);
+    }
+
     public void testNegativeScoreImpact() throws Exception {
         DocumentMapper mapper = createDocumentMapper(
             fieldMapping(b -> b.field("type", "rank_feature").field("positive_score_impact", false))

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldTypeTests.java

@@ -42,7 +42,7 @@
 public class RankFeatureFieldTypeTests extends FieldTypeTestCase {
 
     public void testIsNotAggregatable() {
-        MappedFieldType fieldType = new RankFeatureFieldMapper.RankFeatureFieldType("field", Collections.emptyMap(), true);
+        MappedFieldType fieldType = new RankFeatureFieldMapper.RankFeatureFieldType("field", Collections.emptyMap(), true, "standard");
         assertFalse(fieldType.isAggregatable());
     }
 

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeaturesFieldMapperTests.java

@@ -69,6 +69,7 @@ protected void minimalMapping(XContentBuilder b) throws IOException {
     @Override
     protected void registerParameters(ParameterChecker checker) throws IOException {
         checker.registerConflictCheck("positive_score_impact", b -> b.field("positive_score_impact", false));
+        checker.registerConflictCheck("semantic_field_search_analyzer", b -> b.field("semantic_field_search_analyzer", "standard"));
     }
 
     @Override
@@ -95,6 +96,33 @@ public void testDefaults() throws Exception {
         assertTrue(freq1 < freq2);
     }
 
+    public void testSemanticFieldSearchAnalyzer() throws Exception {
+        DocumentMapper mapper = createDocumentMapper(
+            fieldMapping(b -> b.field("type", "rank_features").field("semantic_field_search_analyzer", "standard"))
+        );
+
+        ParsedDocument doc1 = mapper.parse(source(this::writeField));
+
+        IndexableField[] fields = doc1.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+        assertThat(fields[0], Matchers.instanceOf(FeatureField.class));
+        FeatureField featureField1 = null;
+        FeatureField featureField2 = null;
+        for (IndexableField field : fields) {
+            if (field.stringValue().equals("foo")) {
+                featureField1 = (FeatureField) field;
+            } else if (field.stringValue().equals("bar")) {
+                featureField2 = (FeatureField) field;
+            } else {
+                throw new UnsupportedOperationException();
+            }
+        }
+
+        int freq1 = RankFeatureFieldMapperTests.getFrequency(featureField1.tokenStream(null, null));
+        int freq2 = RankFeatureFieldMapperTests.getFrequency(featureField2.tokenStream(null, null));
+        assertTrue(freq1 < freq2);
+    }
+
     public void testNegativeScoreImpact() throws Exception {
         DocumentMapper mapper = createDocumentMapper(
             fieldMapping(b -> b.field("type", "rank_features").field("positive_score_impact", false))

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeaturesFieldTypeTests.java

@@ -37,7 +37,7 @@
 public class RankFeaturesFieldTypeTests extends FieldTypeTestCase {
 
     public void testIsNotAggregatable() {
-        MappedFieldType fieldType = new RankFeaturesFieldMapper.RankFeaturesFieldType("field", Collections.emptyMap(), true);
+        MappedFieldType fieldType = new RankFeaturesFieldMapper.RankFeaturesFieldType("field", Collections.emptyMap(), true, "standard");
         assertFalse(fieldType.isAggregatable());
     }
 }