Bump Apache Lucene from 10.3.2 to 10.4.0

CHANGELOG.md

@@ -66,6 +66,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.apache.commons:commons-text` from 1.14.0 to 1.15.0 ([#20576](https://github.com/opensearch-project/OpenSearch/pull/20576))
 - Bump `aws-actions/configure-aws-credentials` from 5 to 6 ([#20577](https://github.com/opensearch-project/OpenSearch/pull/20577))
 - Bump `netty` from 4.2.9.Final to 4.2.10.Final ([#20586](https://github.com/opensearch-project/OpenSearch/pull/20586))
+- Bump Apache Lucene from 10.3.2 to 10.4.0 ([#20735](https://github.com/opensearch-project/OpenSearch/pull/20735))
 - Bump `reactor-netty` from 1.3.2 to 1.3.3 ([#20589](https://github.com/opensearch-project/OpenSearch/pull/20589))
 - Bump `reactor` from 3.8.2 to 3.8.3 ([#20589](https://github.com/opensearch-project/OpenSearch/pull/20589))
 - Bump `org.jruby.jcodings:jcodings` from 1.0.63 to 1.0.64 ([#20713](https://github.com/opensearch-project/OpenSearch/pull/20713))

gradle/libs.versions.toml

@@ -1,6 +1,6 @@
 [versions]
 opensearch        = "3.6.0"
-lucene            = "10.3.2"
+lucene            = "10.4.0"
 
 bundled_jdk_vendor = "adoptium"
 bundled_jdk = "25.0.2+10"

libs/core/licenses/lucene-core-10.4.0.jar.sha1

@@ -0,0 +1 @@
+7493bc763cd5e91f2a8f7722c2f90d8ce15c6319

libs/core/src/main/java/org/opensearch/Version.java

@@ -135,7 +135,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_3_3_2 = new Version(3030299, org.apache.lucene.util.Version.LUCENE_10_3_1);
     public static final Version V_3_4_0 = new Version(3040099, org.apache.lucene.util.Version.LUCENE_10_3_2);
     public static final Version V_3_5_0 = new Version(3050099, org.apache.lucene.util.Version.LUCENE_10_3_2);
-    public static final Version V_3_6_0 = new Version(3060099, org.apache.lucene.util.Version.LUCENE_10_3_2);
+    public static final Version V_3_6_0 = new Version(3060099, org.apache.lucene.util.Version.LUCENE_10_4_0);
     public static final Version CURRENT = V_3_6_0;
 
     protected static final Map<Integer, Version> idToVersion;

libs/netty4/licenses/lucene-core-10.4.0.jar.sha1

@@ -0,0 +1 @@
+7493bc763cd5e91f2a8f7722c2f90d8ce15c6319

modules/lang-expression/licenses/lucene-expressions-10.4.0.jar.sha1

@@ -0,0 +1 @@
+e50bb1b8f468887a315c456b75f8375ddba5559e

plugins/analysis-icu/licenses/lucene-analysis-icu-10.4.0.jar.sha1

@@ -0,0 +1 @@
+50257f0d7d3fc805d596d6b842f0eeb099c72640

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-10.4.0.jar.sha1

@@ -0,0 +1 @@
+71033f490b07f1f1784724ade66efaa181a69b92

plugins/analysis-nori/licenses/lucene-analysis-nori-10.4.0.jar.sha1

@@ -0,0 +1 @@
+3781153d96b93e8f50c975bb5b60c7634687074a

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-10.4.0.jar.sha1

@@ -0,0 +1 @@
+8bec2c8cba1dd2f16d6899cfa1a7de48c0454c09

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-10.4.0.jar.sha1

@@ -0,0 +1 @@
+e05ae4af9a81735b4bf3c88449a2468331f5455e

plugins/analysis-stempel/licenses/lucene-analysis-stempel-10.4.0.jar.sha1

@@ -0,0 +1 @@
+718244708f4ef6d6ba4133c7e71b11ced8ebaa7d

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-10.4.0.jar.sha1

@@ -0,0 +1 @@
+21d90ece2dbe5df22e98dc54d848c16e1ddc838a

server/build.gradle

@@ -365,6 +365,8 @@ tasks.named("thirdPartyAudit").configure {
             'org.apache.lucene.facet.DrillSideways',
             'org.apache.lucene.facet.DrillSideways$Result',
             'org.apache.lucene.facet.FacetResult',
+            'org.apache.lucene.facet.FacetsCollector',
+            'org.apache.lucene.facet.FacetsCollector$MatchingDocs',
             'org.apache.lucene.facet.FacetsConfig',
             'org.apache.lucene.facet.FacetsConfig$DimConfig',
             'org.apache.lucene.facet.LabelAndValue',

server/licenses/lucene-analysis-common-10.4.0.jar.sha1

@@ -0,0 +1 @@
+61c4a7b27753662619e84261b42f404fda85886f

server/licenses/lucene-backward-codecs-10.4.0.jar.sha1

@@ -0,0 +1 @@
+9cb01ee8ba10cd935133a82bd72d16faf4f8dffe

server/licenses/lucene-core-10.4.0.jar.sha1

@@ -0,0 +1 @@
+7493bc763cd5e91f2a8f7722c2f90d8ce15c6319

server/licenses/lucene-grouping-10.4.0.jar.sha1

@@ -0,0 +1 @@
+8d143b267e6575a08d5483a8b103384536fdd4c9

server/licenses/lucene-highlighter-10.4.0.jar.sha1

@@ -0,0 +1 @@
+a3cd37e7588506447382a9175da271fb37c55e67

server/licenses/lucene-join-10.4.0.jar.sha1

@@ -0,0 +1 @@
+56e08fe428fe88e51dd4c89ca62c6a72bf2f8f57

server/licenses/lucene-memory-10.4.0.jar.sha1

@@ -0,0 +1 @@
+6c6f1080466c5dff5b1d0eeeb3ead745591eb9d1

server/licenses/lucene-misc-10.4.0.jar.sha1

@@ -0,0 +1 @@
+e86d65b1a8b5bf8dc66f4e002fe75ddc886da62c

server/licenses/lucene-queries-10.4.0.jar.sha1

@@ -0,0 +1 @@
+43b2654a448504dc4ac382f6d9b6896e2e8b67b7

server/licenses/lucene-queryparser-10.4.0.jar.sha1

@@ -0,0 +1 @@
+6e3e758b60b1e2549a85139d9498b54565c38eeb

server/licenses/lucene-sandbox-10.4.0.jar.sha1

@@ -0,0 +1 @@
+15976efaae9213b1a61e3509da6f366e243b90c7

server/licenses/lucene-spatial-extras-10.4.0.jar.sha1

@@ -0,0 +1 @@
+1f56784003e51d7b2df7c27a99d7b127e67e97bf

server/licenses/lucene-spatial3d-10.4.0.jar.sha1

@@ -0,0 +1 @@
+fa337dd41be0f0ee99bc4fd463d517adf25e06a3

server/licenses/lucene-suggest-10.4.0.jar.sha1

@@ -0,0 +1 @@
+d86d535c415cf9c79016aaefa9957dab98766a68

server/src/main/java/org/opensearch/common/lucene/Lucene.java

@@ -113,7 +113,7 @@
  * @opensearch.internal
  */
 public class Lucene {
-    public static final String LATEST_CODEC = "Lucene103";
+    public static final String LATEST_CODEC = "Lucene104";
 
     public static final String SOFT_DELETES_FIELD = "__soft_deletes";
     public static final String PARENT_FIELD = "__nested_parent";

server/src/main/java/org/opensearch/index/codec/CodecService.java

@@ -34,7 +34,7 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene103.Lucene103Codec;
+import org.apache.lucene.codecs.lucene104.Lucene104Codec;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.collect.MapBuilder;
@@ -87,23 +87,23 @@ public CodecService(
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
         assert null != indexSettings;
         if (mapperService == null) {
-            codecs.put(DEFAULT_CODEC, new Lucene103Codec());
-            codecs.put(LZ4, new Lucene103Codec());
-            codecs.put(BEST_COMPRESSION_CODEC, new Lucene103Codec(Lucene103Codec.Mode.BEST_COMPRESSION));
-            codecs.put(ZLIB, new Lucene103Codec(Lucene103Codec.Mode.BEST_COMPRESSION));
+            codecs.put(DEFAULT_CODEC, new Lucene104Codec());
+            codecs.put(LZ4, new Lucene104Codec());
+            codecs.put(BEST_COMPRESSION_CODEC, new Lucene104Codec(Lucene104Codec.Mode.BEST_COMPRESSION));
+            codecs.put(ZLIB, new Lucene104Codec(Lucene104Codec.Mode.BEST_COMPRESSION));
         } else {
             // CompositeCodec still delegates to PerFieldMappingPostingFormatCodec
             // We can still support all the compression codecs when composite index is present
             if (mapperService.isCompositeIndexPresent()) {
                 codecs.putAll(compositeCodecFactory.getCompositeIndexCodecs(mapperService, logger));
             } else {
-                codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Lucene103Codec.Mode.BEST_SPEED, mapperService, logger));
-                codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Lucene103Codec.Mode.BEST_SPEED, mapperService, logger));
+                codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Lucene104Codec.Mode.BEST_SPEED, mapperService, logger));
+                codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Lucene104Codec.Mode.BEST_SPEED, mapperService, logger));
                 codecs.put(
                     BEST_COMPRESSION_CODEC,
-                    new PerFieldMappingPostingFormatCodec(Lucene103Codec.Mode.BEST_COMPRESSION, mapperService, logger)
+                    new PerFieldMappingPostingFormatCodec(Lucene104Codec.Mode.BEST_COMPRESSION, mapperService, logger)
                 );
-                codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Lucene103Codec.Mode.BEST_COMPRESSION, mapperService, logger));
+                codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Lucene104Codec.Mode.BEST_COMPRESSION, mapperService, logger));
             }
         }
         codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());

server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java

@@ -36,7 +36,7 @@
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene103.Lucene103Codec;
+import org.apache.lucene.codecs.lucene104.Lucene104Codec;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
 import org.opensearch.common.lucene.Lucene;
 import org.opensearch.index.codec.fuzzy.FuzzyFilterPostingsFormat;
@@ -59,7 +59,7 @@
  *
  * @opensearch.internal
  */
-public class PerFieldMappingPostingFormatCodec extends Lucene103Codec {
+public class PerFieldMappingPostingFormatCodec extends Lucene104Codec {
     private final Logger logger;
     private final MapperService mapperService;
     private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat();

server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java

@@ -10,9 +10,9 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene103.Lucene103Codec;
+import org.apache.lucene.codecs.lucene104.Lucene104Codec;
 import org.opensearch.common.annotation.ExperimentalApi;
-import org.opensearch.index.codec.composite.composite103.Composite103Codec;
+import org.opensearch.index.codec.composite.composite104.Composite104Codec;
 import org.opensearch.index.mapper.MapperService;
 
 import java.util.HashMap;
@@ -32,16 +32,16 @@
 public class CompositeCodecFactory {
 
     // we can use this to track the latest composite codec
-    public static final String COMPOSITE_CODEC = Composite103Codec.COMPOSITE_INDEX_CODEC_NAME;
+    public static final String COMPOSITE_CODEC = Composite104Codec.COMPOSITE_INDEX_CODEC_NAME;
 
     public CompositeCodecFactory() {}
 
     public Map<String, Codec> getCompositeIndexCodecs(MapperService mapperService, Logger logger) {
         Map<String, Codec> codecs = new HashMap<>();
-        codecs.put(DEFAULT_CODEC, new Composite103Codec(Lucene103Codec.Mode.BEST_SPEED, mapperService, logger));
-        codecs.put(LZ4, new Composite103Codec(Lucene103Codec.Mode.BEST_SPEED, mapperService, logger));
-        codecs.put(BEST_COMPRESSION_CODEC, new Composite103Codec(Lucene103Codec.Mode.BEST_COMPRESSION, mapperService, logger));
-        codecs.put(ZLIB, new Composite103Codec(Lucene103Codec.Mode.BEST_COMPRESSION, mapperService, logger));
+        codecs.put(DEFAULT_CODEC, new Composite104Codec(Lucene104Codec.Mode.BEST_SPEED, mapperService, logger));
+        codecs.put(LZ4, new Composite104Codec(Lucene104Codec.Mode.BEST_SPEED, mapperService, logger));
+        codecs.put(BEST_COMPRESSION_CODEC, new Composite104Codec(Lucene104Codec.Mode.BEST_COMPRESSION, mapperService, logger));
+        codecs.put(ZLIB, new Composite104Codec(Lucene104Codec.Mode.BEST_COMPRESSION, mapperService, logger));
         return codecs;
     }
 }

server/src/main/java/org/opensearch/index/codec/composite/backward_codecs/composite101/package-info.java

@@ -7,6 +7,6 @@
  */
 
 /**
- *  Responsible for handling all composite index codecs and operations associated with Composite100 codec
+ *  Responsible for handling all composite index codecs and operations associated with Composite101 codec
  */
 package org.opensearch.index.codec.composite.backward_codecs.composite101;

server/src/main/java/org/opensearch/index/codec/composite/composite103/Composite103Codec.java → server/src/main/java/org/opensearch/index/codec/composite/backward_codecs/composite103/Composite103Codec.java

@@ -6,15 +6,13 @@
  * compatible open source license.
  */
 
-package org.opensearch.index.codec.composite.composite103;
+package org.opensearch.index.codec.composite.backward_codecs.composite103;
 
-import org.apache.logging.log4j.Logger;
+import org.apache.lucene.backward_codecs.lucene103.Lucene103Codec;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.lucene103.Lucene103Codec;
 import org.opensearch.common.annotation.ExperimentalApi;
-import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
 import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat;
 import org.opensearch.index.mapper.MapperService;
 
@@ -34,10 +32,6 @@ public Composite103Codec() {
         this(COMPOSITE_INDEX_CODEC_NAME, new Lucene103Codec(), null);
     }
 
-    public Composite103Codec(Lucene103Codec.Mode compressionMode, MapperService mapperService, Logger logger) {
-        this(COMPOSITE_INDEX_CODEC_NAME, new PerFieldMappingPostingFormatCodec(compressionMode, mapperService, logger), mapperService);
-    }
-
     /**
      * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec and a unique name to
      * this ctor.

server/src/main/java/org/opensearch/index/codec/composite/backward_codecs/composite103/package-info.java

@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ *  Responsible for handling all composite index codecs and operations associated with Composite103 codec
+ */
+package org.opensearch.index.codec.composite.backward_codecs.composite103;

server/src/main/java/org/opensearch/index/codec/composite/composite104/Composite104Codec.java

@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.composite.composite104;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.lucene104.Lucene104Codec;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
+import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat;
+import org.opensearch.index.mapper.MapperService;
+
+/**
+ *  Extends the Codec to support new file formats for composite indices eg: star tree index
+ *  based on the mappings.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class Composite104Codec extends FilterCodec {
+    public static final String COMPOSITE_INDEX_CODEC_NAME = "Composite104Codec";
+    private final MapperService mapperService;
+
+    // needed for SPI - this is used in reader path
+    public Composite104Codec() {
+        this(COMPOSITE_INDEX_CODEC_NAME, new Lucene104Codec(), null);
+    }
+
+    public Composite104Codec(Lucene104Codec.Mode compressionMode, MapperService mapperService, Logger logger) {
+        this(COMPOSITE_INDEX_CODEC_NAME, new PerFieldMappingPostingFormatCodec(compressionMode, mapperService, logger), mapperService);
+    }
+
+    /**
+     * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec and a unique name to
+     * this ctor.
+     *
+     * @param name name of the codec
+     * @param delegate codec delegate
+     * @param mapperService mapper service instance
+     */
+    protected Composite104Codec(String name, Codec delegate, MapperService mapperService) {
+        super(name, delegate);
+        this.mapperService = mapperService;
+    }
+
+    @Override
+    public DocValuesFormat docValuesFormat() {
+        return new Composite912DocValuesFormat(mapperService);
+    }
+}

server/src/main/java/org/opensearch/index/codec/composite/composite103/package-info.java → server/src/main/java/org/opensearch/index/codec/composite/composite104/package-info.java

@@ -7,6 +7,6 @@
  */
 
 /**
- *  Responsible for handling all composite index codecs and operations associated with Composite100 codec
+ *  Responsible for handling all composite index codecs and operations associated with Composite104 codec
  */
-package org.opensearch.index.codec.composite.composite103;
+package org.opensearch.index.codec.composite.composite104;