From c90a3cf0d5ec68f9461adef3bc40026a67ddbe3d Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Tue, 27 Jun 2023 19:41:03 +0530 Subject: [PATCH 1/5] Enabling compression levels as index settings Signed-off-by: Sarthak Aggarwal --- CHANGELOG.md | 1 + .../common/settings/IndexScopedSettings.java | 1 + .../opensearch/index/codec/CodecService.java | 14 +++++++++++++- .../customcodecs/Lucene95CustomCodec.java | 12 +++++++++++- .../Lucene95CustomStoredFieldsFormat.java | 6 ++++++ .../opensearch/index/engine/EngineConfig.java | 12 ++++++++++++ .../org/opensearch/index/codec/CodecTests.java | 14 ++++++++++++++ .../Lucene95CustomStoredFieldsFormatTests.java | 18 ++++++++++++++++++ 8 files changed, 76 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffc9e6d2783ea..045a1cebf4283 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -155,6 +155,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Move ZSTD compression codecs out of the sandbox ([#7908](https://github.com/opensearch-project/OpenSearch/pull/7908)) - Update ZSTD default compression level ([#8471](https://github.com/opensearch-project/OpenSearch/pull/8471)) - [Search Pipelines] Pass pipeline creation context to processor factories ([#8164](https://github.com/opensearch-project/OpenSearch/pull/8164)) +- Enabling compression levels for zstd and zstd_no_dict ([#8312](https://github.com/opensearch-project/OpenSearch/pull/8312)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 08d8199afface..739982036c2af 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -188,6 +188,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { FsDirectoryFactory.INDEX_LOCK_FACTOR_SETTING, Store.FORCE_RAM_TERM_DICT, EngineConfig.INDEX_CODEC_SETTING, + EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING, EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS, IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS, IndexSettings.DEFAULT_PIPELINE, diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index b6dac7bd1596c..439cad063fb11 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -38,6 +38,7 @@ import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode; import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; +import org.opensearch.index.codec.customcodecs.Lucene95CustomCodec; import org.opensearch.index.codec.customcodecs.ZstdCodec; import org.opensearch.index.codec.customcodecs.ZstdNoDictCodec; import org.opensearch.index.mapper.MapperService; @@ -58,7 +59,9 @@ public class CodecService { public static final String DEFAULT_CODEC = "default"; public static final String BEST_COMPRESSION_CODEC = "best_compression"; - /** the raw unfiltered lucene default. useful for testing */ + /** + * the raw unfiltered lucene default. useful for testing + */ public static final String LUCENE_DEFAULT_CODEC = "lucene_default"; public static final String ZSTD_CODEC = "zstd"; public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; @@ -91,6 +94,15 @@ public Codec codec(String name) { return codec; } + public Codec codec(String name, int compressionLevel) { + Lucene95CustomCodec codec = (Lucene95CustomCodec) codecs.get(name); + if (codec == null) { + throw new IllegalArgumentException("failed to find codec [" + name + "]"); + } + codec.updateCompressionLevel(compressionLevel); + return codec; + } + /** * Returns all registered available codec names */ diff --git a/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java index 8aa422a47a073..996eb398cad91 100644 --- a/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java @@ -15,6 +15,8 @@ import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; import org.opensearch.index.mapper.MapperService; +import java.util.Objects; + /** * * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. @@ -31,7 +33,9 @@ public enum Mode { ZSTD_NO_DICT } - private final StoredFieldsFormat storedFieldsFormat; + private final Mode mode; + + private StoredFieldsFormat storedFieldsFormat; /** * Creates a new compression codec with the default compression level. @@ -52,11 +56,13 @@ public Lucene95CustomCodec(Mode mode) { */ public Lucene95CustomCodec(Mode mode, int compressionLevel) { super("Lucene95CustomCodec", new Lucene95Codec()); + this.mode = Objects.requireNonNull(mode); this.storedFieldsFormat = new Lucene95CustomStoredFieldsFormat(mode, compressionLevel); } public Lucene95CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { super("Lucene95CustomCodec", new PerFieldMappingPostingFormatCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, logger)); + this.mode = Objects.requireNonNull(mode); this.storedFieldsFormat = new Lucene95CustomStoredFieldsFormat(mode, compressionLevel); } @@ -69,4 +75,8 @@ public StoredFieldsFormat storedFieldsFormat() { public String toString() { return getClass().getSimpleName(); } + + public void updateCompressionLevel(int compressionLevel) { + this.storedFieldsFormat = new Lucene95CustomStoredFieldsFormat(mode, compressionLevel); + } } diff --git a/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java index f70306afc8562..2816e2907a5f6 100644 --- a/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java @@ -35,6 +35,7 @@ public class Lucene95CustomStoredFieldsFormat extends StoredFieldsFormat { private final CompressionMode zstdNoDictCompressionMode; private final Lucene95CustomCodec.Mode mode; + private final int compressionLevel; /** default constructor */ public Lucene95CustomStoredFieldsFormat() { @@ -58,6 +59,7 @@ public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode) { */ public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode, int compressionLevel) { this.mode = Objects.requireNonNull(mode); + this.compressionLevel = compressionLevel; zstdCompressionMode = new ZstdCompressionMode(compressionLevel); zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); } @@ -122,4 +124,8 @@ StoredFieldsFormat impl(Lucene95CustomCodec.Mode mode) { Lucene95CustomCodec.Mode getMode() { return mode; } + + public int getCompressionLevel() { + return compressionLevel; + } } diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index 7419cf1dadea6..786dfb8c68ec9 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -82,6 +82,7 @@ public final class EngineConfig { private volatile boolean enableGcDeletes = true; private final TimeValue flushMergesAfter; private final String codecName; + private final int compressionLevel; private final ThreadPool threadPool; private final Engine.Warmer warmer; private final Store store; @@ -142,6 +143,13 @@ public Supplier retentionLeasesSupplier() { return s; } }, Property.IndexScope, Property.NodeScope); + public static final Setting INDEX_CODEC_COMPRESSION_LEVEL_SETTING = Setting.intSetting( + "index.codec.compression_level", + 6, + 1, + 6, + Property.IndexScope + ); /** * Configures an index to optimize documents with auto generated ids for append only. If this setting is updated from false @@ -179,6 +187,7 @@ private EngineConfig(Builder builder) { this.codecService = builder.codecService; this.eventListener = builder.eventListener; codecName = builder.indexSettings.getValue(INDEX_CODEC_SETTING); + compressionLevel = builder.indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); // We need to make the indexing buffer for this shard at least as large // as the amount of memory that is available for all engines on the // local node so that decisions to flush segments to disk are made by @@ -250,6 +259,9 @@ public boolean isEnableGcDeletes() { *

*/ public Codec getCodec() { + if (codecName.equals(CodecService.ZSTD_CODEC) || codecName.equals(CodecService.ZSTD_NO_DICT_CODEC)) { + return codecService.codec(codecName, compressionLevel); + } return codecService.codec(codecName); } diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index 016e785f8da13..96d61e43045dc 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -90,6 +90,20 @@ public void testZstdNoDict() throws Exception { assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); } + public void testZstdWithCompressionLevel() throws Exception { + Codec codec = createCodecService(false).codec("zstd", 1); + assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(1, storedFieldsFormat.getCompressionLevel()); + } + + public void testZstdNoDictWithCompressionLevel() throws Exception { + Codec codec = createCodecService(false).codec("zstd_no_dict", 1); + assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(1, storedFieldsFormat.getCompressionLevel()); + } + public void testDefaultMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("default"); assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec); diff --git a/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java b/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java index 4f23450ce0b39..755d271fe5934 100644 --- a/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java @@ -24,4 +24,22 @@ public void testZstdNoDictLucene95CustomCodecMode() { assertEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, lucene95CustomStoredFieldsFormat.getMode()); } + public void testZstdModeWithCompressionLevel() { + Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( + Lucene95CustomCodec.Mode.ZSTD, + 1 + ); + assertEquals(Lucene95CustomCodec.Mode.ZSTD, lucene95CustomStoredFieldsFormat.getMode()); + assertEquals(1, lucene95CustomStoredFieldsFormat.getCompressionLevel()); + } + + public void testZstdNoDictLucene95CustomCodecModeWithCompressionLevel() { + Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( + Lucene95CustomCodec.Mode.ZSTD_NO_DICT, + 1 + ); + assertEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, lucene95CustomStoredFieldsFormat.getMode()); + assertEquals(1, lucene95CustomStoredFieldsFormat.getCompressionLevel()); + } + } From d05587660b6d4489e43d2dcfcff259f1f0e538b7 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Fri, 30 Jun 2023 11:28:35 +0530 Subject: [PATCH 2/5] addressing review comments Signed-off-by: Sarthak Aggarwal --- .../opensearch/index/codec/CodecService.java | 17 +++------ .../customcodecs/Lucene95CustomCodec.java | 12 +----- .../index/codec/customcodecs/ZstdCodec.java | 4 +- .../codec/customcodecs/ZstdNoDictCodec.java | 4 +- .../opensearch/index/engine/EngineConfig.java | 11 +++--- .../opensearch/index/codec/CodecTests.java | 37 +++++++++++++++++-- ...Lucene95CustomStoredFieldsFormatTests.java | 15 ++++++-- 7 files changed, 60 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index 439cad063fb11..07203588eedd0 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -38,13 +38,14 @@ import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode; import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; -import org.opensearch.index.codec.customcodecs.Lucene95CustomCodec; import org.opensearch.index.codec.customcodecs.ZstdCodec; import org.opensearch.index.codec.customcodecs.ZstdNoDictCodec; import org.opensearch.index.mapper.MapperService; import java.util.Map; +import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; + /** * Since Lucene 4.0 low level index segments are read and written through a * codec layer that allows to use use-case specific file formats & @@ -74,10 +75,11 @@ public CodecService(@Nullable MapperService mapperService, Logger logger) { codecs.put(ZSTD_CODEC, new ZstdCodec()); codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec()); } else { + int compressionLevel = mapperService.getIndexSettings().getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); - codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger)); + codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel)); } codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault()); for (String codec : Codec.availableCodecs()) { @@ -94,15 +96,6 @@ public Codec codec(String name) { return codec; } - public Codec codec(String name, int compressionLevel) { - Lucene95CustomCodec codec = (Lucene95CustomCodec) codecs.get(name); - if (codec == null) { - throw new IllegalArgumentException("failed to find codec [" + name + "]"); - } - codec.updateCompressionLevel(compressionLevel); - return codec; - } - /** * Returns all registered available codec names */ diff --git a/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java index 996eb398cad91..8aa422a47a073 100644 --- a/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java @@ -15,8 +15,6 @@ import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; import org.opensearch.index.mapper.MapperService; -import java.util.Objects; - /** * * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. @@ -33,9 +31,7 @@ public enum Mode { ZSTD_NO_DICT } - private final Mode mode; - - private StoredFieldsFormat storedFieldsFormat; + private final StoredFieldsFormat storedFieldsFormat; /** * Creates a new compression codec with the default compression level. @@ -56,13 +52,11 @@ public Lucene95CustomCodec(Mode mode) { */ public Lucene95CustomCodec(Mode mode, int compressionLevel) { super("Lucene95CustomCodec", new Lucene95Codec()); - this.mode = Objects.requireNonNull(mode); this.storedFieldsFormat = new Lucene95CustomStoredFieldsFormat(mode, compressionLevel); } public Lucene95CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { super("Lucene95CustomCodec", new PerFieldMappingPostingFormatCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, logger)); - this.mode = Objects.requireNonNull(mode); this.storedFieldsFormat = new Lucene95CustomStoredFieldsFormat(mode, compressionLevel); } @@ -75,8 +69,4 @@ public StoredFieldsFormat storedFieldsFormat() { public String toString() { return getClass().getSimpleName(); } - - public void updateCompressionLevel(int compressionLevel) { - this.storedFieldsFormat = new Lucene95CustomStoredFieldsFormat(mode, compressionLevel); - } } diff --git a/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java index 68da782421e6e..04c110fceacdf 100644 --- a/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java @@ -32,8 +32,8 @@ public ZstdCodec(int compressionLevel) { super(Mode.ZSTD, compressionLevel); } - public ZstdCodec(MapperService mapperService, Logger logger) { - super(Mode.ZSTD, DEFAULT_COMPRESSION_LEVEL, mapperService, logger); + public ZstdCodec(MapperService mapperService, Logger logger, int compressionLevel) { + super(Mode.ZSTD, compressionLevel, mapperService, logger); } /** The name for this codec. */ diff --git a/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java index 26620473ec116..134f9a14422ad 100644 --- a/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java @@ -32,8 +32,8 @@ public ZstdNoDictCodec(int compressionLevel) { super(Mode.ZSTD_NO_DICT, compressionLevel); } - public ZstdNoDictCodec(MapperService mapperService, Logger logger) { - super(Mode.ZSTD_NO_DICT, DEFAULT_COMPRESSION_LEVEL, mapperService, logger); + public ZstdNoDictCodec(MapperService mapperService, Logger logger, int compressionLevel) { + super(Mode.ZSTD_NO_DICT, compressionLevel, mapperService, logger); } /** The name for this codec. */ diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index 786dfb8c68ec9..a1ce263f26b1f 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -82,7 +82,6 @@ public final class EngineConfig { private volatile boolean enableGcDeletes = true; private final TimeValue flushMergesAfter; private final String codecName; - private final int compressionLevel; private final ThreadPool threadPool; private final Engine.Warmer warmer; private final Store store; @@ -143,6 +142,12 @@ public Supplier retentionLeasesSupplier() { return s; } }, Property.IndexScope, Property.NodeScope); + + /** + * Index setting to change the compression level of zstd and zstd_no_dict lucene codecs. + * Compression Level gives a trade-off between compression ratio and speed. The higher compression level results in higher compression ratio but slower compression and decompression speeds. + * This setting is not realtime updateable. + */ public static final Setting INDEX_CODEC_COMPRESSION_LEVEL_SETTING = Setting.intSetting( "index.codec.compression_level", 6, @@ -187,7 +192,6 @@ private EngineConfig(Builder builder) { this.codecService = builder.codecService; this.eventListener = builder.eventListener; codecName = builder.indexSettings.getValue(INDEX_CODEC_SETTING); - compressionLevel = builder.indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); // We need to make the indexing buffer for this shard at least as large // as the amount of memory that is available for all engines on the // local node so that decisions to flush segments to disk are made by @@ -259,9 +263,6 @@ public boolean isEnableGcDeletes() { *

*/ public Codec getCodec() { - if (codecName.equals(CodecService.ZSTD_CODEC) || codecName.equals(CodecService.ZSTD_NO_DICT_CODEC)) { - return codecService.codec(codecName, compressionLevel); - } return codecService.codec(codecName); } diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index 96d61e43045dc..180a30b5c3ff0 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -43,6 +43,7 @@ import org.apache.lucene.index.SegmentReader; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs; +import org.opensearch.common.Randomness; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; @@ -83,25 +84,31 @@ public void testBestCompression() throws Exception { public void testZstd() throws Exception { Codec codec = createCodecService(false).codec("zstd"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDict() throws Exception { Codec codec = createCodecService(false).codec("zstd_no_dict"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdWithCompressionLevel() throws Exception { - Codec codec = createCodecService(false).codec("zstd", 1); + int randomCompressionLevel = generateRandomNumber(6, 1); + Codec codec = createCodecService(randomCompressionLevel).codec("zstd"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(1, storedFieldsFormat.getCompressionLevel()); + assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictWithCompressionLevel() throws Exception { - Codec codec = createCodecService(false).codec("zstd_no_dict", 1); + int randomCompressionLevel = generateRandomNumber(6, 1); + Codec codec = createCodecService(randomCompressionLevel).codec("zstd_no_dict"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(1, storedFieldsFormat.getCompressionLevel()); + assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } public void testDefaultMapperServiceNull() throws Exception { @@ -117,11 +124,15 @@ public void testBestCompressionMapperServiceNull() throws Exception { public void testZstdMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd_no_dict"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testExceptionCodecNull() { @@ -149,6 +160,19 @@ private CodecService createCodecService(boolean isMapperServiceNull) throws IOEx return new CodecService(null, LogManager.getLogger("test")); } Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); + return buildCodecService(nodeSettings); + } + + private CodecService createCodecService(int randomCompressionLevel) throws IOException { + Settings nodeSettings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) + .put("index.codec.compression_level", randomCompressionLevel) + .build(); + return buildCodecService(nodeSettings); + } + + private CodecService buildCodecService(Settings nodeSettings) throws IOException { + IndexSettings settings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); SimilarityService similarityService = new SimilarityService(settings, null, Collections.emptyMap()); IndexAnalyzers indexAnalyzers = createTestAnalysis(settings, nodeSettings).indexAnalyzers; @@ -180,4 +204,9 @@ private SegmentReader getSegmentReader(Codec codec) throws IOException { dir.close(); return sr; } + + private int generateRandomNumber(int max, int min) { + return Randomness.get().nextInt(max - min + 1) + min; + } + } diff --git a/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java b/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java index 755d271fe5934..a0ef70f0be6b0 100644 --- a/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java @@ -8,6 +8,7 @@ package org.opensearch.index.codec.customcodecs; +import org.opensearch.common.Randomness; import org.opensearch.test.OpenSearchTestCase; public class Lucene95CustomStoredFieldsFormatTests extends OpenSearchTestCase { @@ -25,21 +26,27 @@ public void testZstdNoDictLucene95CustomCodecMode() { } public void testZstdModeWithCompressionLevel() { + int randomCompressionLevel = generateRandomNumber(6, 1); Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( Lucene95CustomCodec.Mode.ZSTD, - 1 + randomCompressionLevel ); assertEquals(Lucene95CustomCodec.Mode.ZSTD, lucene95CustomStoredFieldsFormat.getMode()); - assertEquals(1, lucene95CustomStoredFieldsFormat.getCompressionLevel()); + assertEquals(randomCompressionLevel, lucene95CustomStoredFieldsFormat.getCompressionLevel()); } public void testZstdNoDictLucene95CustomCodecModeWithCompressionLevel() { + int randomCompressionLevel = generateRandomNumber(6, 1); Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( Lucene95CustomCodec.Mode.ZSTD_NO_DICT, - 1 + randomCompressionLevel ); assertEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, lucene95CustomStoredFieldsFormat.getMode()); - assertEquals(1, lucene95CustomStoredFieldsFormat.getCompressionLevel()); + assertEquals(randomCompressionLevel, lucene95CustomStoredFieldsFormat.getCompressionLevel()); + } + + private int generateRandomNumber(int max, int min) { + return Randomness.get().nextInt(max - min + 1) + min; } } From f0f868e7668f90b088ad46198dfce41b022863f4 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Sun, 2 Jul 2023 14:56:23 +0530 Subject: [PATCH 3/5] refactoring codec service Signed-off-by: Sarthak Aggarwal --- .../index/codec/CorrelationCodecService.java | 2 +- .../indices/IndexingMemoryControllerIT.java | 2 +- .../opensearch/index/codec/CodecService.java | 10 ++++---- .../opensearch/index/shard/IndexShard.java | 2 +- .../gateway/PrimaryShardAllocatorTests.java | 11 +++++++-- .../opensearch/index/codec/CodecTests.java | 24 +++++++------------ ...Lucene95CustomStoredFieldsFormatTests.java | 9 ++----- .../engine/EngineConfigFactoryTests.java | 8 ++++--- .../index/engine/InternalEngineTests.java | 8 +++---- .../index/shard/IndexShardTests.java | 2 +- .../index/shard/RefreshListenersTests.java | 2 +- .../IndexingMemoryControllerTests.java | 2 +- .../OngoingSegmentReplicationsTests.java | 2 +- .../replication/common/CopyStateTests.java | 11 ++++++++- .../index/engine/EngineTestCase.java | 12 +++++----- 15 files changed, 58 insertions(+), 49 deletions(-) diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecService.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecService.java index 0b70e7ed66f3d..09d5e1d2c19e3 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecService.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecService.java @@ -27,7 +27,7 @@ public class CorrelationCodecService extends CodecService { * @param codecServiceConfig Generic codec service config */ public CorrelationCodecService(CodecServiceConfig codecServiceConfig) { - super(codecServiceConfig.getMapperService(), codecServiceConfig.getLogger()); + super(codecServiceConfig.getMapperService(), codecServiceConfig.getIndexSettings(), codecServiceConfig.getLogger()); mapperService = codecServiceConfig.getMapperService(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndexingMemoryControllerIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndexingMemoryControllerIT.java index 72de689562e7a..c842ee8564041 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/IndexingMemoryControllerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndexingMemoryControllerIT.java @@ -89,7 +89,7 @@ EngineConfig engineConfigWithLargerIndexingMemory(EngineConfig config) { .mergePolicy(config.getMergePolicy()) .analyzer(config.getAnalyzer()) .similarity(config.getSimilarity()) - .codecService(new CodecService(null, LogManager.getLogger(IndexingMemoryControllerIT.class))) + .codecService(new CodecService(null, indexSettings, LogManager.getLogger(IndexingMemoryControllerIT.class))) .eventListener(config.getEventListener()) .queryCache(config.getQueryCache()) .queryCachingPolicy(config.getQueryCachingPolicy()) diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index 07203588eedd0..5a605c1c67f26 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -38,11 +38,13 @@ import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode; import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; +import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.customcodecs.ZstdCodec; import org.opensearch.index.codec.customcodecs.ZstdNoDictCodec; import org.opensearch.index.mapper.MapperService; import java.util.Map; +import java.util.Objects; import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; @@ -67,15 +69,15 @@ public class CodecService { public static final String ZSTD_CODEC = "zstd"; public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; - public CodecService(@Nullable MapperService mapperService, Logger logger) { + public CodecService(@Nullable MapperService mapperService, IndexSettings indexSettings, Logger logger) { final MapBuilder codecs = MapBuilder.newMapBuilder(); + int compressionLevel = Objects.requireNonNull(indexSettings).getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); if (mapperService == null) { codecs.put(DEFAULT_CODEC, new Lucene95Codec()); codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION)); - codecs.put(ZSTD_CODEC, new ZstdCodec()); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec()); + codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel)); } else { - int compressionLevel = mapperService.getIndexSettings().getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 01c0a12d463ea..e7720e9343b80 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -368,7 +368,7 @@ public IndexShard( assert shardRouting.initializing(); this.shardRouting = shardRouting; final Settings settings = indexSettings.getSettings(); - this.codecService = new CodecService(mapperService, logger); + this.codecService = new CodecService(mapperService, indexSettings, logger); this.warmer = warmer; this.similarityService = similarityService; Objects.requireNonNull(store, "Store must be provided to the index shard"); diff --git a/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java b/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java index d15bb49f5342a..9a264781b0644 100644 --- a/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java +++ b/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java @@ -60,7 +60,9 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; +import org.opensearch.env.Environment; import org.opensearch.env.ShardLockObtainFailedException; +import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecService; import org.opensearch.index.shard.ShardId; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; @@ -69,6 +71,7 @@ import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotShardSizeInfo; import org.junit.Before; +import org.opensearch.test.IndexSettingsModule; import java.util.Arrays; import java.util.Collections; @@ -803,21 +806,25 @@ public TestAllocator addData( } public TestAllocator addData(DiscoveryNode node, String allocationId, boolean primary) { + Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", nodeSettings); return addData( node, allocationId, primary, - ReplicationCheckpoint.empty(shardId, new CodecService(null, null).codec("default").getName()), + ReplicationCheckpoint.empty(shardId, new CodecService(null, indexSettings, null).codec("default").getName()), null ); } public TestAllocator addData(DiscoveryNode node, String allocationId, boolean primary, @Nullable Exception storeException) { + Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", nodeSettings); return addData( node, allocationId, primary, - ReplicationCheckpoint.empty(shardId, new CodecService(null, null).codec("default").getName()), + ReplicationCheckpoint.empty(shardId, new CodecService(null, indexSettings, null).codec("default").getName()), storeException ); } diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index 180a30b5c3ff0..97b0eaccfbe52 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -43,7 +43,6 @@ import org.apache.lucene.index.SegmentReader; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs; -import org.opensearch.common.Randomness; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; @@ -96,7 +95,7 @@ public void testZstdNoDict() throws Exception { } public void testZstdWithCompressionLevel() throws Exception { - int randomCompressionLevel = generateRandomNumber(6, 1); + int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel).codec("zstd"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); @@ -104,7 +103,7 @@ public void testZstdWithCompressionLevel() throws Exception { } public void testZstdNoDictWithCompressionLevel() throws Exception { - int randomCompressionLevel = generateRandomNumber(6, 1); + int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel).codec("zstd_no_dict"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); @@ -155,11 +154,10 @@ private void assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode expect } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { - + Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); if (isMapperServiceNull) { - return new CodecService(null, LogManager.getLogger("test")); + return new CodecService(null, IndexSettingsModule.newIndexSettings("_na", nodeSettings), LogManager.getLogger("test")); } - Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); return buildCodecService(nodeSettings); } @@ -173,12 +171,12 @@ private CodecService createCodecService(int randomCompressionLevel) throws IOExc private CodecService buildCodecService(Settings nodeSettings) throws IOException { - IndexSettings settings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); - SimilarityService similarityService = new SimilarityService(settings, null, Collections.emptyMap()); - IndexAnalyzers indexAnalyzers = createTestAnalysis(settings, nodeSettings).indexAnalyzers; + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); + SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap()); + IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, nodeSettings).indexAnalyzers; MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); MapperService service = new MapperService( - settings, + indexSettings, indexAnalyzers, xContentRegistry(), similarityService, @@ -187,7 +185,7 @@ private CodecService buildCodecService(Settings nodeSettings) throws IOException () -> false, null ); - return new CodecService(service, LogManager.getLogger("test")); + return new CodecService(service, indexSettings, LogManager.getLogger("test")); } private SegmentReader getSegmentReader(Codec codec) throws IOException { @@ -205,8 +203,4 @@ private SegmentReader getSegmentReader(Codec codec) throws IOException { return sr; } - private int generateRandomNumber(int max, int min) { - return Randomness.get().nextInt(max - min + 1) + min; - } - } diff --git a/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java b/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java index a0ef70f0be6b0..e87fb56770e4c 100644 --- a/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java @@ -8,7 +8,6 @@ package org.opensearch.index.codec.customcodecs; -import org.opensearch.common.Randomness; import org.opensearch.test.OpenSearchTestCase; public class Lucene95CustomStoredFieldsFormatTests extends OpenSearchTestCase { @@ -26,7 +25,7 @@ public void testZstdNoDictLucene95CustomCodecMode() { } public void testZstdModeWithCompressionLevel() { - int randomCompressionLevel = generateRandomNumber(6, 1); + int randomCompressionLevel = randomIntBetween(1, 6); Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( Lucene95CustomCodec.Mode.ZSTD, randomCompressionLevel @@ -36,7 +35,7 @@ public void testZstdModeWithCompressionLevel() { } public void testZstdNoDictLucene95CustomCodecModeWithCompressionLevel() { - int randomCompressionLevel = generateRandomNumber(6, 1); + int randomCompressionLevel = randomIntBetween(1, 6); Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( Lucene95CustomCodec.Mode.ZSTD_NO_DICT, randomCompressionLevel @@ -45,8 +44,4 @@ public void testZstdNoDictLucene95CustomCodecModeWithCompressionLevel() { assertEquals(randomCompressionLevel, lucene95CustomStoredFieldsFormat.getCompressionLevel()); } - private int generateRandomNumber(int max, int min) { - return Randomness.get().nextInt(max - min + 1) + min; - } - } diff --git a/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java b/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java index f8bedc76ea994..bf9a86cff8b76 100644 --- a/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java @@ -178,7 +178,7 @@ public Optional getEngineFactory(final IndexSettings indexSetting @Override public Optional getCustomCodecService(IndexSettings indexSettings) { - return Optional.of(new CodecService(null, LogManager.getLogger(getClass()))); + return Optional.of(new CodecService(null, indexSettings, LogManager.getLogger(getClass()))); } @Override @@ -195,7 +195,7 @@ public Optional getEngineFactory(final IndexSettings indexSetting @Override public Optional getCustomCodecService(IndexSettings indexSettings) { - return Optional.of(new CodecService(null, LogManager.getLogger(getClass()))); + return Optional.of(new CodecService(null, indexSettings, LogManager.getLogger(getClass()))); } } @@ -207,7 +207,9 @@ public Optional getEngineFactory(final IndexSettings indexSetting @Override public Optional getCustomCodecServiceFactory(IndexSettings indexSettings) { - return Optional.of(config -> new CodecService(config.getMapperService(), LogManager.getLogger(getClass()))); + return Optional.of( + config -> new CodecService(config.getMapperService(), config.getIndexSettings(), LogManager.getLogger(getClass())) + ); } } diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index 3bd8e09cab777..209c9925bb06d 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -3226,7 +3226,7 @@ public void testFailStart() throws IOException { } public void testSettings() { - CodecService codecService = new CodecService(null, logger); + CodecService codecService = new CodecService(null, engine.config().getIndexSettings(), logger); LiveIndexWriterConfig currentIndexWriterConfig = engine.getCurrentIndexWriterConfig(); assertEquals(engine.config().getCodec().getName(), codecService.codec(codecName).getName()); @@ -3696,7 +3696,7 @@ public void testRecoverFromForeignTranslog() throws IOException { .mergePolicy(newMergePolicy()) .analyzer(config.getAnalyzer()) .similarity(config.getSimilarity()) - .codecService(new CodecService(null, logger)) + .codecService(new CodecService(null, config.getIndexSettings(), logger)) .eventListener(config.getEventListener()) .queryCache(IndexSearcher.getDefaultQueryCache()) .queryCachingPolicy(IndexSearcher.getDefaultQueryCachingPolicy()) @@ -3738,7 +3738,7 @@ public CustomTranslogDeletionPolicy(IndexSettings indexSettings, Supplier Date: Tue, 4 Jul 2023 18:28:06 +0530 Subject: [PATCH 4/5] index settings assertion Signed-off-by: Sarthak Aggarwal --- .../main/java/org/opensearch/index/codec/CodecService.java | 4 ++-- .../src/test/java/org/opensearch/index/codec/CodecTests.java | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index 5a605c1c67f26..da64e2954b0d8 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -44,7 +44,6 @@ import org.opensearch.index.mapper.MapperService; import java.util.Map; -import java.util.Objects; import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; @@ -71,7 +70,8 @@ public class CodecService { public CodecService(@Nullable MapperService mapperService, IndexSettings indexSettings, Logger logger) { final MapBuilder codecs = MapBuilder.newMapBuilder(); - int compressionLevel = Objects.requireNonNull(indexSettings).getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + assert null != indexSettings; + int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); if (mapperService == null) { codecs.put(DEFAULT_CODEC, new Lucene95Codec()); codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION)); diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index 97b0eaccfbe52..b0d904392407c 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -138,6 +138,10 @@ public void testExceptionCodecNull() { assertThrows(IllegalArgumentException.class, () -> createCodecService(true).codec(null)); } + public void testExceptionIndexSettingsNull() { + assertThrows(AssertionError.class, () -> new CodecService(null, null, LogManager.getLogger("test"))); + } + // write some docs with it, inspect .si to see this was the used compression private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); From fddebace1473a537cf5fb7b98820d02abb9fdd09 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Thu, 6 Jul 2023 14:06:32 +0530 Subject: [PATCH 5/5] default compression level change in the settings Signed-off-by: Sarthak Aggarwal --- .../src/main/java/org/opensearch/index/engine/EngineConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index a1ce263f26b1f..f4aa5e248ac31 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -150,7 +150,7 @@ public Supplier retentionLeasesSupplier() { */ public static final Setting INDEX_CODEC_COMPRESSION_LEVEL_SETTING = Setting.intSetting( "index.codec.compression_level", - 6, + 3, 1, 6, Property.IndexScope