Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Add configuration for file cache size to max remote data ratio to prevent oversubscription of file cache ([#8606](https://github.com/opensearch-project/OpenSearch/pull/8606))
- Disallow compression level to be set for default and best_compression index codecs ([#8737]()https://github.com/opensearch-project/OpenSearch/pull/8737)
- Prioritize replica shard movement during shard relocation ([#8875](https://github.com/opensearch-project/OpenSearch/pull/8875))
- Introducing Default and Best Compression codecs as their algorithm name ([#9123]()https://github.com/opensearch-project/OpenSearch/pull/9123)

### Dependencies
- Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,15 @@ public void testReindexingMultipleCodecs() throws InterruptedException, Executio
Map<String, String> codecMap = Map.of(
"best_compression",
"BEST_COMPRESSION",
"zlib",
"BEST_COMPRESSION",
"zstd_no_dict",
"ZSTD_NO_DICT",
"zstd",
"ZSTD",
"default",
"BEST_SPEED",
"lz4",
"BEST_SPEED"
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,15 @@ public void testForceMergeMultipleCodecs() throws ExecutionException, Interrupte
Map<String, String> codecMap = Map.of(
"best_compression",
"BEST_COMPRESSION",
"zlib",
"BEST_COMPRESSION",
"zstd_no_dict",
"ZSTD_NO_DICT",
"zstd",
"ZSTD",
"default",
"BEST_SPEED",
"lz4",
"BEST_SPEED"
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,9 @@ public class CodecService {
private final Map<String, Codec> codecs;

public static final String DEFAULT_CODEC = "default";
public static final String LZ4 = "lz4";
public static final String BEST_COMPRESSION_CODEC = "best_compression";
public static final String ZLIB = "zlib";
/**
* the raw unfiltered lucene default. useful for testing
*/
Expand All @@ -74,12 +76,16 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe
int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING);
if (mapperService == null) {
codecs.put(DEFAULT_CODEC, new Lucene95Codec());
codecs.put(LZ4, new Lucene95Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION));
codecs.put(ZLIB, new Lucene95Codec(Mode.BEST_COMPRESSION));
codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel));
codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel));
} else {
codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel));
codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,15 +130,18 @@ public Supplier<RetentionLeases> retentionLeasesSupplier() {
public static final Setting<String> INDEX_CODEC_SETTING = new Setting<>("index.codec", "default", s -> {
switch (s) {
case "default":
case "lz4":
case "best_compression":
case "zlib":
case "zstd":
case "zstd_no_dict":
case "lucene_default":
return s;
default:
if (Codec.availableCodecs().contains(s) == false) { // we don't error message the not officially supported ones
throw new IllegalArgumentException(
"unknown value for [index.codec] must be one of [default, best_compression, zstd, zstd_no_dict] but was: " + s
"unknown value for [index.codec] must be one of [default, lz4, best_compression, zlib, zstd, zstd_no_dict] but was: "
+ s
);
}
return s;
Expand Down Expand Up @@ -182,8 +185,10 @@ private static void doValidateCodecSettings(final String codec) {
case "zstd_no_dict":
return;
case "best_compression":
case "zlib":
case "lucene_default":
case "default":
case "lz4":
break;
default:
if (Codec.availableCodecs().contains(codec)) {
Expand Down
12 changes: 12 additions & 0 deletions server/src/test/java/org/opensearch/index/codec/CodecTests.java
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,18 @@ public void testBestCompression() throws Exception {
assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec);
}

public void testLZ4() throws Exception {
Codec codec = createCodecService(false).codec("lz4");
assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec);
assert codec instanceof PerFieldMappingPostingFormatCodec;
}

public void testZlib() throws Exception {
Codec codec = createCodecService(false).codec("zlib");
assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec);
assert codec instanceof PerFieldMappingPostingFormatCodec;
}

public void testZstd() throws Exception {
Codec codec = createCodecService(false).codec("zstd");
assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,9 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase {
*/
public static final List<String> CODECS = List.of(
CodecService.DEFAULT_CODEC,
CodecService.LZ4,
CodecService.BEST_COMPRESSION_CODEC,
CodecService.ZLIB,
CodecService.ZSTD_CODEC,
CodecService.ZSTD_NO_DICT_CODEC
);
Expand Down