diff --git a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java index 49b0971..8f71cf6 100644 --- a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java +++ b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java @@ -26,7 +26,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; -import org.opensearch.index.codec.customcodecs.Lucene103QatCodec; +import org.opensearch.index.codec.customcodecs.Lucene104QatCodec; import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -107,9 +107,9 @@ public void testCreateIndexWithQatSPICodecWithQatHardwareUnavailable() throws IO .put( "index.codec", randomFrom( - Lucene103QatCodec.Mode.QAT_LZ4.getCodec(), - Lucene103QatCodec.Mode.QAT_DEFLATE.getCodec(), - Lucene103QatCodec.Mode.QAT_ZSTD.getCodec() + Lucene104QatCodec.Mode.QAT_LZ4.getCodec(), + Lucene104QatCodec.Mode.QAT_DEFLATE.getCodec(), + Lucene104QatCodec.Mode.QAT_ZSTD.getCodec() ) ) .put("index.codec.compression_level", randomIntBetween(1, 6)) diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomAdditionalCodecs.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomAdditionalCodecs.java index 88c6adc..8227509 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomAdditionalCodecs.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomAdditionalCodecs.java @@ -42,39 +42,39 @@ public Map getCodecs(MapperService mapperService, IndexSettings i final int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); final MapBuilder codecs = MapBuilder.newMapBuilder(); if (mapperService == null) { - codecs.put(ZSTD_CODEC, new Zstd103Codec(compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict103Codec(compressionLevel)); + codecs.put(ZSTD_CODEC, new Zstd104Codec(compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict104Codec(compressionLevel)); if (QatZipperFactory.isQatAvailable()) { codecs.put( QAT_LZ4_CODEC, - new QatLz4103Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }) + new QatLz4104Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }) ); - codecs.put(QAT_DEFLATE_CODEC, new QatDeflate103Codec(compressionLevel, () -> { + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate104Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })); codecs.put( QAT_ZSTD_CODEC, - new QatZstd103Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }) + new QatZstd104Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }) ); } } else { - codecs.put(ZSTD_CODEC, new Zstd103Codec(compressionLevel, defaultCodec)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict103Codec(compressionLevel, defaultCodec)); + codecs.put(ZSTD_CODEC, new Zstd104Codec(compressionLevel, defaultCodec)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict104Codec(compressionLevel, defaultCodec)); if (QatZipperFactory.isQatAvailable()) { codecs.put( QAT_LZ4_CODEC, - new QatLz4103Codec( + new QatLz4104Codec( compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }, defaultCodec ) ); - codecs.put(QAT_DEFLATE_CODEC, new QatDeflate103Codec(compressionLevel, () -> { + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate104Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }, defaultCodec)); codecs.put( QAT_ZSTD_CODEC, - new QatZstd103Codec( + new QatZstd104Codec( compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }, defaultCodec diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java index 809635e..abda987 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -76,8 +76,8 @@ private static boolean isQatCodec(String codecName) { || codecName.equals(Lucene101QatCodec.Mode.QAT_LZ4.getCodec()) || codecName.equals(Lucene101QatCodec.Mode.QAT_DEFLATE.getCodec()) || codecName.equals(Lucene101QatCodec.Mode.QAT_ZSTD.getCodec()) - || codecName.equals(Lucene103QatCodec.Mode.QAT_LZ4.getCodec()) - || codecName.equals(Lucene103QatCodec.Mode.QAT_DEFLATE.getCodec()) - || codecName.equals(Lucene103QatCodec.Mode.QAT_ZSTD.getCodec()); + || codecName.equals(Lucene104QatCodec.Mode.QAT_LZ4.getCodec()) + || codecName.equals(Lucene104QatCodec.Mode.QAT_DEFLATE.getCodec()) + || codecName.equals(Lucene104QatCodec.Mode.QAT_ZSTD.getCodec()); } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene104CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene104CustomCodec.java new file mode 100644 index 0000000..afe9415 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene104CustomCodec.java @@ -0,0 +1,112 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104Codec; + +import java.util.Set; +import java.util.function.Supplier; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. + * Supports two modes zstd and zstd_no_dict. + * Uses Lucene104 as the delegate codec + * + * @opensearch.internal + */ +public abstract class Lucene104CustomCodec extends FilterCodec { + + /** Each mode represents a compression algorithm. */ + public enum Mode { + /** + * ZStandard mode with dictionary + */ + ZSTD("ZSTD104", Set.of("zstd")), + /** + * ZStandard mode without dictionary + */ + ZSTD_NO_DICT("ZSTDNODICT104", Set.of("zstd_no_dict")); + + private final String codec; + private final Set aliases; + + Mode(String codec, Set aliases) { + this.codec = codec; + this.aliases = aliases; + } + + /** + * Returns the Codec that is registered with Lucene + */ + public String getCodec() { + return codec; + } + + /** + * Returns the aliases of the Codec + */ + public Set getAliases() { + return aliases; + } + } + + private final StoredFieldsFormat storedFieldsFormat; + + /** + * Creates a new compression codec with the default compression level. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + */ + public Lucene104CustomCodec(Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new compression codec with the given compression level. We use + * lowercase letters when registering the codec so that we remain consistent with + * the other compression codecs: default, lucene_default, and best_compression. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + * @param compressionLevel The compression level. + */ + public Lucene104CustomCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene104Codec()); + this.storedFieldsFormat = new Lucene104CustomStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use + * lowercase letters when registering the codec so that we remain consistent with + * the other compression codecs: default, lucene_default, and best_compression. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + * @param compressionLevel The compression level. + * @param defaultCodecSupplier Default OpenSearch codec supplier + */ + public Lucene104CustomCodec(Mode mode, int compressionLevel, Supplier defaultCodecSupplier) { + super(mode.getCodec(), defaultCodecSupplier.get()); + this.storedFieldsFormat = new Lucene104CustomStoredFieldsFormat(mode, compressionLevel); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene104CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene104CustomStoredFieldsFormat.java new file mode 100644 index 0000000..97bd211 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene104CustomStoredFieldsFormat.java @@ -0,0 +1,140 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.StoredFieldsReader; +import org.apache.lucene.codecs.StoredFieldsWriter; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +import java.io.IOException; +import java.util.Objects; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** Stored field format used by pluggable codec */ +public class Lucene104CustomStoredFieldsFormat extends StoredFieldsFormat { + + /** A key that we use to map to a mode */ + public static final String MODE_KEY = Lucene104CustomStoredFieldsFormat.class.getSimpleName() + ".mode"; + + protected static final int ZSTD_BLOCK_LENGTH = 10 * 48 * 1024; + protected static final int ZSTD_MAX_DOCS_PER_BLOCK = 4096; + protected static final int ZSTD_BLOCK_SHIFT = 10; + + private final CompressionMode zstdCompressionMode; + private final CompressionMode zstdNoDictCompressionMode; + + private final Lucene104CustomCodec.Mode mode; + private final int compressionLevel; + + /** default constructor */ + public Lucene104CustomStoredFieldsFormat() { + this(Lucene104CustomCodec.Mode.ZSTD, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents ZSTD or ZSTDNODICT + */ + public Lucene104CustomStoredFieldsFormat(Lucene104CustomCodec.Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents ZSTD or ZSTDNODICT + * @param compressionLevel The compression level for the mode. + */ + public Lucene104CustomStoredFieldsFormat(Lucene104CustomCodec.Mode mode, int compressionLevel) { + this.mode = Objects.requireNonNull(mode); + this.compressionLevel = compressionLevel; + zstdCompressionMode = new ZstdCompressionMode(compressionLevel); + zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); + } + + /** + * Returns a {@link StoredFieldsReader} to load stored fields. + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param fn The fieldInfos. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { + if (si.getAttribute(MODE_KEY) != null) { + String value = si.getAttribute(MODE_KEY); + Lucene104CustomCodec.Mode mode = Lucene104CustomCodec.Mode.valueOf(value); + return impl(mode).fieldsReader(directory, si, fn, context); + } else { + throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name); + } + } + + /** + * Returns a {@link StoredFieldsReader} to write stored fields. + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { + String previous = si.putAttribute(MODE_KEY, mode.name()); + if (previous != null && previous.equals(mode.name()) == false) { + throw new IllegalStateException( + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + ); + } + return impl(mode).fieldsWriter(directory, si, context); + } + + StoredFieldsFormat impl(Lucene104CustomCodec.Mode mode) { + switch (mode) { + case ZSTD: + return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstd", this.zstdCompressionMode); + case ZSTD_NO_DICT: + return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstdNoDict", this.zstdNoDictCompressionMode); + default: + throw new IllegalStateException("Unsupported compression mode: " + mode); + } + } + + private StoredFieldsFormat getCustomCompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode) { + return new Lucene90CompressingStoredFieldsFormat( + formatName, + compressionMode, + ZSTD_BLOCK_LENGTH, + ZSTD_MAX_DOCS_PER_BLOCK, + ZSTD_BLOCK_SHIFT + ); + } + + public Lucene104CustomCodec.Mode getMode() { + return mode; + } + + /** + * Returns the compression level. + */ + public int getCompressionLevel() { + return compressionLevel; + } + + public CompressionMode getCompressionMode() { + return mode == Lucene104CustomCodec.Mode.ZSTD_NO_DICT ? zstdNoDictCompressionMode : zstdCompressionMode; + } + +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene104QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene104QatCodec.java new file mode 100644 index 0000000..b143208 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene104QatCodec.java @@ -0,0 +1,138 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104Codec; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. + * + * @opensearch.internal + */ +public abstract class Lucene104QatCodec extends FilterCodec { + /** Each mode represents a compression algorithm. */ + public enum Mode { + /** QAT lz4 mode. */ + QAT_LZ4("QATLZ4104", Set.of("qat_lz4")), + + /** QAT deflate mode. */ + QAT_DEFLATE("QATDEFLATE104", Set.of("qat_deflate")), + + /** QAT zstd mode. */ + QAT_ZSTD("QATZSTD104", Set.of("qat_zstd")); + + private final String codec; + private final Set aliases; + + Mode(String codec, Set aliases) { + this.codec = codec; + this.aliases = aliases; + } + + /** Returns the Codec that is registered with Lucene */ + public String getCodec() { + return codec; + } + + /** Returns the aliases of the Codec */ + public Set getAliases() { + return aliases; + } + } + + /** The default compression mode. */ + public static final Mode DEFAULT_COMPRESSION_MODE = Mode.QAT_LZ4; + + private final StoredFieldsFormat storedFieldsFormat; + + /** + * Creates a new compression codec with the default compression level. + * + * @param mode The compression codec (QAT_LZ4, QAT_DEFLATE, or QAT_ZSTD). + */ + public Lucene104QatCodec(Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4, QAT_DEFLATE, or QAT_ZSTD). + * @param compressionLevel The compression level. + */ + public Lucene104QatCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene104Codec()); + this.storedFieldsFormat = new Lucene104QatStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4, QAT_DEFLATE, or QAT_ZSTD). + * @param compressionLevel The compression level. + * @param supplier supplier for QAT mode. + */ + public Lucene104QatCodec(Mode mode, int compressionLevel, Supplier supplier) { + super(mode.getCodec(), new Lucene104Codec()); + this.storedFieldsFormat = new Lucene104QatStoredFieldsFormat(mode, compressionLevel, supplier); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4, QAT_DEFLATE, or QAT_ZSTD). + * @param defaultCodecSupplier default opensearch codec supplier + * @param compressionLevel The compression level. + */ + public Lucene104QatCodec(Mode mode, Supplier defaultCodecSupplier, int compressionLevel) { + super(mode.getCodec(), defaultCodecSupplier.get()); + this.storedFieldsFormat = new Lucene104QatStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4, QAT_DEFLATE, or QAT_ZSTD). + * @param compressionLevel The compression level. + * @param supplier supplier for QAT mode. + * @param defaultCodecSupplier default opensearch codec supplier + */ + public Lucene104QatCodec(Mode mode, int compressionLevel, Supplier supplier, Supplier defaultCodecSupplier) { + super(mode.getCodec(), defaultCodecSupplier.get()); + this.storedFieldsFormat = new Lucene104QatStoredFieldsFormat(mode, compressionLevel, supplier); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene104QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene104QatStoredFieldsFormat.java new file mode 100644 index 0000000..65ed128 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene104QatStoredFieldsFormat.java @@ -0,0 +1,209 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.StoredFieldsReader; +import org.apache.lucene.codecs.StoredFieldsWriter; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +import java.io.IOException; +import java.util.Objects; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_QAT_MODE; + +/** Stored field format used by pluggable codec */ +public class Lucene104QatStoredFieldsFormat extends StoredFieldsFormat { + + /** A key that we use to map to a mode */ + public static final String MODE_KEY = Lucene104QatStoredFieldsFormat.class.getSimpleName() + ".mode"; + + private static final int QAT_DEFLATE_BLOCK_LENGTH = 10 * 48 * 1024; + private static final int QAT_DEFLATE_MAX_DOCS_PER_BLOCK = 4096; + private static final int QAT_DEFLATE_BLOCK_SHIFT = 10; + + private static final int QAT_LZ4_BLOCK_LENGTH = 10 * 8 * 1024; + private static final int QAT_LZ4_MAX_DOCS_PER_BLOCK = 4096; + private static final int QAT_LZ4_BLOCK_SHIFT = 10; + + private static final int QAT_ZSTD_BLOCK_LENGTH = 10 * 48 * 1024; + private static final int QAT_ZSTD_MAX_DOCS_PER_BLOCK = 4096; + private static final int QAT_ZSTD_BLOCK_SHIFT = 10; + + private final QatCompressionMode qatCompressionMode; + private final Lucene104QatCodec.Mode mode; + + /** default constructor */ + public Lucene104QatStoredFieldsFormat() { + this(Lucene104QatCodec.DEFAULT_COMPRESSION_MODE, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents QAT_LZ4, QAT_DEFLATE, or QAT_ZSTD + */ + public Lucene104QatStoredFieldsFormat(Lucene104QatCodec.Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents QAT_LZ4, QAT_DEFLATE, or QAT_ZSTD + * @param compressionLevel The compression level for the mode. + */ + public Lucene104QatStoredFieldsFormat(Lucene104QatCodec.Mode mode, int compressionLevel) { + this(mode, compressionLevel, () -> { return DEFAULT_QAT_MODE; }); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents QAT_LZ4, QAT_DEFLATE, or QAT_ZSTD + * @param supplier a supplier for QAT acceleration mode. + */ + public Lucene104QatStoredFieldsFormat(Lucene104QatCodec.Mode mode, Supplier supplier) { + this(mode, DEFAULT_COMPRESSION_LEVEL, supplier); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents QAT_LZ4, QAT_DEFLATE, or QAT_ZSTD + * @param compressionLevel The compression level for the mode. + * @param supplier a supplier for QAT acceleration mode. + */ + public Lucene104QatStoredFieldsFormat(Lucene104QatCodec.Mode mode, int compressionLevel, Supplier supplier) { + this.mode = Objects.requireNonNull(mode); + qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier); + } + + /** + * Returns a {@link StoredFieldsReader} to load stored fields. + * + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param fn The fieldInfos. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { + if (si.getAttribute(MODE_KEY) != null) { + String value = si.getAttribute(MODE_KEY); + Lucene104QatCodec.Mode mode = Lucene104QatCodec.Mode.valueOf(value); + return impl(mode).fieldsReader(directory, si, fn, context); + } else { + throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name); + } + } + + /** + * Returns a {@link StoredFieldsReader} to write stored fields. + * + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { + String previous = si.putAttribute(MODE_KEY, mode.name()); + if (previous != null && previous.equals(mode.name()) == false) { + throw new IllegalStateException( + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + ); + } + return impl(mode).fieldsWriter(directory, si, context); + } + + private StoredFieldsFormat impl(Lucene104QatCodec.Mode mode) { + switch (mode) { + case QAT_LZ4: + return getQatCompressingStoredFieldsFormat( + "QatStoredFieldsLz4", + qatCompressionMode, + QAT_LZ4_BLOCK_LENGTH, + QAT_LZ4_MAX_DOCS_PER_BLOCK, + QAT_LZ4_BLOCK_SHIFT + ); + case QAT_DEFLATE: + return getQatCompressingStoredFieldsFormat( + "QatStoredFieldsDeflate", + qatCompressionMode, + QAT_DEFLATE_BLOCK_LENGTH, + QAT_DEFLATE_MAX_DOCS_PER_BLOCK, + QAT_DEFLATE_BLOCK_SHIFT + ); + case QAT_ZSTD: + return getQatCompressingStoredFieldsFormat( + "QatStoredFieldsZstd", + qatCompressionMode, + QAT_ZSTD_BLOCK_LENGTH, + QAT_ZSTD_MAX_DOCS_PER_BLOCK, + QAT_ZSTD_BLOCK_SHIFT + ); + default: + throw new IllegalStateException("Unsupported compression mode: " + mode); + } + } + + private StoredFieldsFormat getQatCompressingStoredFieldsFormat( + String formatName, + CompressionMode compressionMode, + int blockSize, + int maxDocs, + int blockShift + ) { + return new Lucene90CompressingStoredFieldsFormat(formatName, compressionMode, blockSize, maxDocs, blockShift); + } + + /** + * Gets the mode of compression. + * + * @return either QAT_LZ4, QAT_DEFLATE, or QAT_ZSTD + */ + public Lucene104QatCodec.Mode getMode() { + return mode; + } + + /** + * + * @return the CompressionMode instance. + */ + public QatCompressionMode getCompressionMode() { + return qatCompressionMode; + } + + /** + * Returns {@link QatZipper.Algorithm} instance that corresponds codec's {@link Lucene104QatCodec.Mode mode} + * @param mode codec's {@link Lucene104QatCodec.Mode mode} + * @return the {@link QatZipper.Algorithm} instance that corresponds codec's {@link Lucene104QatCodec.Mode mode} + */ + private static QatZipper.Algorithm getAlgorithm(Lucene104QatCodec.Mode mode) { + switch (mode) { + case QAT_LZ4: + return QatZipper.Algorithm.LZ4; + case QAT_DEFLATE: + return QatZipper.Algorithm.DEFLATE; + case QAT_ZSTD: + return QatZipper.Algorithm.ZSTD; + default: + throw new IllegalStateException("Unsupported compression mode: " + mode); + } + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate104Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate104Codec.java new file mode 100644 index 0000000..98953b1 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate104Codec.java @@ -0,0 +1,93 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.Codec; +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * QatDeflate101Codec provides a DEFLATE compressor using the qat-java library. + */ +public class QatDeflate104Codec extends Lucene104QatCodec implements CodecSettings, CodecAliases { + + /** Creates a new QatDeflate101Codec instance with the default compression level. */ + public QatDeflate104Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new QatDeflate101Codec instance. + * + * @param compressionLevel The compression level. + */ + public QatDeflate104Codec(int compressionLevel) { + super(Mode.QAT_DEFLATE, compressionLevel); + } + + /** + * Creates a new QatDeflate101Codec instance with the default compression level. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatDeflate104Codec(int compressionLevel, Supplier supplier) { + super(Mode.QAT_DEFLATE, compressionLevel, supplier); + } + + /** + * Creates a new QatDeflate101Codec instance. + * + * @param defaultCodecSupplier default opensearch codec supplier + * @param compressionLevel The compression level. + */ + public QatDeflate104Codec(Supplier defaultCodecSupplier, int compressionLevel) { + super(Mode.QAT_DEFLATE, defaultCodecSupplier, compressionLevel); + } + + /** + * Creates a new QatDeflate101Codec instance. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode + * @param defaultCodecSupplier default opensearch codec supplier + */ + public QatDeflate104Codec(int compressionLevel, Supplier supplier, Supplier defaultCodecSupplier) { + super(Mode.QAT_DEFLATE, compressionLevel, supplier, defaultCodecSupplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + if (!QatZipperFactory.isQatAvailable()) { + return Set.of(); + } + return Mode.QAT_DEFLATE.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4104Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4104Codec.java new file mode 100644 index 0000000..9458436 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4104Codec.java @@ -0,0 +1,93 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.Codec; +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * QatLz4104Codec provides an LZ4 compressor using the qat-java library. + */ +public class QatLz4104Codec extends Lucene104QatCodec implements CodecSettings, CodecAliases { + + /** Creates a new QatLz4104Codec instance with the default compression level. */ + public QatLz4104Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new QatLz4104Codec instance. + * + * @param compressionLevel The compression level. + */ + public QatLz4104Codec(int compressionLevel) { + super(Mode.QAT_LZ4, compressionLevel); + } + + /** + * Creates a new QatLz4104Codec instance with the default compression level. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatLz4104Codec(int compressionLevel, Supplier supplier) { + super(Mode.QAT_LZ4, compressionLevel, supplier); + } + + /** + * Creates a new QatLz4104Codec instance. + * + * @param defaultCodecSupplier default opensearch codec supplier + * @param compressionLevel The compression level. + */ + public QatLz4104Codec(Supplier defaultCodecSupplier, int compressionLevel) { + super(Mode.QAT_LZ4, defaultCodecSupplier, compressionLevel); + } + + /** + * Creates a new QatLz4104Codec instance. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + * @param defaultCodecSupplier default opensearch codec supplier + */ + public QatLz4104Codec(int compressionLevel, Supplier supplier, Supplier defaultCodecSupplier) { + super(Mode.QAT_LZ4, compressionLevel, supplier, defaultCodecSupplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + if (!QatZipperFactory.isQatAvailable()) { + return Set.of(); + } + return Mode.QAT_LZ4.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatZstd104Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatZstd104Codec.java new file mode 100644 index 0000000..58bb6a2 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatZstd104Codec.java @@ -0,0 +1,93 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.Codec; +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * QatZstd104Codec provides a ZSTD compressor using the qat-java library. + */ +public class QatZstd104Codec extends Lucene104QatCodec implements CodecSettings, CodecAliases { + + /** Creates a new QatZstd104Codec instance with the default compression level. */ + public QatZstd104Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new QatZstd104Codec instance. + * + * @param compressionLevel The compression level. + */ + public QatZstd104Codec(int compressionLevel) { + super(Mode.QAT_ZSTD, compressionLevel); + } + + /** + * Creates a new QatZstd104Codec instance with the default compression level. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatZstd104Codec(int compressionLevel, Supplier supplier) { + super(Mode.QAT_ZSTD, compressionLevel, supplier); + } + + /** + * Creates a new QatZstd104Codec instance. + * + * @param defaultCodecSupplier default opensearch codec supplier + * @param compressionLevel The compression level. + */ + public QatZstd104Codec(Supplier defaultCodecSupplier, int compressionLevel) { + super(Mode.QAT_ZSTD, defaultCodecSupplier, compressionLevel); + } + + /** + * Creates a new QatZstd104Codec instance. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + * @param defaultCodecSupplier default opensearch codec supplier + */ + public QatZstd104Codec(int compressionLevel, Supplier supplier, Supplier defaultCodecSupplier) { + super(Mode.QAT_ZSTD, compressionLevel, supplier, defaultCodecSupplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + if (!QatZipperFactory.isQatAvailable()) { + return Set.of(); + } + return Mode.QAT_ZSTD.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd104Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd104Codec.java new file mode 100644 index 0000000..d9612f6 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd104Codec.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.Codec; +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; +import java.util.function.Supplier; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * ZstdCodec provides ZSTD compressor using the zstd-jni library. + */ +public class Zstd104Codec extends Lucene104CustomCodec implements CodecSettings, CodecAliases { + + /** Creates a new ZstdCodec instance with the default compression level. */ + public Zstd104Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new ZstdCodec instance. + * + * @param compressionLevel The compression level. + */ + public Zstd104Codec(int compressionLevel) { + super(Mode.ZSTD, compressionLevel); + } + + /** + * Creates a new ZstdCodec instance. + * + * @param compressionLevel The compression level. + * @param defaultCodecSupplier default opensearch codec supplier + */ + public Zstd104Codec(int compressionLevel, Supplier defaultCodecSupplier) { + super(Mode.ZSTD, compressionLevel, defaultCodecSupplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + return Mode.ZSTD.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict104Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict104Codec.java new file mode 100644 index 0000000..7c449d5 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict104Codec.java @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.Codec; +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; +import java.util.function.Supplier; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** ZstdNoDictCodec provides ZSTD compressor without a dictionary support. */ +public class ZstdNoDict104Codec extends Lucene104CustomCodec implements CodecSettings, CodecAliases { + + /** Creates a new ZstdNoDictCodec instance with the default compression level. */ + public ZstdNoDict104Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new ZstdNoDictCodec instance. + * + * @param compressionLevel The compression level. + */ + public ZstdNoDict104Codec(int compressionLevel) { + super(Mode.ZSTD_NO_DICT, compressionLevel); + } + + /** + * Creates a new ZstdNoDictCodec instance. + * + * @param compressionLevel The compression level. + * @param defaultCodecSupplier default opensearch codec supplier + */ + public ZstdNoDict104Codec(int compressionLevel, Supplier defaultCodecSupplier) { + super(Mode.ZSTD_NO_DICT, compressionLevel, defaultCodecSupplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + return Mode.ZSTD_NO_DICT.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene103CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103CustomCodec.java similarity index 96% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene103CustomCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103CustomCodec.java index 5c63a92..09291cf 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene103CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103CustomCodec.java @@ -6,12 +6,12 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene103; +import org.apache.lucene.backward_codecs.lucene103.Lucene103Codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene103.Lucene103Codec; import java.util.Set; import java.util.function.Supplier; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene103CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103CustomStoredFieldsFormat.java similarity index 95% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene103CustomStoredFieldsFormat.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103CustomStoredFieldsFormat.java index 5f1accc..1c49fc4 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene103CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103CustomStoredFieldsFormat.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene103; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.StoredFieldsReader; @@ -17,6 +17,8 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; import java.io.IOException; import java.util.Objects; @@ -62,8 +64,10 @@ public Lucene103CustomStoredFieldsFormat(Lucene103CustomCodec.Mode mode) { public Lucene103CustomStoredFieldsFormat(Lucene103CustomCodec.Mode mode, int compressionLevel) { this.mode = Objects.requireNonNull(mode); this.compressionLevel = compressionLevel; - zstdCompressionMode = new ZstdCompressionMode(compressionLevel); - zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); + zstdCompressionMode = new ZstdCompressionMode(compressionLevel) { + }; + zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel) { + }; } /** diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene103QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103QatCodec.java similarity index 97% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene103QatCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103QatCodec.java index 8ea6ebe..c6db940 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene103QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103QatCodec.java @@ -6,12 +6,12 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene103; +import org.apache.lucene.backward_codecs.lucene103.Lucene103Codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene103.Lucene103Codec; import java.util.Set; import java.util.function.Supplier; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene103QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103QatStoredFieldsFormat.java similarity index 97% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene103QatStoredFieldsFormat.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103QatStoredFieldsFormat.java index 08080ec..3d7f7cc 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene103QatStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103QatStoredFieldsFormat.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene103; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.StoredFieldsReader; @@ -17,6 +17,7 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.opensearch.index.codec.customcodecs.QatCompressionMode; import java.io.IOException; import java.util.Objects; @@ -91,7 +92,8 @@ public Lucene103QatStoredFieldsFormat(Lucene103QatCodec.Mode mode, Supplier supplier) { this.mode = Objects.requireNonNull(mode); - qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier); + qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier) { + }; } /** diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate103Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/QatDeflate103Codec.java similarity index 95% rename from src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate103Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/QatDeflate103Codec.java index b07ae75..86b09c0 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate103Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/QatDeflate103Codec.java @@ -6,12 +6,13 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene103; import org.apache.lucene.codecs.Codec; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.index.engine.EngineConfig; import java.util.Set; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4103Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/QatLz4103Codec.java similarity index 95% rename from src/main/java/org/opensearch/index/codec/customcodecs/QatLz4103Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/QatLz4103Codec.java index c90f739..4a2ccf2 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4103Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/QatLz4103Codec.java @@ -6,12 +6,13 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene103; import org.apache.lucene.codecs.Codec; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.index.engine.EngineConfig; import java.util.Set; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatZstd103Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/QatZstd103Codec.java similarity index 95% rename from src/main/java/org/opensearch/index/codec/customcodecs/QatZstd103Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/QatZstd103Codec.java index 086fb4b..1029bef 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatZstd103Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/QatZstd103Codec.java @@ -6,12 +6,13 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene103; import org.apache.lucene.codecs.Codec; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.index.engine.EngineConfig; import java.util.Set; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd103Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Zstd103Codec.java similarity index 96% rename from src/main/java/org/opensearch/index/codec/customcodecs/Zstd103Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Zstd103Codec.java index 0d2440d..09e39d0 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd103Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Zstd103Codec.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene103; import org.apache.lucene.codecs.Codec; import org.opensearch.common.settings.Setting; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict103Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/ZstdNoDict103Codec.java similarity index 96% rename from src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict103Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/ZstdNoDict103Codec.java index 2b6c371..26f6596 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict103Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/ZstdNoDict103Codec.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene103; import org.apache.lucene.codecs.Codec; import org.opensearch.common.settings.Setting; diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index b59f998..f04b149 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -14,8 +14,13 @@ org.opensearch.index.codec.customcodecs.backward_codecs.lucene101.ZstdNoDict101C org.opensearch.index.codec.customcodecs.backward_codecs.lucene101.QatDeflate101Codec org.opensearch.index.codec.customcodecs.backward_codecs.lucene101.QatLz4101Codec org.opensearch.index.codec.customcodecs.backward_codecs.lucene101.QatZstd101Codec -org.opensearch.index.codec.customcodecs.Zstd103Codec -org.opensearch.index.codec.customcodecs.ZstdNoDict103Codec -org.opensearch.index.codec.customcodecs.QatDeflate103Codec -org.opensearch.index.codec.customcodecs.QatLz4103Codec -org.opensearch.index.codec.customcodecs.QatZstd103Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene103.Zstd103Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene103.ZstdNoDict103Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene103.QatDeflate103Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene103.QatLz4103Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene103.QatZstd103Codec +org.opensearch.index.codec.customcodecs.Zstd104Codec +org.opensearch.index.codec.customcodecs.ZstdNoDict104Codec +org.opensearch.index.codec.customcodecs.QatDeflate104Codec +org.opensearch.index.codec.customcodecs.QatLz4104Codec +org.opensearch.index.codec.customcodecs.QatZstd104Codec diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java index f7ede77..9cb375b 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene103.Lucene103Codec; +import org.apache.lucene.codecs.lucene104.Lucene104Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; @@ -86,15 +86,15 @@ public void setup() { public void testZstd() throws Exception { Codec codec = createCodecService(false).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene103CustomCodec.Mode.ZSTD, codec); - Lucene103CustomStoredFieldsFormat storedFieldsFormat = (Lucene103CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104CustomCodec.Mode.ZSTD, codec); + Lucene104CustomStoredFieldsFormat storedFieldsFormat = (Lucene104CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdWithCompositeIndex() throws Exception { Codec codec = createCodecService(false, true).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene103CustomCodec.Mode.ZSTD, codec); - Lucene103CustomStoredFieldsFormat storedFieldsFormat = (Lucene103CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104CustomCodec.Mode.ZSTD, codec); + Lucene104CustomStoredFieldsFormat storedFieldsFormat = (Lucene104CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); // assert docValues to be of compositeCodec's docValuesFormat assert codec.docValuesFormat() instanceof Composite912DocValuesFormat; @@ -102,15 +102,15 @@ public void testZstdWithCompositeIndex() throws Exception { public void testZstdNoDict() throws Exception { Codec codec = createCodecService(false).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene103CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene103CustomStoredFieldsFormat storedFieldsFormat = (Lucene103CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene104CustomStoredFieldsFormat storedFieldsFormat = (Lucene104CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictWithCompositeIndex() throws Exception { Codec codec = createCodecService(false, true).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene103CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene103CustomStoredFieldsFormat storedFieldsFormat = (Lucene103CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene104CustomStoredFieldsFormat storedFieldsFormat = (Lucene104CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); // assert docValues to be of compositeCodec's docValuesFormat assert codec.docValuesFormat() instanceof Composite912DocValuesFormat; @@ -127,16 +127,16 @@ public void testZstdDeprecatedCodec() { public void testZstdWithCompressionLevel() throws Exception { int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "zstd").codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene103CustomCodec.Mode.ZSTD, codec); - Lucene103CustomStoredFieldsFormat storedFieldsFormat = (Lucene103CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104CustomCodec.Mode.ZSTD, codec); + Lucene104CustomStoredFieldsFormat storedFieldsFormat = (Lucene104CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictWithCompressionLevel() throws Exception { int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "zstd_no_dict").codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene103CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene103CustomStoredFieldsFormat storedFieldsFormat = (Lucene103CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene104CustomStoredFieldsFormat storedFieldsFormat = (Lucene104CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } @@ -177,25 +177,25 @@ public void testZstandardCompressionLevelSupport() throws Exception { public void testDefaultMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("default"); - assertStoredFieldsCompressionEquals(Lucene103Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene104Codec.Mode.BEST_SPEED, codec); } public void testBestCompressionMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene103Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene104Codec.Mode.BEST_COMPRESSION, codec); } public void testZstdMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene103CustomCodec.Mode.ZSTD, codec); - Lucene103CustomStoredFieldsFormat storedFieldsFormat = (Lucene103CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104CustomCodec.Mode.ZSTD, codec); + Lucene104CustomStoredFieldsFormat storedFieldsFormat = (Lucene104CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene103CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene103CustomStoredFieldsFormat storedFieldsFormat = (Lucene103CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene104CustomStoredFieldsFormat storedFieldsFormat = (Lucene104CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } @@ -205,14 +205,14 @@ public void testQatCodecsNotAvailable() throws IOException { assertThrows(IllegalArgumentException.class, () -> createCodecService(false).codec("qat_deflate")); assertThrows(IllegalArgumentException.class, () -> createCodecService(false).codec("qat_zstd")); - QatLz4103Codec qatLz4103Codec = new QatLz4103Codec(); - assertTrue(qatLz4103Codec.aliases().isEmpty()); + QatLz4104Codec qatLz4104Codec = new QatLz4104Codec(); + assertTrue(qatLz4104Codec.aliases().isEmpty()); - QatDeflate103Codec qatDeflate101Codec = new QatDeflate103Codec(); - assertTrue(qatDeflate101Codec.aliases().isEmpty()); + QatDeflate104Codec qatDeflate104Codec = new QatDeflate104Codec(); + assertTrue(qatDeflate104Codec.aliases().isEmpty()); - QatZstd103Codec qatZstd101Codec = new QatZstd103Codec(); - assertTrue(qatZstd101Codec.aliases().isEmpty()); + QatZstd104Codec qatZstd104Codec = new QatZstd104Codec(); + assertTrue(qatZstd104Codec.aliases().isEmpty()); } } @@ -228,18 +228,18 @@ public void testAdditionalCodecsQatUnavailable() throws IOException { } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene103Codec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene104Codec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene103Codec.Mode.valueOf(v)); + assertEquals(expected, Lucene104Codec.Mode.valueOf(v)); } - private void assertStoredFieldsCompressionEquals(Lucene103CustomCodec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene104CustomCodec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene103CustomStoredFieldsFormat.MODE_KEY); + String v = sr.getSegmentInfo().info.getAttribute(Lucene104CustomStoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene103CustomCodec.Mode.valueOf(v)); + assertEquals(expected, Lucene104CustomCodec.Mode.valueOf(v)); } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene104CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene104CustomStoredFieldsFormatTests.java new file mode 100644 index 0000000..b9b2c68 --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene104CustomStoredFieldsFormatTests.java @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.test.OpenSearchTestCase; + +public class Lucene104CustomStoredFieldsFormatTests extends OpenSearchTestCase { + + public void testDefaultLucene104CustomCodecMode() { + Lucene104CustomStoredFieldsFormat lucene104CustomStoredFieldsFormat = new Lucene104CustomStoredFieldsFormat(); + assertEquals(Lucene104CustomCodec.Mode.ZSTD, lucene104CustomStoredFieldsFormat.getMode()); + } + + public void testZstdNoDictLucene104CustomCodecMode() { + Lucene104CustomStoredFieldsFormat lucene104CustomStoredFieldsFormat = new Lucene104CustomStoredFieldsFormat( + Lucene104CustomCodec.Mode.ZSTD_NO_DICT + ); + assertEquals(Lucene104CustomCodec.Mode.ZSTD_NO_DICT, lucene104CustomStoredFieldsFormat.getMode()); + } + + public void testZstdModeWithCompressionLevel() { + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene104CustomStoredFieldsFormat lucene104CustomStoredFieldsFormat = new Lucene104CustomStoredFieldsFormat( + Lucene104CustomCodec.Mode.ZSTD, + randomCompressionLevel + ); + assertEquals(Lucene104CustomCodec.Mode.ZSTD, lucene104CustomStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene104CustomStoredFieldsFormat.getCompressionLevel()); + } + + public void testZstdNoDictLucene104CustomCodecModeWithCompressionLevel() { + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene104CustomStoredFieldsFormat lucene104CustomStoredFieldsFormat = new Lucene104CustomStoredFieldsFormat( + Lucene104CustomCodec.Mode.ZSTD_NO_DICT, + randomCompressionLevel + ); + assertEquals(Lucene104CustomCodec.Mode.ZSTD_NO_DICT, lucene104CustomStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene104CustomStoredFieldsFormat.getCompressionLevel()); + } + + public void testCompressionModes() { + Lucene104CustomStoredFieldsFormat lucene104CustomStoredFieldsFormat = new Lucene104CustomStoredFieldsFormat(); + assertTrue(lucene104CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdCompressionMode); + } + + public void testZstdNoDictCompressionModes() { + Lucene104CustomStoredFieldsFormat lucene104CustomStoredFieldsFormat = new Lucene104CustomStoredFieldsFormat( + Lucene104CustomCodec.Mode.ZSTD_NO_DICT + ); + assertTrue(lucene104CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdNoDictCompressionMode); + } + +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene104QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene104QatStoredFieldsFormatTests.java new file mode 100644 index 0000000..a8420b1 --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene104QatStoredFieldsFormatTests.java @@ -0,0 +1,90 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.test.OpenSearchTestCase; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assume.assumeThat; + +public class Lucene104QatStoredFieldsFormatTests extends OpenSearchTestCase { + + public void testLz4Lucene104QatCodecMode() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene104QatStoredFieldsFormat lucene104QatStoredFieldsFormat = new Lucene104QatStoredFieldsFormat(Lucene104QatCodec.Mode.QAT_LZ4); + assertEquals(Lucene104QatCodec.Mode.QAT_LZ4, lucene104QatStoredFieldsFormat.getMode()); + } + + public void testDeflateLucene104QatCodecMode() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene104QatStoredFieldsFormat lucene104QatStoredFieldsFormat = new Lucene104QatStoredFieldsFormat( + Lucene104QatCodec.Mode.QAT_DEFLATE + ); + assertEquals(Lucene104QatCodec.Mode.QAT_DEFLATE, lucene104QatStoredFieldsFormat.getMode()); + } + + public void testZstdLucene104QatCodecMode() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene104QatStoredFieldsFormat lucene104QatStoredFieldsFormat = new Lucene104QatStoredFieldsFormat(Lucene104QatCodec.Mode.QAT_ZSTD); + assertEquals(Lucene104QatCodec.Mode.QAT_ZSTD, lucene104QatStoredFieldsFormat.getMode()); + } + + public void testLz4Lucene104QatCodecModeWithCompressionLevel() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene104QatStoredFieldsFormat lucene104QatStoredFieldsFormat = new Lucene104QatStoredFieldsFormat( + Lucene104QatCodec.Mode.QAT_LZ4, + randomCompressionLevel + ); + assertEquals(Lucene104QatCodec.Mode.QAT_LZ4, lucene104QatStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene104QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); + } + + public void testDeflateLucene104QatCodecModeWithCompressionLevel() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene104QatStoredFieldsFormat lucene104QatStoredFieldsFormat = new Lucene104QatStoredFieldsFormat( + Lucene104QatCodec.Mode.QAT_DEFLATE, + randomCompressionLevel + ); + assertEquals(Lucene104QatCodec.Mode.QAT_DEFLATE, lucene104QatStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene104QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); + } + + public void testZstdLucene104QatCodecModeWithCompressionLevel() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene104QatStoredFieldsFormat lucene104QatStoredFieldsFormat = new Lucene104QatStoredFieldsFormat( + Lucene104QatCodec.Mode.QAT_ZSTD, + randomCompressionLevel + ); + assertEquals(Lucene104QatCodec.Mode.QAT_ZSTD, lucene104QatStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene104QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); + } + + public void testLz4CompressionModes() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene104QatStoredFieldsFormat lucene104QatStoredFieldsFormat = new Lucene104QatStoredFieldsFormat(Lucene104QatCodec.Mode.QAT_LZ4); + assertTrue(lucene104QatStoredFieldsFormat.getCompressionMode() instanceof QatCompressionMode); + } + + public void testDeflateCompressionModes() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene104QatStoredFieldsFormat lucene104QatStoredFieldsFormat = new Lucene104QatStoredFieldsFormat( + Lucene104QatCodec.Mode.QAT_DEFLATE + ); + assertTrue(lucene104QatStoredFieldsFormat.getCompressionMode() instanceof QatCompressionMode); + } + + public void testZstdCompressionModes() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene104QatStoredFieldsFormat lucene104QatStoredFieldsFormat = new Lucene104QatStoredFieldsFormat(Lucene104QatCodec.Mode.QAT_ZSTD); + assertTrue(lucene104QatStoredFieldsFormat.getCompressionMode() instanceof QatCompressionMode); + } +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java index c85a58c..c73823a 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java @@ -78,24 +78,24 @@ public void setup() { public void testQatLz4() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(false).codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene103QatCodec.Mode.QAT_LZ4, codec); - Lucene103QatStoredFieldsFormat storedFieldsFormat = (Lucene103QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104QatCodec.Mode.QAT_LZ4, codec); + Lucene104QatStoredFieldsFormat storedFieldsFormat = (Lucene104QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatDeflate() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(false).codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene103QatCodec.Mode.QAT_DEFLATE, codec); - Lucene103QatStoredFieldsFormat storedFieldsFormat = (Lucene103QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104QatCodec.Mode.QAT_DEFLATE, codec); + Lucene104QatStoredFieldsFormat storedFieldsFormat = (Lucene104QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatZstd() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(false).codec("qat_zstd"); - assertStoredFieldsCompressionEquals(Lucene103QatCodec.Mode.QAT_ZSTD, codec); - Lucene103QatStoredFieldsFormat storedFieldsFormat = (Lucene103QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104QatCodec.Mode.QAT_ZSTD, codec); + Lucene104QatStoredFieldsFormat storedFieldsFormat = (Lucene104QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } @@ -103,8 +103,8 @@ public void testQatLz4WithCompressionLevel() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "qat_lz4").codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene103QatCodec.Mode.QAT_LZ4, codec); - Lucene103QatStoredFieldsFormat storedFieldsFormat = (Lucene103QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104QatCodec.Mode.QAT_LZ4, codec); + Lucene104QatStoredFieldsFormat storedFieldsFormat = (Lucene104QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } @@ -112,8 +112,8 @@ public void testQatDeflateWithCompressionLevel() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "qat_deflate").codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene103QatCodec.Mode.QAT_DEFLATE, codec); - Lucene103QatStoredFieldsFormat storedFieldsFormat = (Lucene103QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104QatCodec.Mode.QAT_DEFLATE, codec); + Lucene104QatStoredFieldsFormat storedFieldsFormat = (Lucene104QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } @@ -121,8 +121,8 @@ public void testQatZstdWithCompressionLevel() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "qat_zstd").codec("qat_zstd"); - assertStoredFieldsCompressionEquals(Lucene103QatCodec.Mode.QAT_ZSTD, codec); - Lucene103QatStoredFieldsFormat storedFieldsFormat = (Lucene103QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104QatCodec.Mode.QAT_ZSTD, codec); + Lucene104QatStoredFieldsFormat storedFieldsFormat = (Lucene104QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } @@ -140,33 +140,33 @@ public void testQatCompressionLevelSupport() throws Exception { public void testQatLz4MapperServiceNull() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(true).codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene103QatCodec.Mode.QAT_LZ4, codec); - Lucene103QatStoredFieldsFormat storedFieldsFormat = (Lucene103QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104QatCodec.Mode.QAT_LZ4, codec); + Lucene104QatStoredFieldsFormat storedFieldsFormat = (Lucene104QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatDeflateMapperServiceNull() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(true).codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene103QatCodec.Mode.QAT_DEFLATE, codec); - Lucene103QatStoredFieldsFormat storedFieldsFormat = (Lucene103QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104QatCodec.Mode.QAT_DEFLATE, codec); + Lucene104QatStoredFieldsFormat storedFieldsFormat = (Lucene104QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatZstdMapperServiceNull() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(true).codec("qat_zstd"); - assertStoredFieldsCompressionEquals(Lucene103QatCodec.Mode.QAT_ZSTD, codec); - Lucene103QatStoredFieldsFormat storedFieldsFormat = (Lucene103QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene104QatCodec.Mode.QAT_ZSTD, codec); + Lucene104QatStoredFieldsFormat storedFieldsFormat = (Lucene104QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } - private void assertStoredFieldsCompressionEquals(Lucene103QatCodec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene104QatCodec.Mode expected, Codec actual) throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene103QatStoredFieldsFormat.MODE_KEY); + String v = sr.getSegmentInfo().info.getAttribute(Lucene104QatStoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene103QatCodec.Mode.valueOf(v)); + assertEquals(expected, Lucene104QatCodec.Mode.valueOf(v)); } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene103CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103CustomStoredFieldsFormatTests.java similarity index 92% rename from src/test/java/org/opensearch/index/codec/customcodecs/Lucene103CustomStoredFieldsFormatTests.java rename to src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103CustomStoredFieldsFormatTests.java index 5eb21f4..e524903 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene103CustomStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103CustomStoredFieldsFormatTests.java @@ -6,8 +6,10 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene103; +import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; import org.opensearch.test.OpenSearchTestCase; public class Lucene103CustomStoredFieldsFormatTests extends OpenSearchTestCase { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene103QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103QatStoredFieldsFormatTests.java similarity index 95% rename from src/test/java/org/opensearch/index/codec/customcodecs/Lucene103QatStoredFieldsFormatTests.java rename to src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103QatStoredFieldsFormatTests.java index dc6cc32..d36fece 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene103QatStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene103/Lucene103QatStoredFieldsFormatTests.java @@ -6,8 +6,10 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene103; +import org.opensearch.index.codec.customcodecs.QatCompressionMode; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.is;