Skip to content

Commit d1bd02a

Browse files
committed
Review fixes
Signed-off-by: rayshrey <rayshrey@amazon.com>
1 parent 91a66af commit d1bd02a

7 files changed

Lines changed: 328 additions & 129 deletions

File tree

sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngine.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ public class LuceneIndexingExecutionEngine implements IndexingExecutionEngine<Lu
7777

7878
private final LuceneDataFormat dataFormat;
7979
private final MergeIndexWriter sharedWriter;
80+
private final MapperService mapperService;
8081
private final Store store;
8182
private final Path baseDirectory;
8283
private final Analyzer analyzer;
@@ -101,6 +102,7 @@ public LuceneIndexingExecutionEngine(
101102
throw new IllegalArgumentException("LuceneCommitter must not be null");
102103
}
103104
this.dataFormat = dataFormat;
105+
this.mapperService = mapperService;
104106
this.sharedWriter = luceneCommitter.getIndexWriter();
105107
this.store = store;
106108
this.baseDirectory = store.shardPath().resolve(LuceneDataFormat.LUCENE_FORMAT_NAME);
@@ -158,8 +160,10 @@ public FormatStore getStore(DataFormat dataFormat) {
158160
public Writer<LuceneDocumentInput> createWriter(WriterConfig config) {
159161
assert sharedWriter.isOpen() : "Cannot create writer — shared IndexWriter is closed";
160162
try {
163+
long mappingVersion = mapperService.getIndexSettings().getIndexMetadata().getMappingVersion();
161164
return new LuceneWriter(
162165
config.writerGeneration(),
166+
mappingVersion,
163167
dataFormat,
164168
baseDirectory,
165169
analyzer,

sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneWriter.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ public class LuceneWriter implements Writer<LuceneDocumentInput> {
7979
* Creates a new LuceneWriter for the given generation.
8080
*
8181
* @param writerGeneration the writer generation number
82+
* @param mappingVersion the initial mapping version
8283
* @param dataFormat the Lucene data format descriptor
8384
* @param baseDirectory the base directory under which to create the temp directory
8485
* @param analyzer the analyzer to use for tokenized fields, or null for default
@@ -88,13 +89,15 @@ public class LuceneWriter implements Writer<LuceneDocumentInput> {
8889
*/
8990
public LuceneWriter(
9091
long writerGeneration,
92+
long mappingVersion,
9193
LuceneDataFormat dataFormat,
9294
Path baseDirectory,
9395
Analyzer analyzer,
9496
Codec codec,
9597
Sort indexSort
9698
) throws IOException {
9799
this.writerGeneration = writerGeneration;
100+
this.mappingVersion = mappingVersion;
98101
this.dataFormat = dataFormat;
99102
this.docCount = 0;
100103

sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngineTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ public void testRefreshIncorporatesLuceneSegments() throws IOException {
168168
when(textField.name()).thenReturn("content");
169169

170170
long generation = 1L;
171-
try (LuceneWriter luceneWriter = new LuceneWriter(generation, luceneDataFormat, tempBase, null, Codec.getDefault(), null)) {
171+
try (LuceneWriter luceneWriter = new LuceneWriter(generation, 0L, luceneDataFormat, tempBase, null, Codec.getDefault(), null)) {
172172
for (int i = 0; i < numDocs; i++) {
173173
LuceneDocumentInput input = new LuceneDocumentInput();
174174
input.addField(textField, "doc_" + i);

sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterTests.java

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ private MappedFieldType mockKeywordField(String name) {
6464

6565
public void testAddDocAndFlushProducesSingleSegment() throws IOException {
6666
Path baseDir = createTempDir();
67-
try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
67+
try (LuceneWriter writer = new LuceneWriter(1L, 0L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
6868
int numDocs = randomIntBetween(5, 20);
6969
MappedFieldType textField = mockTextField("content");
7070
for (int i = 0; i < numDocs; i++) {
@@ -95,7 +95,7 @@ public void testRowIdMatchesLuceneDocId() throws IOException {
9595
Path baseDir = createTempDir();
9696
int numDocs = randomIntBetween(10, 50);
9797
MappedFieldType textField = mockTextField("content");
98-
try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
98+
try (LuceneWriter writer = new LuceneWriter(1L, 0L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
9999
for (int i = 0; i < numDocs; i++) {
100100
LuceneDocumentInput input = new LuceneDocumentInput();
101101
input.addField(textField, "doc " + i);
@@ -122,7 +122,7 @@ public void testRowIdMatchesLuceneDocId() throws IOException {
122122

123123
public void testFlushWithNoDocsReturnsEmpty() throws IOException {
124124
Path baseDir = createTempDir();
125-
try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
125+
try (LuceneWriter writer = new LuceneWriter(1L, 0L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
126126
FileInfos fileInfos = writer.flush();
127127
assertTrue(fileInfos.writerFilesMap().isEmpty());
128128
}
@@ -132,7 +132,7 @@ public void testWriterGenerationIsPreserved() throws IOException {
132132
Path baseDir = createTempDir();
133133
long gen = randomLongBetween(1, 100);
134134
MappedFieldType textField = mockTextField("content");
135-
try (LuceneWriter writer = new LuceneWriter(gen, dataFormat, baseDir, null, Codec.getDefault(), null)) {
135+
try (LuceneWriter writer = new LuceneWriter(gen, 0L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
136136
assertThat(writer.generation(), equalTo(gen));
137137

138138
LuceneDocumentInput input = new LuceneDocumentInput();
@@ -149,7 +149,7 @@ public void testWriterGenerationIsPreserved() throws IOException {
149149
public void testKeywordFieldsAreIndexed() throws IOException {
150150
Path baseDir = createTempDir();
151151
MappedFieldType keywordField = mockKeywordField("status");
152-
try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
152+
try (LuceneWriter writer = new LuceneWriter(1L, 0L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
153153
LuceneDocumentInput input = new LuceneDocumentInput();
154154
input.addField(keywordField, "active");
155155
input.setRowId(LuceneDocumentInput.ROW_ID_FIELD, 0);
@@ -171,7 +171,7 @@ public void testUnsupportedFieldTypeIsSilentlySkipped() throws IOException {
171171
when(numericField.typeName()).thenReturn("integer");
172172
when(numericField.name()).thenReturn("count");
173173

174-
try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
174+
try (LuceneWriter writer = new LuceneWriter(1L, 0L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
175175
LuceneDocumentInput input = new LuceneDocumentInput();
176176
// Should not throw — unsupported types are silently skipped (handled by other formats)
177177
input.addField(numericField, 42);
@@ -185,7 +185,7 @@ public void testMixedTextAndKeywordFields() throws IOException {
185185
MappedFieldType textField = mockTextField("title");
186186
MappedFieldType keywordField = mockKeywordField("category");
187187

188-
try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
188+
try (LuceneWriter writer = new LuceneWriter(1L, 0L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
189189
int numDocs = randomIntBetween(5, 15);
190190
for (int i = 0; i < numDocs; i++) {
191191
LuceneDocumentInput input = new LuceneDocumentInput();
@@ -212,7 +212,7 @@ public void testWriteAndFlushEndToEndWithTextAndKeyword() throws IOException {
212212
MappedFieldType keywordField = mockKeywordField("status");
213213
int numDocs = randomIntBetween(5, 20);
214214

215-
try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
215+
try (LuceneWriter writer = new LuceneWriter(1L, 0L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
216216
for (int i = 0; i < numDocs; i++) {
217217
LuceneDocumentInput input = new LuceneDocumentInput();
218218
input.addField(textField, "hello world " + i);
@@ -263,8 +263,8 @@ public void testMultipleWriterGenerationsProduceIsolatedSegments() throws IOExce
263263

264264
// Create both writers without closing them until after verification,
265265
// because close() deletes the temp directory.
266-
LuceneWriter writer1 = new LuceneWriter(gen1, dataFormat, baseDir, null, Codec.getDefault(), null);
267-
LuceneWriter writer2 = new LuceneWriter(gen2, dataFormat, baseDir, null, Codec.getDefault(), null);
266+
LuceneWriter writer1 = new LuceneWriter(gen1, 0L, dataFormat, baseDir, null, Codec.getDefault(), null);
267+
LuceneWriter writer2 = new LuceneWriter(gen2, 0L, dataFormat, baseDir, null, Codec.getDefault(), null);
268268
try {
269269
for (int i = 0; i < numDocs1; i++) {
270270
LuceneDocumentInput input = new LuceneDocumentInput();

0 commit comments

Comments
 (0)