Skip to content

Commit 53d2d6a

Browse files
nsivabalannsivabalan
authored
[HUDI-2744] Fix parsing of metadadata table compaction timestamp when metrics are enabled (#3976)
1 parent 3c43197 commit 53d2d6a

5 files changed

Lines changed: 59 additions & 23 deletions

File tree

‎hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java‎

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ public void testOnlyValidPartitionsAdded(HoodieTableType tableType) throws Excep
223223
@ParameterizedTest
224224
@MethodSource("bootstrapAndTableOperationTestArgs")
225225
public void testTableOperations(HoodieTableType tableType, boolean enableFullScan) throws Exception {
226-
init(tableType, true, enableFullScan);
226+
init(tableType, true, enableFullScan, false);
227227
doWriteInsertAndUpsert(testTable);
228228

229229
// trigger an upsert
@@ -462,27 +462,43 @@ public void testSync(HoodieTableType tableType) throws Exception {
462462
validateMetadata(testTable, emptyList(), true);
463463
}
464464

465+
/**
466+
* Fetches next commit time in seconds from current one.
467+
*
468+
* @param curCommitTime current commit time.
469+
* @return the next valid commit time.
470+
*/
471+
private Long getNextCommitTime(long curCommitTime) {
472+
if ((curCommitTime + 1) % 1000000000000L >= 60) { // max seconds is 60 and hence
473+
return Long.parseLong(HoodieActiveTimeline.createNewInstantTime());
474+
} else {
475+
return curCommitTime + 1;
476+
}
477+
}
478+
465479
@ParameterizedTest
466480
@EnumSource(HoodieTableType.class)
467481
public void testMetadataBootstrapLargeCommitList(HoodieTableType tableType) throws Exception {
468-
init(tableType);
482+
init(tableType, true, true, true);
483+
long baseCommitTime = Long.parseLong(HoodieActiveTimeline.createNewInstantTime());
469484
for (int i = 1; i < 25; i += 7) {
470-
String commitTime1 = ((i > 9) ? ("00000") : ("000000")) + i;
471-
String commitTime2 = ((i > 9) ? ("00000") : ("000000")) + (i + 1);
472-
String commitTime3 = ((i > 9) ? ("00000") : ("000000")) + (i + 2);
473-
String commitTime4 = ((i > 9) ? ("00000") : ("000000")) + (i + 3);
474-
String commitTime5 = ((i > 9) ? ("00000") : ("000000")) + (i + 4);
475-
String commitTime6 = ((i > 9) ? ("00000") : ("000000")) + (i + 5);
476-
String commitTime7 = ((i > 9) ? ("00000") : ("000000")) + (i + 6);
477-
doWriteOperation(testTable, commitTime1, INSERT);
478-
doWriteOperation(testTable, commitTime2);
479-
doClean(testTable, commitTime3, Arrays.asList(commitTime1));
480-
doWriteOperation(testTable, commitTime4);
485+
long commitTime1 = getNextCommitTime(baseCommitTime);
486+
long commitTime2 = getNextCommitTime(commitTime1);
487+
long commitTime3 = getNextCommitTime(commitTime2);
488+
long commitTime4 = getNextCommitTime(commitTime3);
489+
long commitTime5 = getNextCommitTime(commitTime4);
490+
long commitTime6 = getNextCommitTime(commitTime5);
491+
long commitTime7 = getNextCommitTime(commitTime6);
492+
baseCommitTime = commitTime7;
493+
doWriteOperation(testTable, Long.toString(commitTime1), INSERT);
494+
doWriteOperation(testTable, Long.toString(commitTime2));
495+
doClean(testTable, Long.toString(commitTime3), Arrays.asList(Long.toString(commitTime1)));
496+
doWriteOperation(testTable, Long.toString(commitTime4));
481497
if (tableType == MERGE_ON_READ) {
482-
doCompaction(testTable, commitTime5);
498+
doCompaction(testTable, Long.toString(commitTime5));
483499
}
484-
doWriteOperation(testTable, commitTime6);
485-
doRollback(testTable, commitTime6, commitTime7);
500+
doWriteOperation(testTable, Long.toString(commitTime6));
501+
doRollback(testTable, Long.toString(commitTime6), Long.toString(commitTime7));
486502
}
487503
validateMetadata(testTable, emptyList(), true);
488504
}

‎hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,10 +75,10 @@ public void init(HoodieTableType tableType) throws IOException {
7575
}
7676

7777
public void init(HoodieTableType tableType, boolean enableMetadataTable) throws IOException {
78-
init(tableType, enableMetadataTable, true);
78+
init(tableType, enableMetadataTable, true, false);
7979
}
8080

81-
public void init(HoodieTableType tableType, boolean enableMetadataTable, boolean enableFullScan) throws IOException {
81+
public void init(HoodieTableType tableType, boolean enableMetadataTable, boolean enableFullScan, boolean enableMetrics) throws IOException {
8282
this.tableType = tableType;
8383
initPath();
8484
initSparkContexts("TestHoodieMetadata");
@@ -87,8 +87,7 @@ public void init(HoodieTableType tableType, boolean enableMetadataTable, boolean
8787
initMetaClient(tableType);
8888
initTestDataGenerator();
8989
metadataTableBasePath = HoodieTableMetadata.getMetadataTableBasePath(basePath);
90-
writeConfig = getWriteConfigBuilder(HoodieFailedWritesCleaningPolicy.EAGER, true, enableMetadataTable, false,
91-
enableFullScan).build();
90+
writeConfig = getWriteConfigBuilder(HoodieFailedWritesCleaningPolicy.EAGER, true, enableMetadataTable, enableMetrics, enableFullScan).build();
9291
initWriteConfigAndMetatableWriter(writeConfig, enableMetadataTable);
9392
}
9493

‎hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstantTimeGenerator.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
package org.apache.hudi.common.table.timeline;
2020

21+
import java.text.ParseException;
2122
import java.time.Instant;
2223
import java.time.LocalDateTime;
2324
import java.time.ZoneId;
@@ -33,6 +34,7 @@
3334
public class HoodieInstantTimeGenerator {
3435
// Format of the timestamp used for an Instant
3536
private static final String INSTANT_TIMESTAMP_FORMAT = "yyyyMMddHHmmss";
37+
private static final int INSTANT_TIMESTAMP_FORMAT_LENGTH = INSTANT_TIMESTAMP_FORMAT.length();
3638
// Formatter to generate Instant timestamps
3739
private static DateTimeFormatter INSTANT_TIME_FORMATTER = DateTimeFormatter.ofPattern(INSTANT_TIMESTAMP_FORMAT);
3840
// The last Instant timestamp generated
@@ -56,7 +58,7 @@ public static String createNewInstantTime(long milliseconds) {
5658
});
5759
}
5860

59-
public static Date parseInstantTime(String timestamp) {
61+
public static Date parseInstantTime(String timestamp) throws ParseException {
6062
try {
6163
LocalDateTime dt = LocalDateTime.parse(timestamp, INSTANT_TIME_FORMATTER);
6264
return Date.from(dt.atZone(ZoneId.systemDefault()).toInstant());
@@ -65,7 +67,11 @@ public static Date parseInstantTime(String timestamp) {
6567
if (timestamp.equals(ALL_ZERO_TIMESTAMP)) {
6668
return new Date(0);
6769
}
68-
70+
// compaction and cleaning in metadata has special format. handling it by trimming extra chars and treating it with secs granularity
71+
if (timestamp.length() > INSTANT_TIMESTAMP_FORMAT_LENGTH) {
72+
LocalDateTime dt = LocalDateTime.parse(timestamp.substring(0, INSTANT_TIMESTAMP_FORMAT_LENGTH), INSTANT_TIME_FORMATTER);
73+
return Date.from(dt.atZone(ZoneId.systemDefault()).toInstant());
74+
}
6975
throw e;
7076
}
7177
}

‎hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,9 @@ private List<Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>>> readFrom
184184
HoodieRecord<HoodieMetadataPayload> hoodieRecord = null;
185185
// Retrieve record from base file
186186
if (baseFileReader != null) {
187-
HoodieTimer readTimer = new HoodieTimer().startTimer();
187+
HoodieTimer readTimer = new HoodieTimer();
188188
for (String key : keys) {
189+
readTimer.startTimer();
189190
Option<GenericRecord> baseRecord = baseFileReader.getRecordByKey(key);
190191
if (baseRecord.isPresent()) {
191192
hoodieRecord = metadataTableConfig.populateMetaFields()

‎hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.junit.jupiter.api.Test;
3232

3333
import java.io.IOException;
34+
import java.text.ParseException;
3435
import java.util.ArrayList;
3536
import java.util.Collections;
3637
import java.util.Date;
@@ -472,6 +473,19 @@ public void testCreateNewInstantTime() throws Exception {
472473
}
473474
}
474475

476+
@Test
477+
public void testMetadataCompactionInstantDateParsing() throws ParseException {
478+
// default second granularity instant ID
479+
String secondGranularityInstant = "20210101120101";
480+
Date defaultSecsGranularityDate = HoodieActiveTimeline.parseInstantTime(secondGranularityInstant);
481+
// metadata table compaction/cleaning : ms granularity instant ID
482+
String compactionInstant = secondGranularityInstant + "001";
483+
Date msGranularityDate = HoodieActiveTimeline.parseInstantTime(compactionInstant);
484+
assertEquals(0, msGranularityDate.getTime() - defaultSecsGranularityDate.getTime(), "Expected the ms part to be 0");
485+
assertTrue(HoodieTimeline.compareTimestamps(secondGranularityInstant, HoodieTimeline.LESSER_THAN, compactionInstant));
486+
assertTrue(HoodieTimeline.compareTimestamps(compactionInstant, HoodieTimeline.GREATER_THAN, secondGranularityInstant));
487+
}
488+
475489
/**
476490
* Returns an exhaustive list of all possible HoodieInstant.
477491
* @return list of HoodieInstant

0 commit comments

Comments
 (0)