From 804feb3b9a28066ecc15a514e316b3df9978cbc9 Mon Sep 17 00:00:00 2001 From: Divjot Arora Date: Tue, 21 Jul 2026 17:25:22 +0000 Subject: [PATCH 1/2] Allow TimestampType to annotate FLBA(12) --- .../parquet/schema/PrimitiveComparator.java | 41 +++++++++ .../parquet/schema/PrimitiveStringifier.java | 90 +++++++++++++++++-- .../apache/parquet/schema/PrimitiveType.java | 6 ++ .../java/org/apache/parquet/schema/Types.java | 3 + .../column/statistics/TestStatistics.java | 28 ++++++ .../columnindex/TestBinaryTruncator.java | 13 +++ .../columnindex/TestColumnIndexBuilder.java | 47 ++++++++++ .../schema/TestPrimitiveComparator.java | 37 ++++++++ .../schema/TestPrimitiveStringifier.java | 47 +++++++++- .../parquet/schema/TestTypeBuilders.java | 27 +++++- .../parquet/statistics/TestColumnIndexes.java | 6 ++ .../parquet/statistics/TestStatistics.java | 10 ++- 12 files changed, 342 insertions(+), 13 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java index 9d22d25312..d16e721df4 100644 --- a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java +++ b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java @@ -278,6 +278,47 @@ public String toString() { } }; + /** + * Comparator for timestamps encoded as 12-byte little-endian two's-complement values. + */ + static final PrimitiveComparator BINARY_AS_SIGNED_TIMESTAMP_COMPARATOR = new BinaryComparator() { + @Override + int compareBinary(Binary b1, Binary b2) { + if (b1.length() != 12 || b2.length() != 12) { + throw new IllegalArgumentException( + "Timestamp binary length must be 12 bytes, got " + b1.length() + " and " + b2.length()); + } + + ByteBuffer bb1 = b1.toByteBuffer(); + ByteBuffer bb2 = b2.toByteBuffer(); + // The buffers may be slices with a non-zero position (e.g. a ByteBuffer-backed Binary), so + // index relative to position() rather than absolute offset 0. Byte 11 (position + 11) is the + // most significant byte and carries the sign; byte 0 (position) is least significant. + int p1 = bb1.position(); + int p2 = bb2.position(); + + // If one value is negative and the other is positive, one is trivially greater. + boolean neg1 = bb1.get(p1 + 11) < 0; + boolean neg2 = bb2.get(p2 + 11) < 0; + if (neg1 != neg2) { + return neg1 ? -1 : 1; + } + + for (int i = 11; i >= 0; --i) { + int diff = toUnsigned(bb1.get(p1 + i)) - toUnsigned(bb2.get(p2 + i)); + if (diff != 0) { + return diff; + } + } + return 0; + } + + @Override + public String toString() { + return "BINARY_AS_SIGNED_TIMESTAMP_COMPARATOR"; + } + }; + /** * This comparator is for comparing two float16 values represented in 2 bytes binary. */ diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java index 8000781127..a3a8a19fe1 100644 --- a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java +++ b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java @@ -29,6 +29,7 @@ import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.time.DateTimeException; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; @@ -249,7 +250,7 @@ public String stringify(long value) { return toFormattedString(getInstant(value)); } - private String toFormattedString(Instant instant) { + final String toFormattedString(Instant instant) { return formatter.format(instant); } @@ -266,6 +267,41 @@ Instant getInstant(long value) { } } + /** + * Stringifier implementation for timestamps that handles both int64 and FLBA(12) carriers. + * Values outside of Instant's supported range render as raw integers rather than human-readable + * timestamps. + */ + private abstract static class TimestampStringifier extends DateStringifier { + private TimestampStringifier(String name, String format) { + super(name, format); + } + + @Override + public String stringify(Binary value) { + if (value == null) { + return BINARY_NULL; + } + byte[] littleEndian = value.getBytesUnsafe(); + byte[] bigEndian = new byte[littleEndian.length]; + for (int i = 0; i < littleEndian.length; ++i) { + bigEndian[i] = littleEndian[littleEndian.length - 1 - i]; + } + try { + BigInteger units = new BigInteger(bigEndian); + try { + return toFormattedString(getInstant(units)); + } catch (ArithmeticException | DateTimeException e) { + return units.toString(); + } + } catch (NumberFormatException e) { + return BINARY_INVALID; + } + } + + abstract Instant getInstant(BigInteger value); + } + static final PrimitiveStringifier DATE_STRINGIFIER = new DateStringifier("DATE_STRINGIFIER", "yyyy-MM-dd") { @Override Instant getInstant(int value) { @@ -274,56 +310,96 @@ Instant getInstant(int value) { ; }; + // Converts a count of time units since epoch, held as a BigInteger (the 96-bit FLBA(12) carrier), + // into an Instant. The 96-bit count does not fit in a long, but the whole-second count does, so + // divide first and narrow after. + private static Instant instantFromUnits(BigInteger units, TimeUnit unit) { + BigInteger[] secondsAndUnits = units.divideAndRemainder(BigInteger.valueOf(unit.convert(1, SECONDS))); + long seconds = secondsAndUnits[0].longValueExact(); + long nanos = secondsAndUnits[1].longValue() * unit.toNanos(1); + return Instant.ofEpochSecond(seconds, nanos); + } + static final PrimitiveStringifier TIMESTAMP_MILLIS_STRINGIFIER = - new DateStringifier("TIMESTAMP_MILLIS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSS") { + new TimestampStringifier("TIMESTAMP_MILLIS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSS") { @Override Instant getInstant(long value) { return Instant.ofEpochMilli(value); } + + @Override + Instant getInstant(BigInteger value) { + return instantFromUnits(value, MILLISECONDS); + } }; static final PrimitiveStringifier TIMESTAMP_MICROS_STRINGIFIER = - new DateStringifier("TIMESTAMP_MICROS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSS") { + new TimestampStringifier("TIMESTAMP_MICROS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSS") { @Override Instant getInstant(long value) { return Instant.ofEpochSecond( MICROSECONDS.toSeconds(value), MICROSECONDS.toNanos(value % SECONDS.toMicros(1))); } + + @Override + Instant getInstant(BigInteger value) { + return instantFromUnits(value, MICROSECONDS); + } }; static final PrimitiveStringifier TIMESTAMP_NANOS_STRINGIFIER = - new DateStringifier("TIMESTAMP_NANOS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS") { + new TimestampStringifier("TIMESTAMP_NANOS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS") { @Override Instant getInstant(long value) { return Instant.ofEpochSecond( NANOSECONDS.toSeconds(value), NANOSECONDS.toNanos(value % SECONDS.toNanos(1))); } + + @Override + Instant getInstant(BigInteger value) { + return instantFromUnits(value, NANOSECONDS); + } }; static final PrimitiveStringifier TIMESTAMP_MILLIS_UTC_STRINGIFIER = - new DateStringifier("TIMESTAMP_MILLIS_UTC_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSZ") { + new TimestampStringifier("TIMESTAMP_MILLIS_UTC_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSZ") { @Override Instant getInstant(long value) { return Instant.ofEpochMilli(value); } + + @Override + Instant getInstant(BigInteger value) { + return instantFromUnits(value, MILLISECONDS); + } }; static final PrimitiveStringifier TIMESTAMP_MICROS_UTC_STRINGIFIER = - new DateStringifier("TIMESTAMP_MICROS_UTC_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ") { + new TimestampStringifier("TIMESTAMP_MICROS_UTC_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ") { @Override Instant getInstant(long value) { return Instant.ofEpochSecond( MICROSECONDS.toSeconds(value), MICROSECONDS.toNanos(value % SECONDS.toMicros(1))); } + + @Override + Instant getInstant(BigInteger value) { + return instantFromUnits(value, MICROSECONDS); + } }; static final PrimitiveStringifier TIMESTAMP_NANOS_UTC_STRINGIFIER = - new DateStringifier("TIMESTAMP_NANOS_UTC_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSZ") { + new TimestampStringifier("TIMESTAMP_NANOS_UTC_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSZ") { @Override Instant getInstant(long value) { return Instant.ofEpochSecond( NANOSECONDS.toSeconds(value), NANOSECONDS.toNanos(value % SECONDS.toNanos(1))); } + + @Override + Instant getInstant(BigInteger value) { + return instantFromUnits(value, NANOSECONDS); + } }; private abstract static class TimeStringifier extends PrimitiveStringifier { diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java index c3766fa155..abdb62d353 100644 --- a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java +++ b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java @@ -444,6 +444,12 @@ public Optional visit( : of(PrimitiveComparator.BINARY_AS_FLOAT16_COMPARATOR); } + @Override + public Optional visit( + LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampLogicalType) { + return of(PrimitiveComparator.BINARY_AS_SIGNED_TIMESTAMP_COMPARATOR); + } + @Override public Optional visit( LogicalTypeAnnotation.UnknownLogicalTypeAnnotation unknownLogicalTypeAnnotation) { diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java index 2f12991ab0..8f651c0b87 100644 --- a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java +++ b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java @@ -563,6 +563,9 @@ public Optional visit( @Override public Optional visit( LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampLogicalType) { + if (primitiveType == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) { + return checkFixedPrimitiveType(12, timestampLogicalType); + } return checkInt64PrimitiveType(timestampLogicalType); } diff --git a/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java b/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java index b04f849382..b6e6185ea5 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java @@ -22,6 +22,8 @@ import static java.lang.Float.floatToIntBits; import static org.apache.parquet.bytes.BytesUtils.intToBytes; import static org.apache.parquet.bytes.BytesUtils.longToBytes; +import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.NANOS; +import static org.apache.parquet.schema.LogicalTypeAnnotation.timestampType; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BOOLEAN; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE; @@ -940,6 +942,32 @@ public void testNoopStatistics() { .hasMessage("isSmallerThan is not supported by org.apache.parquet.column.statistics.NoopStatistics"); } + @Test + public void testFlba12TimestampStats() { + PrimitiveType type = Types.required(FIXED_LEN_BYTE_ARRAY) + .length(12) + .as(timestampType(true, NANOS)) + .named("ts"); + BinaryStatistics stats = (BinaryStatistics) Statistics.createStats(type); + + // -1 (all 0xFF) — just before epoch, negative value + byte[] negOne = new byte[12]; + for (int i = 0; i < 12; i++) negOne[i] = (byte) 0xFF; + // +1 (0x01 followed by zeros) — just after epoch, positive value + byte[] posOne = new byte[12]; + posOne[0] = 1; + // epoch (all 0x00) + byte[] epoch = new byte[12]; + + stats.updateStats(Binary.fromConstantByteArray(posOne)); + stats.updateStats(Binary.fromConstantByteArray(negOne)); + stats.updateStats(Binary.fromConstantByteArray(epoch)); + + // min must be the negative value (-1), max must be the positive value (+1) + assertThat(stats.genericGetMin()).isEqualTo(Binary.fromConstantByteArray(negOne)); + assertThat(stats.genericGetMax()).isEqualTo(Binary.fromConstantByteArray(posOne)); + } + @Test public void testBinaryIsSmallerThanNoOverflowForLargeValues() { BinaryStatistics stats = new BinaryStatistics(); diff --git a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestBinaryTruncator.java b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestBinaryTruncator.java index 6b7e79305a..a74c20b707 100644 --- a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestBinaryTruncator.java +++ b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestBinaryTruncator.java @@ -36,6 +36,8 @@ import java.util.Comparator; import java.util.Random; import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit; import org.apache.parquet.schema.PrimitiveStringifier; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Types; @@ -91,6 +93,17 @@ public void testContractNonStringTypes() { testTruncator(Types.required(INT96).named("test_int96"), false); } + @Test + public void testFlba12Timestamp() { + BinaryTruncator truncator = BinaryTruncator.getTruncator(Types.required(FIXED_LEN_BYTE_ARRAY) + .length(12) + .as(LogicalTypeAnnotation.timestampType(true, TimeUnit.NANOS)) + .named("test_fixed_timestamp")); + Binary value = Binary.fromConstantByteArray(new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4}); + assertThat(truncator.truncateMin(value, 4)).isSameAs(value); + assertThat(truncator.truncateMax(value, 4)).isSameAs(value); + } + @Test public void testStringTruncate() { BinaryTruncator truncator = diff --git a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java index 6f11f81fa1..0bea000185 100644 --- a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java +++ b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java @@ -37,12 +37,15 @@ import static org.apache.parquet.filter2.predicate.FilterApi.or; import static org.apache.parquet.filter2.predicate.FilterApi.userDefined; import static org.apache.parquet.filter2.predicate.LogicalInverter.invert; +import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.NANOS; +import static org.apache.parquet.schema.LogicalTypeAnnotation.timestampType; import static org.apache.parquet.schema.OriginalType.DECIMAL; import static org.apache.parquet.schema.OriginalType.UINT_8; import static org.apache.parquet.schema.OriginalType.UTF8; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BOOLEAN; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; @@ -2013,6 +2016,50 @@ public void testBuildPreservesValidColumnIndex() { assertCorrectNullCounts(ci6, 0, 1); } + @Test + public void testBuildFlba12Timestamp() { + // FLBA(12) TIMESTAMP column with negative (pre-1970) and positive (post-1970) values. + // The LE signed comparator must order negatives before zero before positives. + PrimitiveType type = Types.required(FIXED_LEN_BYTE_ARRAY) + .length(12) + .as(timestampType(true, NANOS)) + .named("ts"); + ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE); + assertThat(builder).isInstanceOf(BinaryColumnIndexBuilder.class); + + // -1 (all 0xFF) encodes a value just before epoch — smallest in the test + byte[] negOne = new byte[12]; + for (int i = 0; i < 12; i++) negOne[i] = (byte) 0xFF; + // 0 (all 0x00) — epoch + byte[] epoch = new byte[12]; + // +1 (0x01 followed by zeros) — just after epoch + byte[] posOne = new byte[12]; + posOne[0] = 1; + + StatsBuilder sb = new StatsBuilder(); + // Page 0: only negative values, min=-1 max=-1 + builder.add(sb.stats(type, Binary.fromConstantByteArray(negOne))); + // Page 1: straddles epoch, min=-1 max=+1 + builder.add(sb.stats(type, Binary.fromConstantByteArray(negOne), Binary.fromConstantByteArray(posOne))); + // Page 2: only positive values, min=epoch max=+1 + builder.add(sb.stats(type, Binary.fromConstantByteArray(epoch), Binary.fromConstantByteArray(posOne))); + + ColumnIndex columnIndex = builder.build(); + assertThat(columnIndex).isNotNull(); + // min of page 0 must equal -1 + assertCorrectValues( + columnIndex.getMinValues(), + Binary.fromConstantByteArray(negOne), + Binary.fromConstantByteArray(negOne), + Binary.fromConstantByteArray(epoch)); + // max of page 0 must equal -1 (all values are negative) + assertCorrectValues( + columnIndex.getMaxValues(), + Binary.fromConstantByteArray(negOne), + Binary.fromConstantByteArray(posOne), + Binary.fromConstantByteArray(posOne)); + } + @Test public void testBuildWithoutNullCountsIsNotRejected() { PrimitiveType type = Types.required(INT32).named("test_col"); diff --git a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java index 35de0c8cae..e79aa53908 100644 --- a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java +++ b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java @@ -21,6 +21,7 @@ import static org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_FLOAT16_COMPARATOR; import static org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_FLOAT16_IEEE_754_TOTAL_ORDER_COMPARATOR; import static org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_SIGNED_INTEGER_COMPARATOR; +import static org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_SIGNED_TIMESTAMP_COMPARATOR; import static org.apache.parquet.schema.PrimitiveComparator.BOOLEAN_COMPARATOR; import static org.apache.parquet.schema.PrimitiveComparator.DOUBLE_COMPARATOR; import static org.apache.parquet.schema.PrimitiveComparator.DOUBLE_IEEE_754_TOTAL_ORDER_COMPARATOR; @@ -433,6 +434,42 @@ private void assertOrdering(T v1, T v2, int expectedSignum, Comparator co } } + @Test + public void testBinaryAsSignedIntegerLE12Comparator() { + // 12-byte LE encodings: byte[0]=LSB, byte[11]=MSB/sign + // large negative: 0x80 00..00 (most negative 96-bit value) + byte[] largeNeg = new byte[12]; + largeNeg[11] = (byte) 0x80; + // -256: 0x00 FF FF..FF (little-endian) + byte[] negTwoFiftySix = new byte[12]; + for (int i = 1; i < 12; i++) negTwoFiftySix[i] = (byte) 0xFF; + // -1: all 0xFF + byte[] negOne = new byte[12]; + for (int i = 0; i < 12; i++) negOne[i] = (byte) 0xFF; + // 0: all 0x00 + byte[] zero = new byte[12]; + // +1: 0x01 followed by 0x00s + byte[] posOne = new byte[12]; + posOne[0] = 1; + // +256: byte[0]=0x00, byte[1]=0x01, rest 0x00 — order vs +1 is decided at byte 1 (high byte); + // an LSB-first comparator would wrongly rank +256 < +1 (seeing byte[0]=0 < 1 first) + byte[] posTwoFiftySix = new byte[12]; + posTwoFiftySix[1] = 1; + // large positive: 0x7F FF..FF (most positive 96-bit value) + byte[] largePos = new byte[12]; + for (int i = 0; i < 12; i++) largePos[i] = (byte) 0xFF; + largePos[11] = 0x7F; + testObjectComparator( + BINARY_AS_SIGNED_TIMESTAMP_COMPARATOR, + Binary.fromConstantByteArray(largeNeg), + Binary.fromConstantByteArray(negTwoFiftySix), + Binary.fromConstantByteArray(negOne), + Binary.fromConstantByteArray(zero), + Binary.fromConstantByteArray(posOne), + Binary.fromConstantByteArray(posTwoFiftySix), + Binary.fromConstantByteArray(largePos)); + } + private void checkThrowingUnsupportedException(PrimitiveComparator comparator, Class exclude) { if (Integer.TYPE != exclude) { assertThatThrownBy(() -> comparator.compare(0, 0)) diff --git a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java index f0da51fc0e..e427fbb42e 100644 --- a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java +++ b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java @@ -193,7 +193,22 @@ public void testTimestampMillisStringifier() { assertThat(stringifier.stringify(cal.getTimeInMillis())) .isEqualTo(withZoneString("1948-11-23T20:19:01.009", timezoneAmendment)); - checkThrowingUnsupportedException(stringifier, Long.TYPE); + // The FIXED_LEN_BYTE_ARRAY(12) carrier renders identically to the int64 carrier. Includes a + // pre-epoch (negative) value to exercise the signed little-endian decoding. + assertThat(stringifier.stringify(flba12(0L))) + .isEqualTo(withZoneString("1970-01-01T00:00:00.000", timezoneAmendment)); + cal.clear(); + cal.set(2017, Calendar.DECEMBER, 15, 10, 9, 54); + cal.set(Calendar.MILLISECOND, 120); + assertThat(stringifier.stringify(flba12(cal.getTimeInMillis()))) + .isEqualTo(withZoneString("2017-12-15T10:09:54.120", timezoneAmendment)); + cal.clear(); + cal.set(1948, Calendar.NOVEMBER, 23, 20, 19, 1); + cal.set(Calendar.MILLISECOND, 9); + assertThat(stringifier.stringify(flba12(cal.getTimeInMillis()))) + .isEqualTo(withZoneString("1948-11-23T20:19:01.009", timezoneAmendment)); + + checkThrowingUnsupportedException(stringifier, Long.TYPE, Binary.class); } } @@ -221,7 +236,14 @@ public void testTimestampMicrosStringifier() { assertThat(stringifier.stringify(micros)) .isEqualTo(withZoneString("1848-03-15T09:23:59.764999", timezoneAmendment)); - checkThrowingUnsupportedException(stringifier, Long.TYPE); + // The FIXED_LEN_BYTE_ARRAY(12) carrier renders identically to the int64 carrier, including + // the pre-epoch (negative) value above which exercises the signed little-endian decoding. + assertThat(stringifier.stringify(flba12(0L))) + .isEqualTo(withZoneString("1970-01-01T00:00:00.000000", timezoneAmendment)); + assertThat(stringifier.stringify(flba12(micros))) + .isEqualTo(withZoneString("1848-03-15T09:23:59.764999", timezoneAmendment)); + + checkThrowingUnsupportedException(stringifier, Long.TYPE, Binary.class); } } @@ -248,7 +270,14 @@ public void testTimestampNanosStringifier() { assertThat(stringifier.stringify(nanos)) .isEqualTo(withZoneString("1848-03-15T09:23:59.764999999", timezoneAmendment)); - checkThrowingUnsupportedException(stringifier, Long.TYPE); + // The FIXED_LEN_BYTE_ARRAY(12) carrier renders identically to the int64 carrier, including + // the pre-epoch (negative) value above which exercises the signed little-endian decoding. + assertThat(stringifier.stringify(flba12(0L))) + .isEqualTo(withZoneString("1970-01-01T00:00:00.000000000", timezoneAmendment)); + assertThat(stringifier.stringify(flba12(nanos))) + .isEqualTo(withZoneString("1848-03-15T09:23:59.764999999", timezoneAmendment)); + + checkThrowingUnsupportedException(stringifier, Long.TYPE, Binary.class); } } @@ -417,6 +446,18 @@ private Binary toBinary(int... bytes) { return Binary.fromConstantByteArray(array); } + // Encodes a unit count as the extended-precision timestamp carrier: a 12-byte signed + // two's-complement little-endian value (sign-extended, so negatives fill the high bytes with + // 0xFF). + private Binary flba12(long units) { + byte[] le = new byte[12]; + byte fill = (byte) (units < 0 ? 0xFF : 0x00); + for (int i = 0; i < 12; ++i) { + le[i] = i < 8 ? (byte) (units >>> (8 * i)) : fill; + } + return Binary.fromConstantByteArray(le); + } + private void checkThrowingUnsupportedException(PrimitiveStringifier stringifier, Class... excludes) { Set> set = new HashSet<>(List.of(excludes)); if (!set.contains(Integer.TYPE)) { diff --git a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java index d0d00898c0..bcada0f312 100644 --- a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java +++ b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java @@ -18,6 +18,7 @@ */ package org.apache.parquet.schema; +import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit; import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MICROS; import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MILLIS; import static org.apache.parquet.schema.LogicalTypeAnnotation.timestampType; @@ -553,12 +554,19 @@ public void testInt64AnnotationsRejectNonInt64() { .hasMessage( LogicalTypeAnnotation.fromOriginalType(logicalType, null) + " can only annotate INT64"); } + // TIMESTAMP allows FLBA(12); other lengths are still rejected. + // Non-timestamp types still only accept INT64 for any FLBA length. + boolean isTimestamp = logicalType == TIMESTAMP_MILLIS || logicalType == TIMESTAMP_MICROS; + String flbaErrMsg = isTimestamp + ? LogicalTypeAnnotation.fromOriginalType(logicalType, null) + + " can only annotate FIXED_LEN_BYTE_ARRAY(12)" + : LogicalTypeAnnotation.fromOriginalType(logicalType, null) + " can only annotate INT64"; assertThatThrownBy(() -> Types.required(FIXED_LEN_BYTE_ARRAY) .length(1) .as(logicalType) .named("col")) .isInstanceOf(IllegalStateException.class) - .hasMessage(LogicalTypeAnnotation.fromOriginalType(logicalType, null) + " can only annotate INT64"); + .hasMessage(flbaErrMsg); } } @@ -1443,6 +1451,23 @@ public void testTimestampLogicalTypeWithUTCParameter() { assertThat(nonUtcMicrosActual).isEqualTo(nonUtcMicrosExpected); } + @Test + public void testTimestampFlba12LogicalType() { + for (TimeUnit unit : TimeUnit.values()) { + // FLBA(12) with TIMESTAMP annotation is valid. + Types.required(FIXED_LEN_BYTE_ARRAY) + .length(12) + .as(timestampType(true, unit)) + .named("ts"); + // Other FLBA lengths must be rejected. + assertThatThrownBy(() -> Types.required(FIXED_LEN_BYTE_ARRAY) + .length(11) + .as(timestampType(true, unit)) + .named("ts")) + .isInstanceOf(IllegalStateException.class); + } + } + @Test public void testVariantLogicalType() { byte specVersion = 1; diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestColumnIndexes.java b/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestColumnIndexes.java index 6acf5f9dd7..16dc88b931 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestColumnIndexes.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestColumnIndexes.java @@ -120,6 +120,10 @@ public class TestColumnIndexes { .as(OriginalType.INTERVAL) .named("interval"), Types.optional(FIXED_LEN_BYTE_ARRAY).length(16).as(uuidType()).named("uuid"), + Types.optional(FIXED_LEN_BYTE_ARRAY) + .length(12) + .as(timestampType(true, TimeUnit.NANOS)) + .named("timestamp-nanos-flba"), Types.optional(BINARY).as(stringType()).named("always-null")); private static List> buildGenerators(int recordCount, Random random) { @@ -224,6 +228,8 @@ private static List> buildGenerators(int recordCount, Random random) new RandomValues.FixedGenerator(random.nextLong(), 12), random, recordCount, fieldIndex++), sortedOrRandom( new RandomValues.FixedGenerator(random.nextLong(), 16), random, recordCount, fieldIndex++), + sortedOrRandom( + new RandomValues.FixedGenerator(random.nextLong(), 12), random, recordCount, fieldIndex++), null); } diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestStatistics.java b/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestStatistics.java index 499c61c3f6..dfa4dd63ee 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestStatistics.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestStatistics.java @@ -381,7 +381,8 @@ public DataContext( new RandomValues.LongGenerator(random.nextLong()), new RandomValues.LongGenerator(random.nextLong()), new RandomValues.FixedGenerator(random.nextLong(), 12), - new RandomValues.FixedGenerator(random.nextLong(), 2)); + new RandomValues.FixedGenerator(random.nextLong(), 2), + new RandomValues.FixedGenerator(random.nextLong(), 12)); } private static MessageType buildSchema(long seed) { @@ -451,7 +452,12 @@ private static MessageType buildSchema(long seed) { Types.optional(FIXED_LEN_BYTE_ARRAY) .length(2) .named("float16") - .withLogicalTypeAnnotation(LogicalTypeAnnotation.float16Type())); + .withLogicalTypeAnnotation(LogicalTypeAnnotation.float16Type()), + Types.optional(FIXED_LEN_BYTE_ARRAY) + .length(12) + .named("timestamp-nanos-flba") + .withLogicalTypeAnnotation( + LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS))); } private static int calculatePrecision(int byteCnt) { From ca73573c526b31449c6ea2a57b46eb4fea1059fc Mon Sep 17 00:00:00 2001 From: Divjot Arora Date: Thu, 23 Jul 2026 12:44:28 +0000 Subject: [PATCH 2/2] address PR comments --- .../parquet/schema/PrimitiveComparator.java | 32 +++----- .../java/org/apache/parquet/schema/Types.java | 11 ++- .../schema/TestPrimitiveComparator.java | 5 ++ .../parquet/schema/TestTypeBuilders.java | 75 ++++++++++++------- .../converter/ParquetMetadataConverter.java | 9 ++- .../TestParquetMetadataConverter.java | 25 +++++++ 6 files changed, 102 insertions(+), 55 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java index d16e721df4..8ef548a990 100644 --- a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java +++ b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java @@ -20,6 +20,7 @@ import java.io.Serializable; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.Comparator; import org.apache.parquet.io.api.Binary; @@ -288,29 +289,16 @@ int compareBinary(Binary b1, Binary b2) { throw new IllegalArgumentException( "Timestamp binary length must be 12 bytes, got " + b1.length() + " and " + b2.length()); } - - ByteBuffer bb1 = b1.toByteBuffer(); - ByteBuffer bb2 = b2.toByteBuffer(); - // The buffers may be slices with a non-zero position (e.g. a ByteBuffer-backed Binary), so - // index relative to position() rather than absolute offset 0. Byte 11 (position + 11) is the - // most significant byte and carries the sign; byte 0 (position) is least significant. - int p1 = bb1.position(); - int p2 = bb2.position(); - - // If one value is negative and the other is positive, one is trivially greater. - boolean neg1 = bb1.get(p1 + 11) < 0; - boolean neg2 = bb2.get(p2 + 11) < 0; - if (neg1 != neg2) { - return neg1 ? -1 : 1; + ByteBuffer bb1 = b1.toByteBuffer().slice(); + bb1.order(ByteOrder.LITTLE_ENDIAN); + ByteBuffer bb2 = b2.toByteBuffer().slice(); + bb2.order(ByteOrder.LITTLE_ENDIAN); + // Signed comparison of the high 4 bytes followed by unsigned comparison of the low 8 bytes. + int hiResult = Integer.compare(bb1.getInt(8), bb2.getInt(8)); + if (hiResult != 0) { + return hiResult; } - - for (int i = 11; i >= 0; --i) { - int diff = toUnsigned(bb1.get(p1 + i)) - toUnsigned(bb2.get(p2 + i)); - if (diff != 0) { - return diff; - } - } - return 0; + return Long.compareUnsigned(bb1.getLong(0), bb2.getLong(0)); } @Override diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java index 8f651c0b87..19359cbbeb 100644 --- a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java +++ b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java @@ -563,10 +563,13 @@ public Optional visit( @Override public Optional visit( LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampLogicalType) { - if (primitiveType == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) { - return checkFixedPrimitiveType(12, timestampLogicalType); - } - return checkInt64PrimitiveType(timestampLogicalType); + Preconditions.checkState( + primitiveType == PrimitiveTypeName.INT64 + || (primitiveType == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY + && length == 12), + "%s can only annotate INT64 or FIXED_LEN_BYTE_ARRAY(12)", + timestampLogicalType); + return Optional.of(true); } @Override diff --git a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java index e79aa53908..c07fdb8109 100644 --- a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java +++ b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java @@ -455,6 +455,10 @@ public void testBinaryAsSignedIntegerLE12Comparator() { // an LSB-first comparator would wrongly rank +256 < +1 (seeing byte[0]=0 < 1 first) byte[] posTwoFiftySix = new byte[12]; posTwoFiftySix[1] = 1; + // +2^63: low 8 bytes have bit 63 set (LE: bytes 0..6=0x00, byte 7=0x80), high 4 bytes=0x00. + // Sits between small positives and largePos; fails if the lower 8 bytes are compared signed. + byte[] posTwo63 = new byte[12]; + posTwo63[7] = (byte) 0x80; // large positive: 0x7F FF..FF (most positive 96-bit value) byte[] largePos = new byte[12]; for (int i = 0; i < 12; i++) largePos[i] = (byte) 0xFF; @@ -467,6 +471,7 @@ public void testBinaryAsSignedIntegerLE12Comparator() { Binary.fromConstantByteArray(zero), Binary.fromConstantByteArray(posOne), Binary.fromConstantByteArray(posTwoFiftySix), + Binary.fromConstantByteArray(posTwo63), Binary.fromConstantByteArray(largePos)); } diff --git a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java index bcada0f312..c69886c347 100644 --- a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java +++ b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java @@ -18,7 +18,6 @@ */ package org.apache.parquet.schema; -import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit; import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MICROS; import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MILLIS; import static org.apache.parquet.schema.LogicalTypeAnnotation.timestampType; @@ -535,7 +534,8 @@ public void testInt32AnnotationsRejectNonInt32() { @Test public void testInt64Annotations() { - OriginalType[] types = new OriginalType[] {TIME_MICROS, TIMESTAMP_MILLIS, TIMESTAMP_MICROS, UINT_64, INT_64}; + // Test the non-timestamp annotations for INT64. Timestamps are tested separately below. + OriginalType[] types = new OriginalType[] {TIME_MICROS, UINT_64, INT_64}; for (OriginalType logicalType : types) { PrimitiveType expected = new PrimitiveType(REQUIRED, INT64, "col", logicalType); PrimitiveType date = Types.required(INT64).as(logicalType).named("col"); @@ -545,7 +545,8 @@ public void testInt64Annotations() { @Test public void testInt64AnnotationsRejectNonInt64() { - OriginalType[] types = new OriginalType[] {TIME_MICROS, TIMESTAMP_MILLIS, TIMESTAMP_MICROS, UINT_64, INT_64}; + // Test the non-timestamp annotations for INT64. Timestamps are tested separately below. + OriginalType[] types = new OriginalType[] {TIME_MICROS, UINT_64, INT_64}; for (final OriginalType logicalType : types) { PrimitiveTypeName[] nonInt64 = new PrimitiveTypeName[] {BOOLEAN, INT32, INT96, DOUBLE, FLOAT, BINARY}; for (final PrimitiveTypeName type : nonInt64) { @@ -554,19 +555,54 @@ public void testInt64AnnotationsRejectNonInt64() { .hasMessage( LogicalTypeAnnotation.fromOriginalType(logicalType, null) + " can only annotate INT64"); } - // TIMESTAMP allows FLBA(12); other lengths are still rejected. - // Non-timestamp types still only accept INT64 for any FLBA length. - boolean isTimestamp = logicalType == TIMESTAMP_MILLIS || logicalType == TIMESTAMP_MICROS; - String flbaErrMsg = isTimestamp - ? LogicalTypeAnnotation.fromOriginalType(logicalType, null) - + " can only annotate FIXED_LEN_BYTE_ARRAY(12)" - : LogicalTypeAnnotation.fromOriginalType(logicalType, null) + " can only annotate INT64"; assertThatThrownBy(() -> Types.required(FIXED_LEN_BYTE_ARRAY) .length(1) .as(logicalType) .named("col")) .isInstanceOf(IllegalStateException.class) - .hasMessage(flbaErrMsg); + .hasMessage(LogicalTypeAnnotation.fromOriginalType(logicalType, null) + " can only annotate INT64"); + } + } + + @Test + public void testTimestampAnnotations() { + // Test timestamp annotations for both INT64 and FLBA(12). + OriginalType[] types = new OriginalType[] {TIMESTAMP_MILLIS, TIMESTAMP_MICROS}; + for (OriginalType logicalType : types) { + PrimitiveType expectedInt64 = new PrimitiveType(REQUIRED, INT64, "col", logicalType); + PrimitiveType dateInt64 = Types.required(INT64).as(logicalType).named("col"); + assertThat(dateInt64).isEqualTo(expectedInt64); + + PrimitiveType expectedFlba12 = new PrimitiveType(REQUIRED, FIXED_LEN_BYTE_ARRAY, 12, "col", logicalType); + PrimitiveType dateFlba12 = Types.required(FIXED_LEN_BYTE_ARRAY) + .length(12) + .as(logicalType) + .named("col"); + assertThat(dateFlba12).isEqualTo(expectedFlba12); + } + } + + @Test + public void testTimestampAnnotationsRejectNonTimestamp() { + // Test timestamp annotations for both INT64 and FLBA(12). + OriginalType[] types = new OriginalType[] {TIMESTAMP_MILLIS, TIMESTAMP_MICROS}; + for (OriginalType logicalType : types) { + // Invalid primitive types are rejected. + PrimitiveTypeName[] nonTimestamp = new PrimitiveTypeName[] {BOOLEAN, INT32, INT96, DOUBLE, FLOAT, BINARY}; + for (PrimitiveTypeName type : nonTimestamp) { + assertThatThrownBy(() -> Types.required(type).as(logicalType).named("col")) + .isInstanceOf(IllegalStateException.class) + .hasMessage(LogicalTypeAnnotation.fromOriginalType(logicalType, null) + + " can only annotate INT64 or FIXED_LEN_BYTE_ARRAY(12)"); + } + // Invalid FLBA lengths are rejected. + assertThatThrownBy(() -> Types.required(FIXED_LEN_BYTE_ARRAY) + .length(11) + .as(logicalType) + .named("col")) + .isInstanceOf(IllegalStateException.class) + .hasMessage(LogicalTypeAnnotation.fromOriginalType(logicalType, null) + + " can only annotate INT64 or FIXED_LEN_BYTE_ARRAY(12)"); } } @@ -1451,23 +1487,6 @@ public void testTimestampLogicalTypeWithUTCParameter() { assertThat(nonUtcMicrosActual).isEqualTo(nonUtcMicrosExpected); } - @Test - public void testTimestampFlba12LogicalType() { - for (TimeUnit unit : TimeUnit.values()) { - // FLBA(12) with TIMESTAMP annotation is valid. - Types.required(FIXED_LEN_BYTE_ARRAY) - .length(12) - .as(timestampType(true, unit)) - .named("ts"); - // Other FLBA lengths must be rejected. - assertThatThrownBy(() -> Types.required(FIXED_LEN_BYTE_ARRAY) - .length(11) - .as(timestampType(true, unit)) - .named("ts")) - .isInstanceOf(IllegalStateException.class); - } - } - @Test public void testVariantLogicalType() { byte specVersion = 1; diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java index 8600b2ced4..9fbcf18840 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java @@ -318,7 +318,14 @@ public void visit(PrimitiveType primitiveType) { element.setRepetition_type(toParquetRepetition(primitiveType.getRepetition())); element.setType(getType(primitiveType.getPrimitiveTypeName())); if (primitiveType.getLogicalTypeAnnotation() != null) { - element.setConverted_type(convertToConvertedType(primitiveType.getLogicalTypeAnnotation())); + // The TimestampType logical type may have a converted type, but only for the INT64 + // physical type. + boolean suppressConvertedType = primitiveType.getLogicalTypeAnnotation() + instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation + && primitiveType.getPrimitiveTypeName() != PrimitiveTypeName.INT64; + if (!suppressConvertedType) { + element.setConverted_type(convertToConvertedType(primitiveType.getLogicalTypeAnnotation())); + } element.setLogicalType(convertToLogicalType(primitiveType.getLogicalTypeAnnotation())); } if (primitiveType.getDecimalMetadata() != null) { diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java index 1a523521bd..e23bbe0b00 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java @@ -449,6 +449,18 @@ public void testTimeLogicalTypes() { .required(PrimitiveTypeName.INT64) .as(timestampType(true, NANOS)) .named("aTimestampUtcNanos") + .required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) + .length(12) + .as(timestampType(true, MILLIS)) + .named("aTimestampFlbaMillis") + .required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) + .length(12) + .as(timestampType(true, MICROS)) + .named("aTimestampFlbaMicros") + .required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) + .length(12) + .as(timestampType(true, NANOS)) + .named("aTimestampFlbaNanos") .required(PrimitiveTypeName.INT32) .as(timeType(false, MILLIS)) .named("aTimeNonUtcMillis") @@ -469,6 +481,19 @@ public void testTimeLogicalTypes() { .named("aTimeUtcNanos") .named("Message"); List parquetSchema = parquetMetadataConverter.toParquetSchema(expected); + // FLBA(12) MILLIS/MICROS must not write a legacy converted_type (it is INT64-only). + SchemaElement flbaMillis = parquetSchema.stream() + .filter(e -> "aTimestampFlbaMillis".equals(e.getName())) + .findFirst() + .get(); + assertThat(flbaMillis.isSetConverted_type()).isFalse(); + assertThat(flbaMillis.isSetLogicalType()).isTrue(); + SchemaElement flbaMicros = parquetSchema.stream() + .filter(e -> "aTimestampFlbaMicros".equals(e.getName())) + .findFirst() + .get(); + assertThat(flbaMicros.isSetConverted_type()).isFalse(); + assertThat(flbaMicros.isSetLogicalType()).isTrue(); MessageType schema = parquetMetadataConverter.fromParquetSchema(parquetSchema, null); assertThat(schema).isEqualTo(expected); }