From 3f4ab3dd78b86b00e4cfb9cec5598605ba59cf55 Mon Sep 17 00:00:00 2001 From: Raghav Aggarwal Date: Fri, 3 Jul 2026 22:12:52 +0530 Subject: [PATCH] HIVE-29695: ClassCastException in MapJoin when reading Parquet Strings as Date, Timestamp, or TimestampLocalTZ --- .../ql/exec/vector/VectorizedBatchUtil.java | 1 + .../hive/ql/io/parquet/ParquetTypeUtils.java | 55 ++++ .../ql/io/parquet/convert/ETypeConverter.java | 33 ++- .../serde/ArrayWritableObjectInspector.java | 3 + .../ParquetDataColumnReaderFactory.java | 217 ++++++++++++-- .../VectorizedPrimitiveColumnReader.java | 27 +- .../parquet/convert/TestETypeConverter.java | 30 ++ .../clientpositive/parquet_string_date.q | 36 +++ .../clientpositive/parquet_string_timestamp.q | 40 +++ .../parquet_string_timestamplocaltz.q | 40 +++ .../llap/parquet_string_date.q.out | 266 ++++++++++++++++++ .../llap/parquet_string_timestamp.q.out | 263 +++++++++++++++++ .../parquet_string_timestamplocaltz.q.out | 205 ++++++++++++++ 13 files changed, 1189 insertions(+), 27 deletions(-) create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ParquetTypeUtils.java create mode 100644 ql/src/test/queries/clientpositive/parquet_string_date.q create mode 100644 ql/src/test/queries/clientpositive/parquet_string_timestamp.q create mode 100644 ql/src/test/queries/clientpositive/parquet_string_timestamplocaltz.q create mode 100644 ql/src/test/results/clientpositive/llap/parquet_string_date.q.out create mode 100644 ql/src/test/results/clientpositive/llap/parquet_string_timestamp.q.out create mode 100644 ql/src/test/results/clientpositive/llap/parquet_string_timestamplocaltz.q.out diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java index d4d4af31a945..e8d06d38b682 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java @@ -146,6 +146,7 @@ public static ColumnVector createColumnVector(TypeInfo typeInfo, case DATE: return new DateColumnVector(VectorizedRowBatch.DEFAULT_SIZE); case TIMESTAMP: + case TIMESTAMPLOCALTZ: return new TimestampColumnVector(VectorizedRowBatch.DEFAULT_SIZE); case INTERVAL_DAY_TIME: return new IntervalDayTimeColumnVector(VectorizedRowBatch.DEFAULT_SIZE); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ParquetTypeUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ParquetTypeUtils.java new file mode 100644 index 000000000000..de0711df9139 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ParquetTypeUtils.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hive.ql.io.parquet; + +import org.apache.hadoop.hive.common.type.Date; +import org.apache.hadoop.hive.common.type.Timestamp; +import org.apache.hadoop.hive.common.type.TimestampTZ; +import org.apache.hadoop.hive.common.type.TimestampTZUtil; +import java.nio.charset.StandardCharsets; +import java.time.ZoneId; + +import java.util.function.Function; + +public class ParquetTypeUtils { + + private static T parseString(byte[] bytes, Function parser) { + if (bytes == null || bytes.length < 8) { + return null; + } + try { + String s = new String(bytes, StandardCharsets.UTF_8); + return parser.apply(s); + } catch (Exception e) { + return null; + } + } + + public static Date parseDate(byte[] bytes) { + return parseString(bytes, Date::valueOf); + } + + public static Timestamp parseTimestamp(byte[] bytes) { + return parseString(bytes, Timestamp::valueOf); + } + + public static TimestampTZ parseTimestampTZ(byte[] bytes, ZoneId defaultTimeZone) { + return parseString(bytes, s -> TimestampTZUtil.parse(s, defaultTimeZone)); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java index c1bbd35ddeaa..be66e0cd12bb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java @@ -24,8 +24,10 @@ import java.util.TimeZone; import com.google.common.base.MoreObjects; +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.Timestamp; +import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTime; @@ -40,6 +42,8 @@ import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; +import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable; +import org.apache.hadoop.hive.ql.io.parquet.ParquetTypeUtils; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; @@ -508,8 +512,7 @@ PrimitiveConverter getConverter(final PrimitiveType type, final int index, final // of code paths that do not provide the typeInfo in those cases we default to Text. This idiom is also // followed by for example the BigDecimal converter in which if there is no type information, // it defaults to the widest representation - if (hiveTypeInfo instanceof PrimitiveTypeInfo) { - PrimitiveTypeInfo t = (PrimitiveTypeInfo) hiveTypeInfo; + if (hiveTypeInfo instanceof PrimitiveTypeInfo t) { switch (t.getPrimitiveCategory()) { case CHAR: return new BinaryConverter(type, parent, index) { @@ -525,9 +528,33 @@ protected HiveVarcharWritable convert(Binary binary) { return new HiveVarcharWritable(binary.getBytes(), ((VarcharTypeInfo) hiveTypeInfo).getLength()); } }; + case DATE: + return new BinaryConverter(type, parent, index) { + @Override + protected DateWritableV2 convert(Binary binary) { + Date date = ParquetTypeUtils.parseDate(binary.getBytes()); + return date != null ? new DateWritableV2(date) : null; + } + }; + case TIMESTAMP: + return new BinaryConverter(type, parent, index) { + @Override + protected TimestampWritableV2 convert(Binary binary) { + Timestamp ts = ParquetTypeUtils.parseTimestamp(binary.getBytes()); + return ts != null ? new TimestampWritableV2(ts) : null; + } + }; + case TIMESTAMPLOCALTZ: + return new BinaryConverter(type, parent, index) { + @Override + protected TimestampLocalTZWritable convert(Binary binary) { + TimestampTZ tstz = ParquetTypeUtils.parseTimestampTZ(binary.getBytes(), ZoneId.systemDefault()); + return tstz != null ? new TimestampLocalTZWritable(tstz) : null; + } + }; } } - // STRING type + // Default to STRING type (Text) return new BinaryConverter(type, parent, index) { @Override protected Text convert(Binary binary) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java index 742721dc6780..232a0ab909a3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; @@ -137,6 +138,8 @@ private ObjectInspector getObjectInspector( return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector((CharTypeInfo) typeInfo); } else if (typeInfo.getTypeName().toLowerCase().startsWith(serdeConstants.VARCHAR_TYPE_NAME)) { return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector((VarcharTypeInfo) typeInfo); + } else if (typeInfo instanceof TimestampLocalTZTypeInfo) { + return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector((TimestampLocalTZTypeInfo) typeInfo); } else { throw new UnsupportedOperationException("Unknown field type: " + typeInfo); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java index 472deeca41f8..b988fc00932a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java @@ -18,13 +18,16 @@ package org.apache.hadoop.hive.ql.io.parquet.vector; +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.common.type.HiveBaseChar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.Timestamp; +import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; import org.apache.hadoop.hive.ql.io.parquet.convert.ETypeConverter; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTime; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils; +import org.apache.hadoop.hive.ql.io.parquet.ParquetTypeUtils; import org.apache.hadoop.hive.ql.io.parquet.timestamp.ParquetTimestampUtils; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; @@ -57,6 +60,7 @@ import java.util.Arrays; import java.util.Optional; import java.util.TimeZone; +import java.util.function.Function; import static com.google.common.base.MoreObjects.firstNonNull; @@ -1934,6 +1938,107 @@ private byte[] truncateIfNecesssary(byte[] bytes) { } } + /** + * Reads Parquet string bytes for Date column ("2024-07-09") and converts them to long (days since + * epoch). + * + *

It uses a cache to pre-calculate string-to-date conversions for dictionary encoded pages. + * + *

'parquet.dictionary.page.size' defaults to 1MB. If a column has too many unique values, + * Parquet automatically disables the dictionary (isDict = false) and falls back to the + * ValuesReader flow. Therefore, a 1MB dictionary holds at most ~100,000 date strings (10 bytes + * each). This guarantees our long[] cache will never exceed ~800 KB, and the boolean[] ~100 KB! + */ + public static class TypesFromStringToDatePageReader extends TypesFromStringPageReader { + private long[] dictDateCache; + private boolean[] dictDateCacheValid; + private Date parsedDate = null; + + public TypesFromStringToDatePageReader(ValuesReader realReader, int length) { + super(realReader, length); + } + + public TypesFromStringToDatePageReader(Dictionary dict, int length) { + super(dict, length); + int maxId = dict.getMaxId(); + // IDs are zero-indexed + dictDateCache = new long[maxId + 1]; + dictDateCacheValid = new boolean[maxId + 1]; + for (int i = 0; i <= maxId; i++) { + dictDateCache[i] = parseDateString(dict.decodeToBinary(i).getBytesUnsafe()); + dictDateCacheValid[i] = (parsedDate != null); + } + } + + @Override + public long readLong() { + long result = parseDateString(valuesReader.readBytes().getBytesUnsafe()); + isValid = (parsedDate != null); + return result; + } + + @Override + public long readLong(int id) { + isValid = dictDateCacheValid[id]; + return dictDateCache[id]; + } + + private long parseDateString(byte[] bytes) { + parsedDate = ParquetTypeUtils.parseDate(bytes); + return parsedDate != null ? parsedDate.toEpochDay() : 0; + } + } + + /** + * A cache to pre-calculate string-to-timestamp conversions for dictionary encoded pages. + * + *

'parquet.dictionary.page.size' defaults to 1MB. If a column has too many unique values, + * Parquet automatically disables the dictionary (isDict = false) and falls back to the + * ValuesReader flow. + */ + public static class TypesFromStringToTimestampPageReader extends TypesFromStringPageReader { + private Timestamp[] dictTimestampCache; + private boolean[] dictTimestampCacheValid; + private final Function parser; + + public TypesFromStringToTimestampPageReader( + ValuesReader realReader, int length, Function parser) { + super(realReader, length); + this.parser = parser; + } + + public TypesFromStringToTimestampPageReader( + Dictionary dict, int length, Function parser) { + super(dict, length); + this.parser = parser; + int maxId = dict.getMaxId(); + // IDs are zero-indexed + dictTimestampCache = new Timestamp[maxId + 1]; + dictTimestampCacheValid = new boolean[maxId + 1]; + for (int i = 0; i <= maxId; i++) { + dictTimestampCache[i] = parseTimestampString(dict.decodeToBinary(i).getBytesUnsafe()); + dictTimestampCacheValid[i] = (dictTimestampCache[i] != null); + } + } + + @Override + public Timestamp readTimestamp() { + Timestamp result = parseTimestampString(valuesReader.readBytes().getBytesUnsafe()); + isValid = (result != null); + return result; + } + + @Override + public Timestamp readTimestamp(int id) { + isValid = dictTimestampCacheValid[id]; + return dictTimestampCache[id]; + } + + private Timestamp parseTimestampString(byte[] bytes) { + return parser.apply(bytes); + } + } + private static ParquetDataColumnReader getDataColumnReaderByTypeHelper(boolean isDictionary, PrimitiveType parquetType, TypeInfo hiveType, @@ -2021,19 +2126,33 @@ private static ParquetDataColumnReader getDataColumnReaderByTypeHelper(boolean i } } + /** + * Finds the right reader for Parquet string and binary data (BINARY and FIXED_LEN_BYTE_ARRAY). + * + *

This method handles cases where the Parquet data type doesn't perfectly match the + * Hive data type (like reading a Parquet String as a Hive Date). This is incredibly important + * during Vectorized MapJoins, because if we don't convert the data into the exact format Hive + * expects, it can crash the query with a ClassCastException. + * + * @param isDict True if the Parquet page uses dictionary encoding + * @param parquetType The primitive type from the Parquet schema + * @param hiveType The expected Hive TypeInfo + * @param valuesReader The fallback values reader if dictionary is not used + * @param dictionary The dictionary reader + * @return A configured ParquetDataColumnReader + */ private static ParquetDataColumnReader getConvertorFromBinary(boolean isDict, PrimitiveType parquetType, TypeInfo hiveType, ValuesReader valuesReader, Dictionary dictionary) { - LogicalTypeAnnotation logicalType = parquetType.getLogicalTypeAnnotation(); - // max length for varchar and char cases int length = getVarcharLength(hiveType); - TypeInfo realHiveType = (hiveType instanceof ListTypeInfo) ? - ((ListTypeInfo) hiveType).getListElementTypeInfo() : - (hiveType instanceof MapTypeInfo) ? - ((MapTypeInfo) hiveType).getMapValueTypeInfo() : hiveType; + TypeInfo realHiveType = switch (hiveType) { + case ListTypeInfo listType -> listType.getListElementTypeInfo(); + case MapTypeInfo mapType -> mapType.getMapValueTypeInfo(); + case null, default -> hiveType; + }; String typeName = TypeInfoUtils.getBaseName(realHiveType.getTypeName()); @@ -2042,12 +2161,20 @@ private static ParquetDataColumnReader getConvertorFromBinary(boolean isDict, int hiveScale = (typeName.equalsIgnoreCase(serdeConstants.DECIMAL_TYPE_NAME)) ? ((DecimalTypeInfo) realHiveType).getScale() : 0; + LogicalTypeAnnotation logicalType = parquetType.getLogicalTypeAnnotation(); + if (logicalType == null) { - return isDict ? new DefaultParquetDataColumnReader(dictionary, length) : new - DefaultParquetDataColumnReader(valuesReader, length); + // Check if we need to convert the string into a specific Hive type (like DATE) + ParquetDataColumnReader reader = + getReaderForString(realHiveType, isDict, dictionary, valuesReader, length); + return reader != null + ? reader + : (isDict + ? new DefaultParquetDataColumnReader(dictionary, length) + : new DefaultParquetDataColumnReader(valuesReader, length)); } - Optional reader = parquetType.getLogicalTypeAnnotation() + Optional reader = logicalType .accept(new LogicalTypeAnnotationVisitor<>() { @Override public Optional visit( DecimalLogicalTypeAnnotation logicalTypeAnnotation) { @@ -2062,18 +2189,74 @@ private static ParquetDataColumnReader getConvertorFromBinary(boolean isDict, @Override public Optional visit( StringLogicalTypeAnnotation logicalTypeAnnotation) { - return isDict ? Optional - .of(new TypesFromStringPageReader(dictionary, length)) : Optional - .of(new TypesFromStringPageReader(valuesReader, length)); + // Only try to find a specialized reader (like for DATE) right when we actually need it. + // If we don't find one, we just fall back to reading it as a normal string. + ParquetDataColumnReader reader = getReaderForString( + realHiveType, isDict, dictionary, valuesReader, length); + return Optional.of(reader != null ? reader : + (isDict + ? new TypesFromStringPageReader(dictionary, length) + : new TypesFromStringPageReader(valuesReader, length))); } }); - if (reader.isPresent()) { - return reader.get(); - } + // If the visitor above didn't find a reader (e.g. for unsupported logical types), + // default to a standard binary reader. + return reader.orElseGet( + () -> + isDict + ? new DefaultParquetDataColumnReader(dictionary, length) + : new DefaultParquetDataColumnReader(valuesReader, length)); + } - return isDict ? new DefaultParquetDataColumnReader(dictionary, length) : new - DefaultParquetDataColumnReader(valuesReader, length); + /** + * Returns a specialized reader if we need to convert Parquet string bytes into a specific Hive + * data type (like DATE). + * + *

If the type doesn't need special string conversion, it simply returns null. This makes it + * easy to add support for other data types in the future if they also need to be converted from + * strings during vectorized reads. + */ + private static ParquetDataColumnReader getReaderForString( + TypeInfo hiveType, + boolean isDictionary, + Dictionary dictionary, + ValuesReader valuesReader, + int length) { + if (hiveType instanceof PrimitiveTypeInfo ptInfo) { + return switch (ptInfo.getPrimitiveCategory()) { + case DATE -> + isDictionary + ? new TypesFromStringToDatePageReader(dictionary, length) + : new TypesFromStringToDatePageReader(valuesReader, length); + case TIMESTAMP -> + isDictionary + ? new TypesFromStringToTimestampPageReader( + dictionary, length, ParquetTypeUtils::parseTimestamp) + : new TypesFromStringToTimestampPageReader( + valuesReader, length, ParquetTypeUtils::parseTimestamp); + case TIMESTAMPLOCALTZ -> { + final ZoneId defaultTimeZone = ZoneId.systemDefault(); + Function tzParser = + bytes -> { + TimestampTZ tsTz = ParquetTypeUtils.parseTimestampTZ(bytes, defaultTimeZone); + return tsTz != null + ? Timestamp.ofEpochSecond( + tsTz.getEpochSecond(), tsTz.getZonedDateTime().getNano(), defaultTimeZone) + : null; + }; + yield isDictionary + ? new TypesFromStringToTimestampPageReader(dictionary, length, tzParser) + : new TypesFromStringToTimestampPageReader(valuesReader, length, tzParser); + } + case STRING, VARCHAR, CHAR -> + isDictionary + ? new TypesFromStringPageReader(dictionary, length) + : new TypesFromStringPageReader(valuesReader, length); + default -> null; + }; + } + return null; } public static ParquetDataColumnReader getDataColumnReaderByTypeOnDictionary( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java index be2622153bbe..9eb77a452c2d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java @@ -13,6 +13,7 @@ */ package org.apache.hadoop.hive.ql.io.parquet.vector; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DateColumnVector; @@ -145,6 +146,7 @@ private void readBatchHelper( } break; case TIMESTAMP: + case TIMESTAMPLOCALTZ: readTimestamp(num, (TimestampColumnVector) column, rowId); break; case INTERVAL_DAY_TIME: @@ -441,18 +443,23 @@ private void readTimestamp(int total, TimestampColumnVector c, int rowId) throws switch (descriptor.getType()) { //INT64 is not yet supported case INT96: - c.set(rowId, dataColumn.readTimestamp().toSqlTimestamp()); - break; case INT64: - c.set(rowId, dataColumn.readTimestamp().toSqlTimestamp()); + case BINARY: + case FIXED_LEN_BYTE_ARRAY: + Timestamp t = dataColumn.readTimestamp(); + if (dataColumn.isValid()) { + c.set(rowId, t.toSqlTimestamp()); + c.isNull[rowId] = false; + c.isRepeating = + c.isRepeating && ((c.time[0] == c.time[rowId]) && (c.nanos[0] == c.nanos[rowId])); + } else { + setNullValue(c, rowId); + } break; default: throw new IOException( "Unsupported parquet logical type: " + type.getLogicalTypeAnnotation().toString() + " for timestamp"); } - c.isNull[rowId] = false; - c.isRepeating = - c.isRepeating && ((c.time[0] == c.time[rowId]) && (c.nanos[0] == c.nanos[rowId])); } else { setNullValue(c, rowId); } @@ -638,11 +645,17 @@ private void decodeDictionaryIds( } break; case TIMESTAMP: + case TIMESTAMPLOCALTZ: TimestampColumnVector tsc = (TimestampColumnVector) column; tsc.setUsingProlepticCalendar(true); for (int i = rowId; i < rowId + num; ++i) { if (!column.isNull[i]) { - tsc.set(i, dictionary.readTimestamp((int) dictionaryIds.vector[i]).toSqlTimestamp()); + Timestamp t = dictionary.readTimestamp((int) dictionaryIds.vector[i]); + if (dictionary.isValid()) { + tsc.set(i, t.toSqlTimestamp()); + } else { + setNullValue(column, i); + } } } break; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/convert/TestETypeConverter.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/convert/TestETypeConverter.java index 3173d2db9007..ff0b31a3982c 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/convert/TestETypeConverter.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/convert/TestETypeConverter.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getPrimitiveTypeInfo; import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.stringTypeInfo; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.nio.ByteBuffer; @@ -33,6 +34,7 @@ import org.apache.hadoop.hive.ql.io.parquet.convert.ETypeConverter.BinaryConverter; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTime; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.io.HiveCharWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; @@ -341,6 +343,34 @@ public void testGetTextConverterNoHiveTypeInfo() { assertEquals(value, textWritable.toString()); } + @Test + public void testGetTextConverterForDate() { + PrimitiveType primitiveType = + Types.optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("value"); + String value = "2026-01-01"; + DateWritableV2 dateWritable = + (DateWritableV2) + getWritableFromBinaryConverter( + TypeInfoFactory.dateTypeInfo, primitiveType, Binary.fromString(value)); + assertEquals(org.apache.hadoop.hive.common.type.Date.valueOf(value), dateWritable.get()); + } + + @Test + public void testGetTextConverterForDateInvalid() { + PrimitiveType primitiveType = + Types.optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("value"); + String value = "invalid_date"; + DateWritableV2 dateWritable = + (DateWritableV2) + getWritableFromBinaryConverter( + TypeInfoFactory.dateTypeInfo, primitiveType, Binary.fromString(value)); + assertNull(dateWritable); + } + @Test public void testGetIntConverterForTinyInt() throws Exception { PrimitiveType primitiveType = Types.optional(PrimitiveTypeName.INT32) diff --git a/ql/src/test/queries/clientpositive/parquet_string_date.q b/ql/src/test/queries/clientpositive/parquet_string_date.q new file mode 100644 index 000000000000..c7b05803d291 --- /dev/null +++ b/ql/src/test/queries/clientpositive/parquet_string_date.q @@ -0,0 +1,36 @@ +CREATE TABLE dummy_string_date ( + id INT, + col1 STRING +) STORED AS PARQUET; + +INSERT INTO dummy_string_date VALUES (1, '2026-01-01'); + +CREATE EXTERNAL TABLE test_parquet_date ( + id INT, + col1 DATE +) STORED AS PARQUET +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/dummy_string_date'; + +CREATE TABLE small_table ( + id INT, + date_col DATE +); +INSERT INTO small_table VALUES (1, '2026-01-01'); + +SET hive.auto.convert.join=true; +SET hive.vectorized.execution.enabled=false; + +SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM test_parquet_date a +JOIN small_table b ON a.id = b.id; + +SET hive.vectorized.execution.enabled=true; + +EXPLAIN VECTORIZATION DETAIL +SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM test_parquet_date a +JOIN small_table b ON a.id = b.id; + +SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM test_parquet_date a +JOIN small_table b ON a.id = b.id; diff --git a/ql/src/test/queries/clientpositive/parquet_string_timestamp.q b/ql/src/test/queries/clientpositive/parquet_string_timestamp.q new file mode 100644 index 000000000000..d2e1d6ff4d44 --- /dev/null +++ b/ql/src/test/queries/clientpositive/parquet_string_timestamp.q @@ -0,0 +1,40 @@ +set hive.vectorized.execution.enabled=true; + +CREATE TABLE dummy_string_timestamp ( + c1 string, + c2 string +) STORED AS PARQUET; + +INSERT INTO dummy_string_timestamp VALUES +('2023-01-01 01:02:03', '2023-01-01 01:02:03'), +('invalid_timestamp', '2023-01-02 01:02:03'), +(NULL, '2023-01-03 01:02:03'); + +CREATE EXTERNAL TABLE test_parquet_timestamp ( + c1 timestamp, + c2 timestamp +) STORED AS PARQUET +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/dummy_string_timestamp'; + +CREATE TABLE small_table_timestamp ( + c1 timestamp +); +INSERT INTO small_table_timestamp VALUES ('2023-01-01 01:02:03'), ('2023-01-02 01:02:03'); + +SET hive.auto.convert.join=true; +SET hive.vectorized.execution.enabled=false; + +SELECT /*+ MAPJOIN(small_table_timestamp) */ a.c1 +FROM test_parquet_timestamp a +JOIN small_table_timestamp b ON a.c2 = b.c1; + +SET hive.vectorized.execution.enabled=true; + +EXPLAIN VECTORIZATION DETAIL +SELECT /*+ MAPJOIN(small_table_timestamp) */ a.c1 +FROM test_parquet_timestamp a +JOIN small_table_timestamp b ON a.c2 = b.c1; + +SELECT /*+ MAPJOIN(small_table_timestamp) */ a.c1 +FROM test_parquet_timestamp a +JOIN small_table_timestamp b ON a.c2 = b.c1; diff --git a/ql/src/test/queries/clientpositive/parquet_string_timestamplocaltz.q b/ql/src/test/queries/clientpositive/parquet_string_timestamplocaltz.q new file mode 100644 index 000000000000..63be67f5d276 --- /dev/null +++ b/ql/src/test/queries/clientpositive/parquet_string_timestamplocaltz.q @@ -0,0 +1,40 @@ +set hive.vectorized.execution.enabled=true; + +CREATE TABLE dummy_string_timestamplocaltz ( + c1 string, + c2 string +) STORED AS PARQUET; + +INSERT INTO dummy_string_timestamplocaltz VALUES +('2023-01-01 01:02:03 America/Los_Angeles', '2023-01-01 01:02:03 America/Los_Angeles'), +('invalid_timestamp_tz', '2023-01-02 01:02:03 America/Los_Angeles'), +(NULL, '2023-01-03 01:02:03 America/Los_Angeles'); + +CREATE EXTERNAL TABLE test_parquet_timestamplocaltz ( + c1 timestamp with local time zone, + c2 timestamp with local time zone +) STORED AS PARQUET +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/dummy_string_timestamplocaltz'; + +CREATE TABLE small_table_timestamplocaltz ( + c1 timestamp with local time zone +); +INSERT INTO small_table_timestamplocaltz VALUES ('2023-01-01 01:02:03 America/Los_Angeles'), ('2023-01-02 01:02:03 America/Los_Angeles'); + +SET hive.auto.convert.join=true; +SET hive.vectorized.execution.enabled=false; + +SELECT /*+ MAPJOIN(small_table_timestamplocaltz) */ a.c1 +FROM test_parquet_timestamplocaltz a +JOIN small_table_timestamplocaltz b ON a.c2 = b.c1; + +SET hive.vectorized.execution.enabled=true; + +EXPLAIN VECTORIZATION DETAIL +SELECT /*+ MAPJOIN(small_table_timestamplocaltz) */ a.c1 +FROM test_parquet_timestamplocaltz a +JOIN small_table_timestamplocaltz b ON a.c2 = b.c1; + +SELECT /*+ MAPJOIN(small_table_timestamplocaltz) */ a.c1 +FROM test_parquet_timestamplocaltz a +JOIN small_table_timestamplocaltz b ON a.c2 = b.c1; diff --git a/ql/src/test/results/clientpositive/llap/parquet_string_date.q.out b/ql/src/test/results/clientpositive/llap/parquet_string_date.q.out new file mode 100644 index 000000000000..32d6520a01ae --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/parquet_string_date.q.out @@ -0,0 +1,266 @@ +PREHOOK: query: CREATE TABLE dummy_string_date ( + id INT, + col1 STRING +) STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dummy_string_date +POSTHOOK: query: CREATE TABLE dummy_string_date ( + id INT, + col1 STRING +) STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dummy_string_date +PREHOOK: query: INSERT INTO dummy_string_date VALUES (1, '2026-01-01') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dummy_string_date +POSTHOOK: query: INSERT INTO dummy_string_date VALUES (1, '2026-01-01') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dummy_string_date +POSTHOOK: Lineage: dummy_string_date.col1 SCRIPT [] +POSTHOOK: Lineage: dummy_string_date.id SCRIPT [] +PREHOOK: query: CREATE EXTERNAL TABLE test_parquet_date ( + id INT, + col1 DATE +) STORED AS PARQUET +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@test_parquet_date +POSTHOOK: query: CREATE EXTERNAL TABLE test_parquet_date ( + id INT, + col1 DATE +) STORED AS PARQUET +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_parquet_date +PREHOOK: query: CREATE TABLE small_table ( + id INT, + date_col DATE +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@small_table +POSTHOOK: query: CREATE TABLE small_table ( + id INT, + date_col DATE +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_table +PREHOOK: query: INSERT INTO small_table VALUES (1, '2026-01-01') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@small_table +POSTHOOK: query: INSERT INTO small_table VALUES (1, '2026-01-01') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@small_table +POSTHOOK: Lineage: small_table.date_col SCRIPT [] +POSTHOOK: Lineage: small_table.id SCRIPT [] +PREHOOK: query: SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM test_parquet_date a +JOIN small_table b ON a.id = b.id +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table +PREHOOK: Input: default@test_parquet_date +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM test_parquet_date a +JOIN small_table b ON a.id = b.id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table +POSTHOOK: Input: default@test_parquet_date +#### A masked pattern was here #### +2026-01-01 +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM test_parquet_date a +JOIN small_table b ON a.id = b.id +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table +PREHOOK: Input: default@test_parquet_date +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM test_parquet_date a +JOIN small_table b ON a.id = b.id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table +POSTHOOK: Input: default@test_parquet_date +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: id is not null (type: boolean) + Statistics: Num rows: 25 Data size: 1260 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:id:int, 1:col1:date, 2:ROW__ID:struct, 3:ROW__IS__DELETED:boolean] + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 0:int) + predicate: id is not null (type: boolean) + Statistics: Num rows: 20 Data size: 1008 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: int), col1 (type: date) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 20 Data size: 1008 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Map Join Vectorization: + bigTableKeyColumns: 0:int + bigTableRetainColumnNums: [1] + bigTableValueColumns: 1:date + className: VectorMapJoinInnerBigOnlyLongOperator + native: true + nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true + nonOuterSmallTableKeyMapping: [] + projectedOutput: 1:date + hashTableImplementationType: OPTIMIZED + outputColumnNames: _col1 + input vertices: + 1 Map 2 + Statistics: Num rows: 22 Data size: 1108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: date) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [1] + Statistics: Num rows: 22 Data size: 1108 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 22 Data size: 1108 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 2 + includeColumns: [0, 1] + dataColumns: id:int, col1:date + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Map 2 + Map Operator Tree: + TableScan + alias: b + filterExpr: id is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:id:int, 1:date_col:date, 2:ROW__ID:struct, 3:ROW__IS__DELETED:boolean] + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 0:int) + predicate: id is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0] + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 0:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 2 + includeColumns: [0] + dataColumns: id:int, date_col:date + partitionColumnCount: 0 + scratchColumnTypeNames: [] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM test_parquet_date a +JOIN small_table b ON a.id = b.id +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table +PREHOOK: Input: default@test_parquet_date +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM test_parquet_date a +JOIN small_table b ON a.id = b.id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table +POSTHOOK: Input: default@test_parquet_date +#### A masked pattern was here #### +2026-01-01 diff --git a/ql/src/test/results/clientpositive/llap/parquet_string_timestamp.q.out b/ql/src/test/results/clientpositive/llap/parquet_string_timestamp.q.out new file mode 100644 index 000000000000..bdbc4fec321e --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/parquet_string_timestamp.q.out @@ -0,0 +1,263 @@ +PREHOOK: query: CREATE TABLE dummy_string_timestamp ( + c1 string, + c2 string +) STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dummy_string_timestamp +POSTHOOK: query: CREATE TABLE dummy_string_timestamp ( + c1 string, + c2 string +) STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dummy_string_timestamp +PREHOOK: query: INSERT INTO dummy_string_timestamp VALUES +('2023-01-01 01:02:03', '2023-01-01 01:02:03'), +('invalid_timestamp', '2023-01-02 01:02:03'), +(NULL, '2023-01-03 01:02:03') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dummy_string_timestamp +POSTHOOK: query: INSERT INTO dummy_string_timestamp VALUES +('2023-01-01 01:02:03', '2023-01-01 01:02:03'), +('invalid_timestamp', '2023-01-02 01:02:03'), +(NULL, '2023-01-03 01:02:03') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dummy_string_timestamp +POSTHOOK: Lineage: dummy_string_timestamp.c1 SCRIPT [] +POSTHOOK: Lineage: dummy_string_timestamp.c2 SCRIPT [] +PREHOOK: query: CREATE EXTERNAL TABLE test_parquet_timestamp ( + c1 timestamp, + c2 timestamp +) STORED AS PARQUET +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@test_parquet_timestamp +POSTHOOK: query: CREATE EXTERNAL TABLE test_parquet_timestamp ( + c1 timestamp, + c2 timestamp +) STORED AS PARQUET +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_parquet_timestamp +PREHOOK: query: CREATE TABLE small_table_timestamp ( + c1 timestamp +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@small_table_timestamp +POSTHOOK: query: CREATE TABLE small_table_timestamp ( + c1 timestamp +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_table_timestamp +PREHOOK: query: INSERT INTO small_table_timestamp VALUES ('2023-01-01 01:02:03'), ('2023-01-02 01:02:03') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@small_table_timestamp +POSTHOOK: query: INSERT INTO small_table_timestamp VALUES ('2023-01-01 01:02:03'), ('2023-01-02 01:02:03') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@small_table_timestamp +POSTHOOK: Lineage: small_table_timestamp.c1 SCRIPT [] +PREHOOK: query: SELECT /*+ MAPJOIN(small_table_timestamp) */ a.c1 +FROM test_parquet_timestamp a +JOIN small_table_timestamp b ON a.c2 = b.c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table_timestamp +PREHOOK: Input: default@test_parquet_timestamp +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+ MAPJOIN(small_table_timestamp) */ a.c1 +FROM test_parquet_timestamp a +JOIN small_table_timestamp b ON a.c2 = b.c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table_timestamp +POSTHOOK: Input: default@test_parquet_timestamp +#### A masked pattern was here #### +2023-01-01 01:02:03 +NULL +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT /*+ MAPJOIN(small_table_timestamp) */ a.c1 +FROM test_parquet_timestamp a +JOIN small_table_timestamp b ON a.c2 = b.c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table_timestamp +PREHOOK: Input: default@test_parquet_timestamp +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT /*+ MAPJOIN(small_table_timestamp) */ a.c1 +FROM test_parquet_timestamp a +JOIN small_table_timestamp b ON a.c2 = b.c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table_timestamp +POSTHOOK: Input: default@test_parquet_timestamp +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: c2 is not null (type: boolean) + Statistics: Num rows: 30 Data size: 2080 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:c1:timestamp, 1:c2:timestamp, 2:ROW__ID:struct, 3:ROW__IS__DELETED:boolean] + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 1:timestamp) + predicate: c2 is not null (type: boolean) + Statistics: Num rows: 25 Data size: 1733 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c1 (type: timestamp), c2 (type: timestamp) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 25 Data size: 1733 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: timestamp) + 1 _col0 (type: timestamp) + Map Join Vectorization: + bigTableKeyColumns: 1:timestamp + bigTableRetainColumnNums: [0] + bigTableValueColumns: 0:timestamp + className: VectorMapJoinInnerBigOnlyMultiKeyOperator + native: true + nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true + nonOuterSmallTableKeyMapping: [] + projectedOutput: 0:timestamp + hashTableImplementationType: OPTIMIZED + outputColumnNames: _col0 + input vertices: + 1 Map 2 + Statistics: Num rows: 27 Data size: 1906 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 27 Data size: 1906 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 2 + includeColumns: [0, 1] + dataColumns: c1:timestamp, c2:timestamp + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Map 2 + Map Operator Tree: + TableScan + alias: b + filterExpr: c1 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:c1:timestamp, 1:ROW__ID:struct, 2:ROW__IS__DELETED:boolean] + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 0:timestamp) + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: timestamp) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0] + Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: timestamp) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: timestamp) + Reduce Sink Vectorization: + className: VectorReduceSinkMultiKeyOperator + keyColumns: 0:timestamp + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [0] + dataColumns: c1:timestamp + partitionColumnCount: 0 + scratchColumnTypeNames: [] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT /*+ MAPJOIN(small_table_timestamp) */ a.c1 +FROM test_parquet_timestamp a +JOIN small_table_timestamp b ON a.c2 = b.c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table_timestamp +PREHOOK: Input: default@test_parquet_timestamp +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+ MAPJOIN(small_table_timestamp) */ a.c1 +FROM test_parquet_timestamp a +JOIN small_table_timestamp b ON a.c2 = b.c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table_timestamp +POSTHOOK: Input: default@test_parquet_timestamp +#### A masked pattern was here #### +2023-01-01 01:02:03 +NULL diff --git a/ql/src/test/results/clientpositive/llap/parquet_string_timestamplocaltz.q.out b/ql/src/test/results/clientpositive/llap/parquet_string_timestamplocaltz.q.out new file mode 100644 index 000000000000..1231cef887b5 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/parquet_string_timestamplocaltz.q.out @@ -0,0 +1,205 @@ +PREHOOK: query: CREATE TABLE dummy_string_timestamplocaltz ( + c1 string, + c2 string +) STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dummy_string_timestamplocaltz +POSTHOOK: query: CREATE TABLE dummy_string_timestamplocaltz ( + c1 string, + c2 string +) STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dummy_string_timestamplocaltz +PREHOOK: query: INSERT INTO dummy_string_timestamplocaltz VALUES +('2023-01-01 01:02:03 America/Los_Angeles', '2023-01-01 01:02:03 America/Los_Angeles'), +('invalid_timestamp_tz', '2023-01-02 01:02:03 America/Los_Angeles'), +(NULL, '2023-01-03 01:02:03 America/Los_Angeles') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dummy_string_timestamplocaltz +POSTHOOK: query: INSERT INTO dummy_string_timestamplocaltz VALUES +('2023-01-01 01:02:03 America/Los_Angeles', '2023-01-01 01:02:03 America/Los_Angeles'), +('invalid_timestamp_tz', '2023-01-02 01:02:03 America/Los_Angeles'), +(NULL, '2023-01-03 01:02:03 America/Los_Angeles') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dummy_string_timestamplocaltz +POSTHOOK: Lineage: dummy_string_timestamplocaltz.c1 SCRIPT [] +POSTHOOK: Lineage: dummy_string_timestamplocaltz.c2 SCRIPT [] +PREHOOK: query: CREATE EXTERNAL TABLE test_parquet_timestamplocaltz ( + c1 timestamp with local time zone, + c2 timestamp with local time zone +) STORED AS PARQUET +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@test_parquet_timestamplocaltz +POSTHOOK: query: CREATE EXTERNAL TABLE test_parquet_timestamplocaltz ( + c1 timestamp with local time zone, + c2 timestamp with local time zone +) STORED AS PARQUET +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_parquet_timestamplocaltz +PREHOOK: query: CREATE TABLE small_table_timestamplocaltz ( + c1 timestamp with local time zone +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@small_table_timestamplocaltz +POSTHOOK: query: CREATE TABLE small_table_timestamplocaltz ( + c1 timestamp with local time zone +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_table_timestamplocaltz +PREHOOK: query: INSERT INTO small_table_timestamplocaltz VALUES ('2023-01-01 01:02:03 America/Los_Angeles'), ('2023-01-02 01:02:03 America/Los_Angeles') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@small_table_timestamplocaltz +POSTHOOK: query: INSERT INTO small_table_timestamplocaltz VALUES ('2023-01-01 01:02:03 America/Los_Angeles'), ('2023-01-02 01:02:03 America/Los_Angeles') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@small_table_timestamplocaltz +POSTHOOK: Lineage: small_table_timestamplocaltz.c1 SCRIPT [] +PREHOOK: query: SELECT /*+ MAPJOIN(small_table_timestamplocaltz) */ a.c1 +FROM test_parquet_timestamplocaltz a +JOIN small_table_timestamplocaltz b ON a.c2 = b.c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table_timestamplocaltz +PREHOOK: Input: default@test_parquet_timestamplocaltz +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+ MAPJOIN(small_table_timestamplocaltz) */ a.c1 +FROM test_parquet_timestamplocaltz a +JOIN small_table_timestamplocaltz b ON a.c2 = b.c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table_timestamplocaltz +POSTHOOK: Input: default@test_parquet_timestamplocaltz +#### A masked pattern was here #### +2023-01-01 01:02:03.0 US/Pacific +NULL +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT /*+ MAPJOIN(small_table_timestamplocaltz) */ a.c1 +FROM test_parquet_timestamplocaltz a +JOIN small_table_timestamplocaltz b ON a.c2 = b.c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table_timestamplocaltz +PREHOOK: Input: default@test_parquet_timestamplocaltz +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT /*+ MAPJOIN(small_table_timestamplocaltz) */ a.c1 +FROM test_parquet_timestamplocaltz a +JOIN small_table_timestamplocaltz b ON a.c2 = b.c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table_timestamplocaltz +POSTHOOK: Input: default@test_parquet_timestamplocaltz +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: c2 is not null (type: boolean) + Statistics: Num rows: 38 Data size: 2640 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: c2 is not null (type: boolean) + Statistics: Num rows: 32 Data size: 2223 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c1 (type: timestamp with local time zone), c2 (type: timestamp with local time zone) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 32 Data size: 2223 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: timestamp with local time zone) + 1 _col0 (type: timestamp with local time zone) + outputColumnNames: _col0 + input vertices: + 1 Map 2 + Statistics: Num rows: 35 Data size: 2445 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 35 Data size: 2445 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + notVectorizedReason: Predicate expression for FILTER operator: Vectorizing data type timestamp with local time zone not supported + vectorized: false + Map 2 + Map Operator Tree: + TableScan + alias: b + filterExpr: c1 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c1 (type: timestamp with local time zone) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: timestamp with local time zone) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: timestamp with local time zone) + Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + notVectorizedReason: Predicate expression for FILTER operator: Vectorizing data type timestamp with local time zone not supported + vectorized: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT /*+ MAPJOIN(small_table_timestamplocaltz) */ a.c1 +FROM test_parquet_timestamplocaltz a +JOIN small_table_timestamplocaltz b ON a.c2 = b.c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table_timestamplocaltz +PREHOOK: Input: default@test_parquet_timestamplocaltz +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+ MAPJOIN(small_table_timestamplocaltz) */ a.c1 +FROM test_parquet_timestamplocaltz a +JOIN small_table_timestamplocaltz b ON a.c2 = b.c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table_timestamplocaltz +POSTHOOK: Input: default@test_parquet_timestamplocaltz +#### A masked pattern was here #### +2023-01-01 01:02:03.0 US/Pacific +NULL