From 7429ee406c5f0ff68249c2cf525d607662f9c0ac Mon Sep 17 00:00:00 2001 From: Dmitriy Fingerman Date: Tue, 7 Jul 2026 13:49:37 -0400 Subject: [PATCH 1/2] HIVE-29699: Iceberg: Add support for the Unknown data type --- .../iceberg/hive/HiveSchemaConverter.java | 2 + .../apache/iceberg/hive/HiveSchemaUtil.java | 2 + .../iceberg/hive/TestHiveSchemaUtil.java | 30 +++++ .../apache/iceberg/mr/hive/Deserializer.java | 10 ++ .../IcebergObjectInspector.java | 3 + .../mr/hive/vector/HiveValueConverter.java | 2 + .../mr/hive/TestHiveIcebergUnknownType.java | 89 ++++++++++++++ .../TestIcebergObjectInspector.java | 12 +- .../src/test/queries/positive/unknown_type.q | 40 ++++++ .../test/results/positive/unknown_type.q.out | 115 ++++++++++++++++++ .../java/org/apache/hive/jdbc/JdbcColumn.java | 4 + .../apache/hadoop/hive/ql/parse/HiveParser.g | 9 +- .../hive/ql/exec/vector/VectorAssignRow.java | 6 + .../ql/exec/vector/VectorDeserializeRow.java | 3 + .../hive/ql/exec/vector/VectorExtractRow.java | 2 + .../ql/exec/vector/VectorSerializeRow.java | 11 ++ .../ql/exec/vector/VectorizationContext.java | 2 + .../ql/exec/vector/VectorizedBatchUtil.java | 2 + .../VectorExpressionWriterFactory.java | 25 ++++ .../parquet/convert/HiveSchemaConverter.java | 4 +- .../apache/hadoop/hive/ql/metadata/Table.java | 2 + .../calcite/translator/TypeConverter.java | 6 + .../hive/ql/parse/BaseSemanticAnalyzer.java | 1 + .../hive/ql/parse/SemanticAnalyzer.java | 7 +- .../hadoop/hive/ql/stats/TestStatsUtils.java | 1 + .../unknown_type_non_iceberg_table.q | 1 + .../unknown_type_non_iceberg_table.q.out | 5 + serde/if/serde.thrift | 1 + .../hadoop/hive/serde/serdeConstants.java | 1 + .../hadoop/hive/serde2/lazy/LazyFactory.java | 5 + .../serde2/lazybinary/LazyBinaryUtils.java | 6 + .../objectinspector/ObjectInspector.java | 2 +- .../ObjectInspectorFactory.java | 4 + .../UnknownObjectInspector.java | 46 +++++++ .../hadoop/hive/serde2/thrift/Type.java | 8 +- .../hive/serde2/typeinfo/TypeInfoFactory.java | 4 + .../hive/serde2/typeinfo/TypeInfoUtils.java | 44 +++++++ .../hive/serde2/typeinfo/UnknownTypeInfo.java | 69 +++++++++++ .../serde2/typeinfo/TestUnknownTypeInfo.java | 79 ++++++++++++ service-rpc/if/TCLIService.thrift | 7 +- .../rpc/thrift/TCLIServiceConstants.java | 2 + .../hive/service/rpc/thrift/TTypeId.java | 5 +- .../src/gen/thrift/gen-php/TTypeId.php | 3 + .../apache/hive/service/cli/ColumnValue.java | 2 + .../hadoop/hive/metastore/ColumnType.java | 2 + 45 files changed, 676 insertions(+), 10 deletions(-) create mode 100644 iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergUnknownType.java create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/unknown_type.q create mode 100644 iceberg/iceberg-handler/src/test/results/positive/unknown_type.q.out create mode 100644 ql/src/test/queries/clientnegative/unknown_type_non_iceberg_table.q create mode 100644 ql/src/test/results/clientnegative/unknown_type_non_iceberg_table.q.out create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnknownObjectInspector.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnknownTypeInfo.java create mode 100644 serde/src/test/org/apache/hadoop/hive/serde2/typeinfo/TestUnknownTypeInfo.java diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java index 07631516cd49..e7a0ad915711 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java @@ -186,6 +186,8 @@ Type convertType(TypeInfo typeInfo, String defaultValue, boolean shouldAddInitia return Types.ListType.ofOptional(listId, listType); case VARIANT: return Types.VariantType.get(); + case UNKNOWN: + return Types.UnknownType.get(); default: throw new IllegalArgumentException("Unknown type " + typeInfo.getCategory()); } diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java index 3f18deab4eaf..306fb343e226 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java @@ -411,6 +411,8 @@ public static String convertToTypeString(Type type) { return String.format("map<%s,%s>", convert(mapType.keyType()), convert(mapType.valueType())); case VARIANT: return "variant"; + case UNKNOWN: + return "unknown"; default: throw new UnsupportedOperationException(type + " is not supported"); } diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java index 6234a6f8db50..838344ac62d8 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.iceberg.Schema; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; @@ -171,6 +172,35 @@ public void testConversionWithoutLastComment() { assertThat(schema.asStruct()).isEqualTo(expected.asStruct()); } + @Test + public void testUnknownTypeConvertToIcebergSchema() { + List hiveSchema = ImmutableList.of( + new FieldSchema("id", serdeConstants.INT_TYPE_NAME, null), + new FieldSchema("placeholder", serdeConstants.UNKNOWN_TYPE_NAME, "placeholder comment")); + Schema expected = new Schema( + optional(1, "id", Types.IntegerType.get()), + optional(2, "placeholder", Types.UnknownType.get(), "placeholder comment")); + + assertThat(HiveSchemaUtil.convert(hiveSchema).asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + public void testUnknownTypeConvertToHiveSchema() { + Schema icebergSchema = new Schema( + optional(1, "id", Types.IntegerType.get()), + optional(2, "placeholder", Types.UnknownType.get(), "placeholder comment")); + List expected = ImmutableList.of( + new FieldSchema("id", serdeConstants.INT_TYPE_NAME, null), + new FieldSchema("placeholder", serdeConstants.UNKNOWN_TYPE_NAME, "placeholder comment")); + + assertThat(HiveSchemaUtil.convert(icebergSchema)).isEqualTo(expected); + } + + @Test + public void testUnknownTypeAndTypeInfoConvert() { + checkConvert(TypeInfoFactory.getUnknownTypeInfo(), Types.UnknownType.get()); + } + protected List getSupportedFieldSchemas() { List fields = Lists.newArrayList(); fields.add(new FieldSchema("c_float", serdeConstants.FLOAT_TYPE_NAME, "float comment")); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/Deserializer.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/Deserializer.java index dbafe176f6bd..95bc7fd650f0 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/Deserializer.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/Deserializer.java @@ -37,6 +37,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.schema.SchemaWithPartnerVisitor; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Type.PrimitiveType; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.ListType; @@ -110,6 +111,15 @@ public FieldDeserializer field(NestedField field, ObjectInspectorPair pair, Fiel @Override public FieldDeserializer primitive(PrimitiveType type, ObjectInspectorPair pair) { + if (type.typeId() == Type.TypeID.UNKNOWN) { + return o -> { + if (o != null) { + throw new IllegalArgumentException("UNKNOWN type columns only accept NULL values"); + } + return null; + }; + } + return o -> { if (o == null) { return null; diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java index 6f4b563fa5f3..bf5158d62a52 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java @@ -23,6 +23,7 @@ import javax.annotation.Nullable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.UnknownObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -106,6 +107,8 @@ public ObjectInspector primitive(Type.PrimitiveType primitiveType) { IcebergTimestampObjectInspectorHive3.get(9); case TIME: return IcebergTimeObjectInspector.get(); + case UNKNOWN: + return UnknownObjectInspector.get(); default: throw new IllegalArgumentException(primitiveType.typeId() + " type is not supported"); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveValueConverter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveValueConverter.java index 0804a7abe9fb..e5e61c46606b 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveValueConverter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveValueConverter.java @@ -104,6 +104,8 @@ public static Object convert(Type type, Object object) { case DOUBLE: case STRING: return object; + case UNKNOWN: + return null; default: throw new UnsupportedOperationException("Not a supported type: " + type); } diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergUnknownType.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergUnknownType.java new file mode 100644 index 000000000000..fd2647964c62 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergUnknownType.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive; + +import java.util.Collection; +import java.util.List; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Schema; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.mr.hive.test.TestTables.TestTableType; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runners.Parameterized.Parameters; + +import static org.apache.iceberg.types.Types.NestedField.optional; + +public class TestHiveIcebergUnknownType extends HiveIcebergStorageHandlerWithEngineBase { + + @Parameters(name = "fileFormat={0}, catalog={1}, isVectorized={2}, formatVersion={3}") + public static Collection parameters() { + return HiveIcebergStorageHandlerWithEngineBase.getParameters(p -> + p.fileFormat() == FileFormat.PARQUET && + p.testTableType() == TestTableType.HIVE_CATALOG && + p.formatVersion() == 3); + } + + @Test + public void testUnknownTypeReadWrite() { + Schema schema = new Schema( + optional(1, "id", Types.IntegerType.get()), + optional(2, "placeholder", Types.UnknownType.get())); + + TableIdentifier table = TableIdentifier.of("default", "unknown_type_basic"); + + testTables.createTable(shell, table.name(), + schema, org.apache.iceberg.PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of(), formatVersion, + ImmutableMap.of()); + + shell.executeStatement( + String.format("INSERT INTO %s VALUES (1, NULL), (2, NULL)", table)); + + List rows = shell.executeStatement( + String.format("SELECT id, placeholder FROM %s ORDER BY id", table)); + + Assert.assertEquals(2, rows.size()); + Assert.assertEquals(1, rows.get(0)[0]); + Assert.assertNull(rows.get(0)[1]); + Assert.assertEquals(2, rows.get(1)[0]); + Assert.assertNull(rows.get(1)[1]); + } + + @Test + public void testUnknownTypeDdl() { + TableIdentifier table = TableIdentifier.of("default", "unknown_type_ddl"); + + shell.executeStatement(String.format( + "CREATE EXTERNAL TABLE %s (id INT, placeholder UNKNOWN) STORED BY ICEBERG " + + "TBLPROPERTIES ('format-version'='3')", table)); + + shell.executeStatement(String.format("INSERT INTO %s VALUES (1, NULL)", table)); + + List rows = shell.executeStatement( + String.format("SELECT id, placeholder FROM %s", table)); + + Assert.assertEquals(1, rows.size()); + Assert.assertEquals(1, rows.get(0)[0]); + Assert.assertNull(rows.get(0)[1]); + } +} diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java index a4f2dcc10f5b..7e9a3eb6f47c 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.UnknownObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -32,6 +33,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import static org.apache.iceberg.types.Types.NestedField.optional; import static org.apache.iceberg.types.Types.NestedField.required; @@ -60,7 +62,8 @@ public class TestIcebergObjectInspector { Types.NestedField.required(20, "nested_field", Types.StringType.get(), "nested field comment")), "struct comment" ), - required(21, "time_field", Types.TimeType.get(), "time comment") + required(21, "time_field", Types.TimeType.get(), "time comment"), + optional(22, "unknown_field", Types.UnknownType.get(), "unknown comment") ); @Test @@ -205,6 +208,13 @@ public void testIcebergObjectInspector() { Assertions.assertEquals("time_field", timeField.getFieldName()); Assertions.assertEquals("time comment", timeField.getFieldComment()); Assertions.assertEquals(IcebergTimeObjectInspector.get(), timeField.getFieldObjectInspector()); + + // unknown + StructField unknownField = soi.getStructFieldRef("unknown_field"); + Assertions.assertEquals(22, unknownField.getFieldID()); + Assertions.assertEquals("unknown_field", unknownField.getFieldName()); + Assertions.assertEquals("unknown comment", unknownField.getFieldComment()); + Assertions.assertEquals(UnknownObjectInspector.get(), unknownField.getFieldObjectInspector()); } private static ObjectInspector getPrimitiveObjectInspector(Class clazz) { diff --git a/iceberg/iceberg-handler/src/test/queries/positive/unknown_type.q b/iceberg/iceberg-handler/src/test/queries/positive/unknown_type.q new file mode 100644 index 000000000000..60810519740b --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/unknown_type.q @@ -0,0 +1,40 @@ +-- SORT_QUERY_RESULTS +-- Mask neededVirtualColumns due to non-strict order +--! qt:replace:/(\s+neededVirtualColumns:\s)(.*)/$1#Masked#/ +-- Mask random uuid +--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/ +-- Mask a random snapshot id +--! qt:replace:/(\s+current-snapshot-id\s+)\S+(\s*)/$1#Masked#/ +-- Mask added file size +--! qt:replace:/(\S\"added-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ +-- Mask total file size +--! qt:replace:/(\S\"total-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ +-- Mask current-snapshot-timestamp-ms +--! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/ +-- Mask removed file size +--! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ +-- Mask iceberg version +--! qt:replace:/(\S\"iceberg-version\\\":\\\")(\w+\s\w+\s\d+\.\d+\.\d+\s\(\w+\s\w+\))(\\\")/$1#Masked#$3/ + +-- Create test table with unknown placeholder column +CREATE EXTERNAL TABLE unknown_test_basic ( + id INT, + placeholder UNKNOWN +) STORED BY ICEBERG tblproperties('format-version'='3'); + +-- Unknown columns are not stored; only NULL values are accepted +INSERT INTO unknown_test_basic VALUES +(1, NULL), +(2, NULL); + +SELECT id, placeholder FROM unknown_test_basic ORDER BY id; + +-- Add another unknown column to an existing table +ALTER TABLE unknown_test_basic ADD COLUMNS (extra UNKNOWN); + +INSERT INTO unknown_test_basic VALUES +(3, NULL, NULL); + +SELECT id, placeholder, extra FROM unknown_test_basic ORDER BY id; + +DESC FORMATTED unknown_test_basic; diff --git a/iceberg/iceberg-handler/src/test/results/positive/unknown_type.q.out b/iceberg/iceberg-handler/src/test/results/positive/unknown_type.q.out new file mode 100644 index 000000000000..d5f0566abfbc --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/unknown_type.q.out @@ -0,0 +1,115 @@ +PREHOOK: query: CREATE EXTERNAL TABLE unknown_test_basic ( + id INT, + placeholder UNKNOWN +) STORED BY ICEBERG tblproperties('format-version'='3') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@unknown_test_basic +POSTHOOK: query: CREATE EXTERNAL TABLE unknown_test_basic ( + id INT, + placeholder UNKNOWN +) STORED BY ICEBERG tblproperties('format-version'='3') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@unknown_test_basic +PREHOOK: query: INSERT INTO unknown_test_basic VALUES +(1, NULL), +(2, NULL) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@unknown_test_basic +POSTHOOK: query: INSERT INTO unknown_test_basic VALUES +(1, NULL), +(2, NULL) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@unknown_test_basic +PREHOOK: query: SELECT id, placeholder FROM unknown_test_basic ORDER BY id +PREHOOK: type: QUERY +PREHOOK: Input: default@unknown_test_basic +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT id, placeholder FROM unknown_test_basic ORDER BY id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@unknown_test_basic +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 NULL +2 NULL +PREHOOK: query: ALTER TABLE unknown_test_basic ADD COLUMNS (extra UNKNOWN) +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@unknown_test_basic +PREHOOK: Output: default@unknown_test_basic +POSTHOOK: query: ALTER TABLE unknown_test_basic ADD COLUMNS (extra UNKNOWN) +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@unknown_test_basic +POSTHOOK: Output: default@unknown_test_basic +PREHOOK: query: INSERT INTO unknown_test_basic VALUES +(3, NULL, NULL) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@unknown_test_basic +POSTHOOK: query: INSERT INTO unknown_test_basic VALUES +(3, NULL, NULL) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@unknown_test_basic +PREHOOK: query: SELECT id, placeholder, extra FROM unknown_test_basic ORDER BY id +PREHOOK: type: QUERY +PREHOOK: Input: default@unknown_test_basic +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT id, placeholder, extra FROM unknown_test_basic ORDER BY id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@unknown_test_basic +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 NULL NULL +2 NULL NULL +3 NULL NULL +PREHOOK: query: DESC FORMATTED unknown_test_basic +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@unknown_test_basic +POSTHOOK: query: DESC FORMATTED unknown_test_basic +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@unknown_test_basic +# col_name data_type comment +id int +placeholder unknown +extra unknown + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + EXTERNAL TRUE + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"placeholder\",\"required\":false,\"type\":\"unknown\"},{\"id\":3,\"name\":\"extra\",\"required\":false,\"type\":\"unknown\"}]} + current-snapshot-id #Masked# + current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"3\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-timestamp-ms #Masked# + format-version 3 + metadata_location hdfs://### HDFS PATH ### + numFiles 2 + numRows 3 + parquet.compression zstd + previous_metadata_location hdfs://### HDFS PATH ### + rawDataSize 0 + serialization.format 1 + snapshot-count 2 + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG + totalSize #Masked# +#### A masked pattern was here #### + uuid #Masked# + write.delete.mode merge-on-read + write.merge.mode merge-on-read + write.metadata.delete-after-commit.enabled true + write.update.mode merge-on-read + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [] diff --git a/jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java b/jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java index d3b7098fa7fe..2747ae751d46 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java +++ b/jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java @@ -168,6 +168,8 @@ static Type typeStringToHiveType(String type) throws SQLException { /* fall through */ case "null": return Type.NULL_TYPE; + case serdeConstants.UNKNOWN_TYPE_NAME: + return Type.UNKNOWN_TYPE; default: throw new SQLException("Unrecognized column type: " + type); } @@ -224,6 +226,8 @@ static String getColumnTypeName(String type) throws SQLException { /* fall through */ case "null": return serdeConstants.VOID_TYPE_NAME; + case serdeConstants.UNKNOWN_TYPE_NAME: + return serdeConstants.UNKNOWN_TYPE_NAME; case "map": return serdeConstants.MAP_TYPE_NAME; case "array": diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 0033aa3251c6..fba3f2972210 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -2420,7 +2420,8 @@ type | structType | mapType | unionType - | variantType; + | variantType + | unknownType; primitiveType @init { pushMsg("primitive type specification", state); } @@ -2486,6 +2487,12 @@ variantType : KW_VARIANT -> TOK_VARIANT ; +unknownType +@init { pushMsg("unknown type", state); } +@after { popMsg(state); } + : KW_UNKNOWN -> TOK_UNKNOWN + ; + setOperator @init { pushMsg("set operator", state); } @after { popMsg(state); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java index 5e3768660f62..9f73060cc0b4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java @@ -626,6 +626,9 @@ private void assignRowColumn( assignRowColumn(unionColumnVector.fields[tag], batchIndex, objectTypeInfos.get(tag), union.getObject()); } break; + case UNKNOWN: + VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex); + return; default: throw new RuntimeException("Category " + targetTypeInfo.getCategory().name() + " not supported"); } @@ -986,6 +989,9 @@ private void assignConvertRowColumn(ColumnVector columnVector, int batchIndex, unionObjectInspector.getField(tag)); } break; + case UNKNOWN: + VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex); + return; default: throw new RuntimeException("Category " + targetCategory.name() + " not supported"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java index c1f5d8ca32c3..6410ff80e93d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java @@ -355,6 +355,8 @@ private Field allocateComplexField(TypeInfo sourceTypeInfo) { new UnionComplexTypeHelper(fields); return new Field(category, unionHelper, sourceTypeInfo, new VectorUnionDeserializer()); } + case UNKNOWN: + return new Field(category, null, sourceTypeInfo, new VectorVoidDeserializer()); default: throw new RuntimeException("Category " + category + " not supported"); } @@ -368,6 +370,7 @@ private Field allocateField(TypeInfo sourceTypeInfo) { case MAP: case STRUCT: case UNION: + case UNKNOWN: return allocateComplexField(sourceTypeInfo); default: throw new RuntimeException("Category " + sourceTypeInfo.getCategory() + " not supported"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java index b3be81646c5c..9945f957ef59 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java @@ -564,6 +564,8 @@ public Object extractRowColumn( standardUnion.setObject(object); return standardUnion; } + case UNKNOWN: + return null; default: throw new RuntimeException("Category " + category.name() + " not supported"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java index fd4fffe292f7..deca72908002 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java @@ -176,6 +176,10 @@ private Field createField(TypeInfo typeInfo) { field.children = createFields(objectTypeInfos.toArray(new TypeInfo[objectTypeInfos.size()])); field.writer = new VectorSerializeUnionWriter(); break; + case UNKNOWN: + field.children = new Field[0]; + field.writer = new VectorSerializeUnknownWriter(); + break; default: throw new RuntimeException(); } @@ -579,6 +583,13 @@ void serialize(Object colInfo, Field field, int adjustedBatchIndex) throws IOExc } } + class VectorSerializeUnknownWriter extends VectorSerializeWriter { + @Override + void serialize(Object colInfo, Field field, int adjustedBatchIndex) throws IOException { + // Unknown type values are always null and handled before the writer is invoked. + } + } + public boolean getHasAnyNulls() { return hasAnyNulls; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index 894231e991e0..c6d2625eb778 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -4581,6 +4581,8 @@ public static ColumnVector.Type getColumnVectorTypeFromTypeInfo(TypeInfo typeInf throw new HiveException("Unexpected primitive type category " + primitiveCategory); } } + case UNKNOWN: + return ColumnVector.Type.VOID; default: throw new HiveException("Unexpected type category " + typeInfo.getCategory()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java index 4be9e1435f03..d4d4af31a945 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java @@ -208,6 +208,8 @@ public static ColumnVector createColumnVector(TypeInfo typeInfo, createColumnVector(mapTypeInfo.getMapKeyTypeInfo()), createColumnVector(mapTypeInfo.getMapValueTypeInfo())); } + case UNKNOWN: + return new VoidColumnVector(VectorizedRowBatch.DEFAULT_SIZE); default: throw new RuntimeException("Vectorization is not supported for datatype:" + typeInfo.getCategory()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java index 163347febeb9..4c70bac54260 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java @@ -738,6 +738,9 @@ public static VectorExpressionWriter genVectorExpressionWritable( return genVectorExpressionWritableUnion( (SettableUnionObjectInspector) fieldObjInspector); + case UNKNOWN: + return genVectorExpressionWritableUnknown(fieldObjInspector); + default: throw new IllegalArgumentException("Unknown type " + fieldObjInspector.getCategory()); @@ -1719,6 +1722,28 @@ public Object setValue(Object union, ColumnVector column, int row) }.init(fieldObjInspector); } + private static VectorExpressionWriter genVectorExpressionWritableUnknown( + ObjectInspector fieldObjInspector) throws HiveException { + return new VectorExpressionWriterBase() { + + @Override + public Object writeValue(ColumnVector column, int row) throws HiveException { + return null; + } + + @Override + public Object setValue(Object row, ColumnVector column, int columnRow) + throws HiveException { + return null; + } + + @Override + public Object initValue(Object ost) throws HiveException { + return null; + } + }.init(fieldObjInspector); + } + // For complex types like STRUCT, MAP, etc we do not support, we need a writer that // does nothing. We assume the Vectorizer class has not validated the query to actually // try and use the complex types. They do show up in inputObjInspector[0] and need to be diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java index a59c15ee5d8c..e1ca092a20a6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java @@ -129,7 +129,9 @@ private static Type convertType(final String name, final TypeInfo typeInfo, Conf return Types.primitive(PrimitiveTypeName.INT32, repetition) .as(LogicalTypeAnnotation.dateType()).named(name); } else if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) { - throw new UnsupportedOperationException("Unknown type not implemented"); + throw new UnsupportedOperationException("Legacy unknown primitive type is not supported for Parquet"); + } else if (typeInfo.equals(TypeInfoFactory.getUnknownTypeInfo())) { + throw new UnsupportedOperationException("Unknown type is not stored in Parquet data files"); } else { throw new IllegalArgumentException("Unknown type: " + typeInfo); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 05faa38d7caf..3774ae2d4180 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -90,6 +90,7 @@ import com.google.common.base.Preconditions; import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.hadoop.hive.serde.serdeConstants.UNKNOWN_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.VARIANT_TYPE_NAME; /** @@ -1393,6 +1394,7 @@ public List getVirtualColumns() { private static boolean isUnsupportedInNonIceberg(String columnType) { return VARIANT_TYPE_NAME.equalsIgnoreCase(columnType) || + UNKNOWN_TYPE_NAME.equalsIgnoreCase(columnType) || TypeInfoFactory.nanoTimestampTypeInfo.getQualifiedName().equalsIgnoreCase(columnType) || TypeInfoFactory.timestampNanoLocalTZTypeInfo.getQualifiedName().equalsIgnoreCase(columnType); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java index 1f7ccd1773cb..9fa5ebf286d0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java @@ -164,6 +164,12 @@ public static RelDataType convert(TypeInfo type, boolean nullable, RelDataTypeFa case UNION: convertedType = convert((UnionTypeInfo) type, dtFactory); break; + case VARIANT: + convertedType = dtFactory.createSqlType(SqlTypeName.OTHER); + break; + case UNKNOWN: + convertedType = dtFactory.createSqlType(SqlTypeName.NULL); + break; } // hive does not have concept of not nullable types return dtFactory.createTypeWithNullability(convertedType, nullable); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 8f763b4857c8..434e131b4d0c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1048,6 +1048,7 @@ public static String getTypeStringFromAST(ASTNode typeNode) TOKEN_TO_TYPE.put(HiveParser.TOK_INTERVAL_DAY_TIME, serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME); TOKEN_TO_TYPE.put(HiveParser.TOK_DECIMAL, serdeConstants.DECIMAL_TYPE_NAME); TOKEN_TO_TYPE.put(HiveParser.TOK_VARIANT, serdeConstants.VARIANT_TYPE_NAME); + TOKEN_TO_TYPE.put(HiveParser.TOK_UNKNOWN, serdeConstants.UNKNOWN_TYPE_NAME); } private static String getTypeName(ASTNode node) throws SemanticException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 7781ded7bfa7..a8a4b18ec583 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -9121,6 +9121,9 @@ private ExprNodeDesc handleConversion(StructField tableField, ColumnInfo rowFiel // JSON-format. However, we may add more operators. // Thus, we still keep the conversion. if (!tableFieldTypeInfo.equals(rowFieldTypeInfo)) { + if (TypeInfoUtils.isVoidCompatibleTarget(rowFieldTypeInfo, tableFieldTypeInfo)) { + return column; + } // need to do some conversions here conversion.set(true); if (tableFieldTypeInfo.getCategory() != Category.PRIMITIVE) { @@ -9343,8 +9346,8 @@ private List genConvertCol(String dest, QB qb, TableDesc tableDesc rowFields.get(posn).getInternalName(), rowFields.get(posn).getTabAlias(), rowFields.get(posn).getIsVirtualCol()); - if (convert && !tableFieldTypeInfo.equals(rowFieldTypeInfo)) { - // need to do some conversions here + if (convert && !tableFieldTypeInfo.equals(rowFieldTypeInfo) + && !TypeInfoUtils.isVoidCompatibleTarget(rowFieldTypeInfo, tableFieldTypeInfo)) { if (tableFieldTypeInfo.getCategory() != Category.PRIMITIVE) { // cannot convert to complex types column = null; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java index 2faefafdee35..bb09151e7884 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java @@ -108,6 +108,7 @@ void testPrimitiveSizeEstimations() throws Exception { exclusions.add(serdeConstants.STRUCT_TYPE_NAME); exclusions.add(serdeConstants.UNION_TYPE_NAME); exclusions.add(serdeConstants.VARIANT_TYPE_NAME); + exclusions.add(serdeConstants.UNKNOWN_TYPE_NAME); Field[] serdeFields = serdeConstants.class.getFields(); for (Field field : serdeFields) { if (!Modifier.isStatic(field.getModifiers())) { diff --git a/ql/src/test/queries/clientnegative/unknown_type_non_iceberg_table.q b/ql/src/test/queries/clientnegative/unknown_type_non_iceberg_table.q new file mode 100644 index 000000000000..85084d25713c --- /dev/null +++ b/ql/src/test/queries/clientnegative/unknown_type_non_iceberg_table.q @@ -0,0 +1 @@ +CREATE EXTERNAL TABLE unknown_test_basic (id INT, placeholder UNKNOWN); diff --git a/ql/src/test/results/clientnegative/unknown_type_non_iceberg_table.q.out b/ql/src/test/results/clientnegative/unknown_type_non_iceberg_table.q.out new file mode 100644 index 000000000000..597c0f28acf8 --- /dev/null +++ b/ql/src/test/results/clientnegative/unknown_type_non_iceberg_table.q.out @@ -0,0 +1,5 @@ +PREHOOK: query: CREATE EXTERNAL TABLE unknown_test_basic (id INT, placeholder UNKNOWN) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@unknown_test_basic +FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. org.apache.hadoop.hive.ql.metadata.HiveException: Column name placeholder cannot be of type 'unknown' as it is not supported in non-Iceberg tables. diff --git a/serde/if/serde.thrift b/serde/if/serde.thrift index f8d0f0a06aca..057f227b75e9 100644 --- a/serde/if/serde.thrift +++ b/serde/if/serde.thrift @@ -71,6 +71,7 @@ const string MAP_TYPE_NAME = "map"; const string STRUCT_TYPE_NAME = "struct"; const string UNION_TYPE_NAME = "uniontype"; const string VARIANT_TYPE_NAME = "variant"; +const string UNKNOWN_TYPE_NAME = "unknown"; const string LIST_COLUMNS = "columns"; const string LIST_COLUMN_TYPES = "columns.types"; diff --git a/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java b/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java index 20e607a67bdb..d43bc3d8d969 100644 --- a/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java +++ b/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java @@ -94,6 +94,7 @@ public class serdeConstants { public static final java.lang.String UNION_TYPE_NAME = "uniontype"; public static final java.lang.String VARIANT_TYPE_NAME = "variant"; + public static final java.lang.String UNKNOWN_TYPE_NAME = "unknown"; public static final java.lang.String LIST_COLUMNS = "columns"; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java index 8b9c012d9b99..4a51e6c90dd6 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java @@ -68,6 +68,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; @@ -370,6 +371,10 @@ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, return LazyObjectInspectorFactory.getLazyUnionObjectInspector(lazyOIs, LazyUtils.getSeparator(lazyParams.getSeparators(), separatorIndex), lazyParams); + case UNKNOWN: + // Unknown columns are always null and are not stored in data files. + return LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector( + TypeInfoFactory.voidTypeInfo, lazyParams); } throw new RuntimeException("Hive LazySerDe Internal error."); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java index ed521eed68b6..f801a42b1125 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; import org.apache.hadoop.io.WritableUtils; @@ -513,6 +514,11 @@ public static ObjectInspector getLazyBinaryObjectInspectorFromTypeInfo( .getLazyBinaryUnionObjectInspector(fieldObjectInspectors); break; } + case UNKNOWN: + // Unknown columns are always null and are not stored in data files. + result = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( + TypeInfoFactory.voidTypeInfo); + break; default: { result = null; } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java index 8e4183b615f4..c342098c3e4c 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java @@ -47,7 +47,7 @@ public interface ObjectInspector extends Cloneable { * */ enum Category { - PRIMITIVE, LIST, MAP, STRUCT, UNION, VARIANT + PRIMITIVE, LIST, MAP, STRUCT, UNION, VARIANT, UNKNOWN }; /** diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java index 8e251d19bb3b..9f6247466b14 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java @@ -367,6 +367,10 @@ public static ObjectInspector getVariantObjectInspector() { return VariantObjectInspector.get(); } + public static ObjectInspector getUnknownObjectInspector() { + return UnknownObjectInspector.get(); + } + public static ColumnarStructObjectInspector getColumnarStructObjectInspector( List structFieldNames, List structFieldObjectInspectors) { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnknownObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnknownObjectInspector.java new file mode 100644 index 000000000000..0facbd6c8101 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnknownObjectInspector.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.objectinspector; + +/** + * ObjectInspector for Iceberg V3 unknown type columns. + *

+ * Unknown values are always null and are not stored in data files. + */ +public final class UnknownObjectInspector implements ObjectInspector { + + private static final ObjectInspector INSTANCE = new UnknownObjectInspector(); + + private UnknownObjectInspector() { + } + + public static ObjectInspector get() { + return INSTANCE; + } + + @Override + public Category getCategory() { + return Category.UNKNOWN; + } + + @Override + public String getTypeName() { + return "unknown"; + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/thrift/Type.java b/serde/src/java/org/apache/hadoop/hive/serde2/thrift/Type.java index ca5abdbbf2cf..6773b8687005 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/thrift/Type.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/thrift/Type.java @@ -109,7 +109,10 @@ public enum Type { USER_DEFINED_TYPE("USER_DEFINED", java.sql.Types.OTHER, TTypeId.USER_DEFINED_TYPE, - true, false); + true, false), + UNKNOWN_TYPE(serdeConstants.UNKNOWN_TYPE_NAME.toUpperCase(), + java.sql.Types.NULL, + TTypeId.UNKNOWN_TYPE); private final String name; private final TTypeId tType; @@ -262,6 +265,9 @@ public static Type getType(TypeInfo typeInfo) { case UNION: { return Type.UNION_TYPE; } + case UNKNOWN: { + return Type.UNKNOWN_TYPE; + } default: { throw new RuntimeException("Unrecognized type: " + typeInfo.getCategory()); } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java index f80c51926043..bee5ac82dc98 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java @@ -281,6 +281,10 @@ public static TypeInfo getVariantTypeInfo() { return VariantTypeInfo.get(); } + public static TypeInfo getUnknownTypeInfo() { + return UnknownTypeInfo.get(); + } + static ConcurrentHashMap cachedListTypeInfo = new ConcurrentHashMap(); public static TypeInfo getListTypeInfo(TypeInfo elementTypeInfo) { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java index b25d35868df7..ef4bdd2ae7f6 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java @@ -400,6 +400,7 @@ private Token expect(String item, String alternative) { && !serdeConstants.STRUCT_TYPE_NAME.equals(t.text) && !serdeConstants.UNION_TYPE_NAME.equals(t.text) && !serdeConstants.VARIANT_TYPE_NAME.equals(t.text) + && !serdeConstants.UNKNOWN_TYPE_NAME.equals(t.text) && null == PrimitiveObjectInspectorUtils .getTypeEntryFromTypeName(t.text) && !t.text.equals(alternative)) { @@ -579,6 +580,11 @@ private TypeInfo parseType() { return TypeInfoFactory.getVariantTypeInfo(); } + // Is this an unknown type? + if (serdeConstants.UNKNOWN_TYPE_NAME.equals(t.text)) { + return TypeInfoFactory.getUnknownTypeInfo(); + } + throw new RuntimeException("Internal error parsing position " + t.position + " of '" + typeInfoString + "'"); } @@ -683,6 +689,14 @@ public static ObjectInspector getStandardWritableObjectInspectorFromTypeInfo( fieldObjectInspectors); break; } + case VARIANT: { + result = ObjectInspectorFactory.getVariantObjectInspector(); + break; + } + case UNKNOWN: { + result = ObjectInspectorFactory.getUnknownObjectInspector(); + break; + } default: { result = null; @@ -765,6 +779,14 @@ public static ObjectInspector getStandardJavaObjectInspectorFromTypeInfo( fieldObjectInspectors); break; } + case VARIANT: { + result = ObjectInspectorFactory.getVariantObjectInspector(); + break; + } + case UNKNOWN: { + result = ObjectInspectorFactory.getUnknownObjectInspector(); + break; + } default: { result = null; } @@ -837,6 +859,14 @@ public static TypeInfo getTypeInfoFromObjectInspector(ObjectInspector oi) { result = TypeInfoFactory.getUnionTypeInfo(objectTypeInfos); break; } + case VARIANT: { + result = TypeInfoFactory.getVariantTypeInfo(); + break; + } + case UNKNOWN: { + result = TypeInfoFactory.getUnknownTypeInfo(); + break; + } default: { throw new RuntimeException("Unknown ObjectInspector category!"); } @@ -1013,6 +1043,16 @@ public static boolean implicitConvertible(PrimitiveCategory from, PrimitiveCateg return true; } + /** + * Returns whether a NULL literal (void) can be assigned to the target type. + * Iceberg unknown columns only accept NULL values. + */ + public static boolean isVoidCompatibleTarget(TypeInfo from, TypeInfo to) { + return from.getCategory() == Category.PRIMITIVE + && ((PrimitiveTypeInfo) from).getPrimitiveCategory() == PrimitiveCategory.VOID + && to.getCategory() == Category.UNKNOWN; + } + /** * Returns whether it is possible to implicitly convert an object of Class * from to Class to. @@ -1022,6 +1062,10 @@ public static boolean implicitConvertible(TypeInfo from, TypeInfo to) { return true; } + if (isVoidCompatibleTarget(from, to)) { + return true; + } + // Reimplemented to use PrimitiveCategory rather than TypeInfo, because // 2 TypeInfos from the same qualified type (varchar, decimal) should still be // seen as equivalent. diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnknownTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnknownTypeInfo.java new file mode 100644 index 000000000000..19c19524de48 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnknownTypeInfo.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serial; +import java.io.Serializable; +import java.util.Objects; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; + +/** + * UnknownTypeInfo represents the TypeInfo of an Iceberg V3 unknown type in Hive. + *

+ * Unknown columns are schema placeholders for undetermined types. They must be optional, + * default to null, and are not stored in data files. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public final class UnknownTypeInfo extends TypeInfo implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private static final UnknownTypeInfo INSTANCE = new UnknownTypeInfo(); + + public static UnknownTypeInfo get() { + return INSTANCE; + } + + @Override + public Category getCategory() { + return Category.UNKNOWN; + } + + @Override + public String getTypeName() { + return "unknown"; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + return o instanceof UnknownTypeInfo; + } + + @Override + public int hashCode() { + return Objects.hash(UnknownTypeInfo.class, getTypeName()); + } +} diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/typeinfo/TestUnknownTypeInfo.java b/serde/src/test/org/apache/hadoop/hive/serde2/typeinfo/TestUnknownTypeInfo.java new file mode 100644 index 000000000000..87a73c2031e4 --- /dev/null +++ b/serde/src/test/org/apache/hadoop/hive/serde2/typeinfo/TestUnknownTypeInfo.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.util.List; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.lazy.LazyFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.UnknownObjectInspector; +import org.apache.hadoop.hive.serde2.thrift.Type; +import org.apache.hadoop.io.Text; +import org.junit.Assert; +import org.junit.Test; + +public class TestUnknownTypeInfo { + + @Test + public void testParseUnknownType() { + TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(serdeConstants.UNKNOWN_TYPE_NAME); + Assert.assertEquals(UnknownTypeInfo.get(), typeInfo); + Assert.assertEquals(ObjectInspector.Category.UNKNOWN, typeInfo.getCategory()); + } + + @Test + public void testUnknownObjectInspector() { + ObjectInspector oi = ObjectInspectorFactory.getUnknownObjectInspector(); + Assert.assertEquals(UnknownObjectInspector.get(), oi); + Assert.assertEquals(ObjectInspector.Category.UNKNOWN, oi.getCategory()); + Assert.assertEquals(serdeConstants.UNKNOWN_TYPE_NAME, oi.getTypeName()); + } + + @Test + public void testVoidIsCompatibleWithUnknownType() { + Assert.assertTrue(TypeInfoUtils.isVoidCompatibleTarget( + TypeInfoFactory.voidTypeInfo, TypeInfoFactory.getUnknownTypeInfo())); + Assert.assertTrue(TypeInfoUtils.implicitConvertible( + TypeInfoFactory.voidTypeInfo, TypeInfoFactory.getUnknownTypeInfo())); + } + + @Test + public void testLazyObjectInspectorForUnknownType() throws SerDeException { + ObjectInspector oi = LazyFactory.createLazyStructInspector( + List.of("placeholder"), + List.of(TypeInfoFactory.getUnknownTypeInfo()), + new byte[] {(byte) 1, (byte) 2, (byte) 3, (byte) 4, (byte) 5, (byte) 6, (byte) 7, (byte) 8}, + new Text("\\N"), false, false, (byte) 0); + Assert.assertNotNull(oi); + } + + @Test + public void testStandardObjectInspectorFromTypeInfo() { + ObjectInspector oi = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(UnknownTypeInfo.get()); + Assert.assertEquals(UnknownObjectInspector.get(), oi); + } + + @Test + public void testThriftTypeMapping() { + Assert.assertEquals(Type.UNKNOWN_TYPE, Type.getType(serdeConstants.UNKNOWN_TYPE_NAME)); + Assert.assertEquals(Type.UNKNOWN_TYPE, Type.getType(UnknownTypeInfo.get())); + } +} diff --git a/service-rpc/if/TCLIService.thrift b/service-rpc/if/TCLIService.thrift index a399e66445c8..ee58cd9979d2 100644 --- a/service-rpc/if/TCLIService.thrift +++ b/service-rpc/if/TCLIService.thrift @@ -94,7 +94,8 @@ enum TTypeId { CHAR_TYPE, INTERVAL_YEAR_MONTH_TYPE, INTERVAL_DAY_TIME_TYPE, - TIMESTAMPLOCALTZ_TYPE + TIMESTAMPLOCALTZ_TYPE, + UNKNOWN_TYPE } const set PRIMITIVE_TYPES = [ @@ -115,7 +116,8 @@ const set PRIMITIVE_TYPES = [ TTypeId.CHAR_TYPE, TTypeId.INTERVAL_YEAR_MONTH_TYPE, TTypeId.INTERVAL_DAY_TIME_TYPE, - TTypeId.TIMESTAMPLOCALTZ_TYPE + TTypeId.TIMESTAMPLOCALTZ_TYPE, + TTypeId.UNKNOWN_TYPE ] const set COMPLEX_TYPES = [ @@ -154,6 +156,7 @@ const map TYPE_NAMES = { TTypeId.INTERVAL_YEAR_MONTH_TYPE: "INTERVAL_YEAR_MONTH" TTypeId.INTERVAL_DAY_TIME_TYPE: "INTERVAL_DAY_TIME" TTypeId.TIMESTAMPLOCALTZ_TYPE: "TIMESTAMP WITH LOCAL TIME ZONE" + TTypeId.UNKNOWN_TYPE: "UNKNOWN" } // Thrift does not support recursively defined types or forward declarations, diff --git a/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TCLIServiceConstants.java b/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TCLIServiceConstants.java index 7b9c26b3d756..e525f2cdae32 100644 --- a/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TCLIServiceConstants.java +++ b/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TCLIServiceConstants.java @@ -29,6 +29,7 @@ PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.INTERVAL_YEAR_MONTH_TYPE); PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.INTERVAL_DAY_TIME_TYPE); PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.TIMESTAMPLOCALTZ_TYPE); + PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.UNKNOWN_TYPE); } public static final java.util.Set COMPLEX_TYPES = java.util.EnumSet.noneOf(TTypeId.class); @@ -69,6 +70,7 @@ TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.TIMESTAMP_TYPE, "TIMESTAMP"); TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.TINYINT_TYPE, "TINYINT"); TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.UNION_TYPE, "UNIONTYPE"); + TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.UNKNOWN_TYPE, "UNKNOWN"); TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.VARCHAR_TYPE, "VARCHAR"); } diff --git a/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TTypeId.java b/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TTypeId.java index 5039ce50b2ef..57108255c1c6 100644 --- a/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TTypeId.java +++ b/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TTypeId.java @@ -31,7 +31,8 @@ public enum TTypeId implements org.apache.thrift.TEnum { CHAR_TYPE(19), INTERVAL_YEAR_MONTH_TYPE(20), INTERVAL_DAY_TIME_TYPE(21), - TIMESTAMPLOCALTZ_TYPE(22); + TIMESTAMPLOCALTZ_TYPE(22), + UNKNOWN_TYPE(23); private final int value; @@ -99,6 +100,8 @@ public static TTypeId findByValue(int value) { return INTERVAL_DAY_TIME_TYPE; case 22: return TIMESTAMPLOCALTZ_TYPE; + case 23: + return UNKNOWN_TYPE; default: return null; } diff --git a/service-rpc/src/gen/thrift/gen-php/TTypeId.php b/service-rpc/src/gen/thrift/gen-php/TTypeId.php index 43f773d2e1e0..ac799c4a173d 100644 --- a/service-rpc/src/gen/thrift/gen-php/TTypeId.php +++ b/service-rpc/src/gen/thrift/gen-php/TTypeId.php @@ -62,6 +62,8 @@ final class TTypeId const TIMESTAMPLOCALTZ_TYPE = 22; + const UNKNOWN_TYPE = 23; + static public $__names = array( 0 => 'BOOLEAN_TYPE', 1 => 'TINYINT_TYPE', @@ -86,6 +88,7 @@ final class TTypeId 20 => 'INTERVAL_YEAR_MONTH_TYPE', 21 => 'INTERVAL_DAY_TIME_TYPE', 22 => 'TIMESTAMPLOCALTZ_TYPE', + 23 => 'UNKNOWN_TYPE', ); } diff --git a/service/src/java/org/apache/hive/service/cli/ColumnValue.java b/service/src/java/org/apache/hive/service/cli/ColumnValue.java index 09ca12787e8e..b994d6fc846b 100644 --- a/service/src/java/org/apache/hive/service/cli/ColumnValue.java +++ b/service/src/java/org/apache/hive/service/cli/ColumnValue.java @@ -219,6 +219,8 @@ public static TColumnValue toTColumnValue(TypeDescriptor typeDescriptor, Object return stringValue((String)value); case NULL_TYPE: return stringValue((String)value); + case UNKNOWN_TYPE: + return stringValue((String)value); default: return null; } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java index 9db912182425..511cfc050b7f 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java @@ -83,6 +83,7 @@ public class ColumnType { public static final String UNION_TYPE_NAME = "uniontype"; public static final String VARIANT_TYPE_NAME = "variant"; + public static final String UNKNOWN_TYPE_NAME = "unknown"; public static final String LIST_COLUMNS = "columns"; @@ -200,6 +201,7 @@ public class ColumnType { STRUCT_TYPE_NAME, UNION_TYPE_NAME, VARIANT_TYPE_NAME, + UNKNOWN_TYPE_NAME, LIST_COLUMNS, LIST_COLUMN_TYPES, COLUMN_NAME_DELIMITER From 7cd06a6e4a99a382c9da41b853e5c5b6a7e421e0 Mon Sep 17 00:00:00 2001 From: Dmitriy Fingerman Date: Fri, 10 Jul 2026 12:41:42 -0400 Subject: [PATCH 2/2] Code review comments. --- .../src/test/queries/positive/unknown_type.q | 16 ++- .../test/results/positive/unknown_type.q.out | 115 ++++++++++++++++-- 2 files changed, 122 insertions(+), 9 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/queries/positive/unknown_type.q b/iceberg/iceberg-handler/src/test/queries/positive/unknown_type.q index 60810519740b..60c9edad9836 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/unknown_type.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/unknown_type.q @@ -37,4 +37,18 @@ INSERT INTO unknown_test_basic VALUES SELECT id, placeholder, extra FROM unknown_test_basic ORDER BY id; -DESC FORMATTED unknown_test_basic; +-- CTLT preserves unknown column types from the source table +CREATE TABLE unknown_test_ctlt LIKE unknown_test_basic STORED BY ICEBERG tblproperties('format-version'='3'); + +DESC FORMATTED unknown_test_ctlt; + +INSERT INTO unknown_test_ctlt SELECT * FROM unknown_test_basic; + +SELECT id, placeholder, extra FROM unknown_test_ctlt ORDER BY id; + +-- CTAS preserves unknown column types from the query schema +CREATE TABLE unknown_test_ctas STORED BY ICEBERG tblproperties('format-version'='3') AS SELECT * FROM unknown_test_basic; + +DESC FORMATTED unknown_test_ctas; + +SELECT id, placeholder, extra FROM unknown_test_ctas ORDER BY id; diff --git a/iceberg/iceberg-handler/src/test/results/positive/unknown_type.q.out b/iceberg/iceberg-handler/src/test/results/positive/unknown_type.q.out index d5f0566abfbc..faa92ff48360 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/unknown_type.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/unknown_type.q.out @@ -63,12 +63,101 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 1 NULL NULL 2 NULL NULL 3 NULL NULL -PREHOOK: query: DESC FORMATTED unknown_test_basic +PREHOOK: query: CREATE TABLE unknown_test_ctlt LIKE unknown_test_basic STORED BY ICEBERG tblproperties('format-version'='3') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@unknown_test_ctlt +POSTHOOK: query: CREATE TABLE unknown_test_ctlt LIKE unknown_test_basic STORED BY ICEBERG tblproperties('format-version'='3') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@unknown_test_ctlt +PREHOOK: query: DESC FORMATTED unknown_test_ctlt PREHOOK: type: DESCTABLE -PREHOOK: Input: default@unknown_test_basic -POSTHOOK: query: DESC FORMATTED unknown_test_basic +PREHOOK: Input: default@unknown_test_ctlt +POSTHOOK: query: DESC FORMATTED unknown_test_ctlt POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@unknown_test_ctlt +# col_name data_type comment +id int +placeholder unknown +extra unknown + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"extra\":\"true\",\"id\":\"true\",\"placeholder\":\"true\"}} + EXTERNAL TRUE + TRANSLATED_TO_EXTERNAL TRUE + bucketing_version 2 + created_with_ctlt true + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"placeholder\",\"required\":false,\"type\":\"unknown\"},{\"id\":3,\"name\":\"extra\",\"required\":false,\"type\":\"unknown\"}]} + format-version 3 + metadata_location hdfs://### HDFS PATH ### + numFiles 0 + numRows 0 + parquet.compression zstd + rawDataSize 0 + snapshot-count 0 + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG + totalSize #Masked# +#### A masked pattern was here #### + uuid #Masked# + write.delete.mode merge-on-read + write.merge.mode merge-on-read + write.metadata.delete-after-commit.enabled true + write.update.mode merge-on-read + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [] +PREHOOK: query: INSERT INTO unknown_test_ctlt SELECT * FROM unknown_test_basic +PREHOOK: type: QUERY +PREHOOK: Input: default@unknown_test_basic +PREHOOK: Output: default@unknown_test_ctlt +POSTHOOK: query: INSERT INTO unknown_test_ctlt SELECT * FROM unknown_test_basic +POSTHOOK: type: QUERY +POSTHOOK: Input: default@unknown_test_basic +POSTHOOK: Output: default@unknown_test_ctlt +PREHOOK: query: SELECT id, placeholder, extra FROM unknown_test_ctlt ORDER BY id +PREHOOK: type: QUERY +PREHOOK: Input: default@unknown_test_ctlt +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT id, placeholder, extra FROM unknown_test_ctlt ORDER BY id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@unknown_test_ctlt +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 NULL NULL +2 NULL NULL +3 NULL NULL +PREHOOK: query: CREATE TABLE unknown_test_ctas STORED BY ICEBERG tblproperties('format-version'='3') AS SELECT * FROM unknown_test_basic +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@unknown_test_basic +PREHOOK: Output: database:default +PREHOOK: Output: default@unknown_test_ctas +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: CREATE TABLE unknown_test_ctas STORED BY ICEBERG tblproperties('format-version'='3') AS SELECT * FROM unknown_test_basic +POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@unknown_test_basic +POSTHOOK: Output: database:default +POSTHOOK: Output: default@unknown_test_ctas +POSTHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: Lineage: unknown_test_ctas.extra SIMPLE [(unknown_test_basic)unknown_test_basic.FieldSchema(name:extra, type:unknown, comment:null), ] +POSTHOOK: Lineage: unknown_test_ctas.id SIMPLE [(unknown_test_basic)unknown_test_basic.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: unknown_test_ctas.placeholder SIMPLE [(unknown_test_basic)unknown_test_basic.FieldSchema(name:placeholder, type:unknown, comment:null), ] +PREHOOK: query: DESC FORMATTED unknown_test_ctas +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@unknown_test_ctas +POSTHOOK: query: DESC FORMATTED unknown_test_ctas +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@unknown_test_ctas # col_name data_type comment id int placeholder unknown @@ -84,19 +173,18 @@ Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"placeholder\",\"required\":false,\"type\":\"unknown\"},{\"id\":3,\"name\":\"extra\",\"required\":false,\"type\":\"unknown\"}]} + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"placeholder\",\"required\":false,\"type\":\"unknown\"},{\"id\":3,\"name\":\"extra\",\"required\":false,\"type\":\"unknown\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"3\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"3\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"3\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 3 metadata_location hdfs://### HDFS PATH ### - numFiles 2 + numFiles 1 numRows 3 parquet.compression zstd previous_metadata_location hdfs://### HDFS PATH ### - rawDataSize 0 serialization.format 1 - snapshot-count 2 + snapshot-count 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -113,3 +201,14 @@ InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat Compressed: No Sort Columns: [] +PREHOOK: query: SELECT id, placeholder, extra FROM unknown_test_ctas ORDER BY id +PREHOOK: type: QUERY +PREHOOK: Input: default@unknown_test_ctas +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT id, placeholder, extra FROM unknown_test_ctas ORDER BY id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@unknown_test_ctas +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 NULL NULL +2 NULL NULL +3 NULL NULL