Skip to content

HIVE-29557: Parquet Vectorization reads NULL values in Arrays and Map as Default Values #6419

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ayushtkn wants to merge 1 commit into
base: master
Choose a base branch
from
Open

HIVE-29557: Parquet Vectorization reads NULL values in Arrays and Map as Default Values #6419

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedListColumnReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ private void fillColumnVector(PrimitiveObjectInspector.PrimitiveCategory categor
for (int i = 0; i < valueList.size(); i++) {
if (valueList.get(i) == null) {
lcv.child.isNull[i] = true;
lcv.child.noNulls = false;
} else {
((LongColumnVector) lcv.child).vector[i] = ((List<Integer>) valueList).get(i);
}
Expand All @@ -287,6 +288,7 @@ private void fillColumnVector(PrimitiveObjectInspector.PrimitiveCategory categor
for (int i = 0; i < valueList.size(); i++) {
if (valueList.get(i) == null) {
lcv.child.isNull[i] = true;
lcv.child.noNulls = false;
} else {
((LongColumnVector) lcv.child).vector[i] = ((List<Long>) valueList).get(i);
}
Expand All @@ -297,6 +299,7 @@ private void fillColumnVector(PrimitiveObjectInspector.PrimitiveCategory categor
for (int i = 0; i < valueList.size(); i++) {
if (valueList.get(i) == null) {
lcv.child.isNull[i] = true;
lcv.child.noNulls = false;
} else {
((DoubleColumnVector) lcv.child).vector[i] = ((List<Double>) valueList).get(i);
}
Expand All @@ -313,6 +316,7 @@ private void fillColumnVector(PrimitiveObjectInspector.PrimitiveCategory categor
if (src == null) {
((BytesColumnVector) lcv.child).setRef(i, src, 0, 0);
lcv.child.isNull[i] = true;
lcv.child.noNulls = false;
} else {
((BytesColumnVector) lcv.child).setRef(i, src, 0, src.length);
}
Expand All @@ -323,6 +327,7 @@ private void fillColumnVector(PrimitiveObjectInspector.PrimitiveCategory categor
for (int i = 0; i < valueList.size(); i++) {
if (valueList.get(i) == null) {
lcv.child.isNull[i] = true;
lcv.child.noNulls = false;
} else {
((DoubleColumnVector) lcv.child).vector[i] = ((List<Float>) valueList).get(i);
}
Expand All @@ -337,6 +342,7 @@ private void fillColumnVector(PrimitiveObjectInspector.PrimitiveCategory categor
for (int i = 0; i < valueList.size(); i++) {
if (valueList.get(i) == null) {
lcv.child.isNull[i] = true;
lcv.child.noNulls = false;
} else {
((DecimalColumnVector) lcv.child).vector[i].set(((List<byte[]>) valueList).get(i), scale);
}
Expand Down
1 change: 1 addition & 0 deletions ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedMapColumnReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,5 +65,6 @@ public void readBatch(int total, ColumnVector column, TypeInfo columnType) throw
mapColumnVector.childCount = keyListColumnVector.childCount;
mapColumnVector.isRepeating = keyListColumnVector.isRepeating
&& valueListColumnVector.isRepeating;
mapColumnVector.noNulls = keyListColumnVector.noNulls && valueListColumnVector.noNulls;
}
}
67 changes: 67 additions & 0 deletions ql/src/test/queries/clientpositive/parquet_array_with_null_vectorization.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
-- SORT_QUERY_RESULTS
SET hive.vectorized.execution.enabled=true;
set hive.vectorized.execution.reduce.enabled=true;
SET hive.fetch.task.conversion=none;

CREATE TABLE test_parquet_array_nulls_bool (
id INT,
arr_prim ARRAY<BOOLEAN>
) STORED AS PARQUET;

INSERT INTO test_parquet_array_nulls_bool VALUES
(1, array(CAST(NULL AS BOOLEAN), CAST(NULL AS BOOLEAN))),
(2, if(1=0, array(true, false), null)),
(3, array(true, CAST(NULL AS BOOLEAN))),
(4, array(true, false));

SELECT * FROM test_parquet_array_nulls_bool;

CREATE TABLE test_parquet_array_nulls_double (
id INT,
arr_prim ARRAY<DOUBLE>
) STORED AS PARQUET;

INSERT INTO test_parquet_array_nulls_double
SELECT 1, array(CAST(NULL AS DOUBLE), CAST(NULL AS DOUBLE))
UNION ALL
SELECT 2, CAST(NULL AS ARRAY<DOUBLE>)
UNION ALL
SELECT 3, array(CAST(3.3 AS DOUBLE), CAST(NULL AS DOUBLE))
UNION ALL
SELECT 4, array(CAST(4.4 AS DOUBLE), CAST(5.5 AS DOUBLE));

SELECT * FROM test_parquet_array_nulls_double;

CREATE TABLE test_parquet_array_nulls_varchar (
id INT,
arr_prim ARRAY<VARCHAR(20)>
) STORED AS PARQUET;

INSERT INTO test_parquet_array_nulls_varchar
SELECT 1, array(CAST(NULL AS VARCHAR(20)), CAST(NULL AS VARCHAR(20)));
INSERT INTO test_parquet_array_nulls_varchar
SELECT 2, CAST(NULL AS ARRAY<VARCHAR(20)>);

SELECT * FROM test_parquet_array_nulls_varchar;

CREATE TABLE test_parquet_array_nulls_float (
id INT,
arr_prim ARRAY<FLOAT>
) STORED AS PARQUET;

INSERT INTO test_parquet_array_nulls_float
SELECT 1, array(CAST(NULL AS FLOAT), CAST(NULL AS FLOAT))
UNION ALL
SELECT 2, CAST(NULL AS ARRAY<FLOAT>)
UNION ALL
SELECT 3, array(CAST(3.3 AS FLOAT), CAST(NULL AS FLOAT))
UNION ALL
SELECT 4, array(CAST(4.4 AS FLOAT), CAST(5.5 AS FLOAT));

SELECT * FROM test_parquet_array_nulls_float;

SET hive.vectorized.execution.enabled=false;
SELECT * FROM test_parquet_array_nulls_bool;
SELECT * FROM test_parquet_array_nulls_double;
SELECT * FROM test_parquet_array_nulls_varchar;
SELECT * FROM test_parquet_array_nulls_float;
238 changes: 238 additions & 0 deletions ql/src/test/results/clientpositive/llap/parquet_array_with_null_vectorization.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,238 @@
PREHOOK: query: CREATE TABLE test_parquet_array_nulls_bool (
id INT,
arr_prim ARRAY<BOOLEAN>
) STORED AS PARQUET
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@test_parquet_array_nulls_bool
POSTHOOK: query: CREATE TABLE test_parquet_array_nulls_bool (
id INT,
arr_prim ARRAY<BOOLEAN>
) STORED AS PARQUET
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@test_parquet_array_nulls_bool
PREHOOK: query: INSERT INTO test_parquet_array_nulls_bool VALUES
(1, array(CAST(NULL AS BOOLEAN), CAST(NULL AS BOOLEAN))),
(2, if(1=0, array(true, false), null)),
(3, array(true, CAST(NULL AS BOOLEAN))),
(4, array(true, false))
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@test_parquet_array_nulls_bool
POSTHOOK: query: INSERT INTO test_parquet_array_nulls_bool VALUES
(1, array(CAST(NULL AS BOOLEAN), CAST(NULL AS BOOLEAN))),
(2, if(1=0, array(true, false), null)),
(3, array(true, CAST(NULL AS BOOLEAN))),
(4, array(true, false))
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@test_parquet_array_nulls_bool
POSTHOOK: Lineage: test_parquet_array_nulls_bool.arr_prim SCRIPT []
POSTHOOK: Lineage: test_parquet_array_nulls_bool.id SCRIPT []
PREHOOK: query: SELECT * FROM test_parquet_array_nulls_bool
PREHOOK: type: QUERY
PREHOOK: Input: default@test_parquet_array_nulls_bool
#### A masked pattern was here ####
POSTHOOK: query: SELECT * FROM test_parquet_array_nulls_bool
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_parquet_array_nulls_bool
#### A masked pattern was here ####
1 [null,null]
2 NULL
3 [true,null]
4 [true,false]
PREHOOK: query: CREATE TABLE test_parquet_array_nulls_double (
id INT,
arr_prim ARRAY<DOUBLE>
) STORED AS PARQUET
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@test_parquet_array_nulls_double
POSTHOOK: query: CREATE TABLE test_parquet_array_nulls_double (
id INT,
arr_prim ARRAY<DOUBLE>
) STORED AS PARQUET
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@test_parquet_array_nulls_double
PREHOOK: query: INSERT INTO test_parquet_array_nulls_double
SELECT 1, array(CAST(NULL AS DOUBLE), CAST(NULL AS DOUBLE))
UNION ALL
SELECT 2, CAST(NULL AS ARRAY<DOUBLE>)
UNION ALL
SELECT 3, array(CAST(3.3 AS DOUBLE), CAST(NULL AS DOUBLE))
UNION ALL
SELECT 4, array(CAST(4.4 AS DOUBLE), CAST(5.5 AS DOUBLE))
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@test_parquet_array_nulls_double
POSTHOOK: query: INSERT INTO test_parquet_array_nulls_double
SELECT 1, array(CAST(NULL AS DOUBLE), CAST(NULL AS DOUBLE))
UNION ALL
SELECT 2, CAST(NULL AS ARRAY<DOUBLE>)
UNION ALL
SELECT 3, array(CAST(3.3 AS DOUBLE), CAST(NULL AS DOUBLE))
UNION ALL
SELECT 4, array(CAST(4.4 AS DOUBLE), CAST(5.5 AS DOUBLE))
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@test_parquet_array_nulls_double
POSTHOOK: Lineage: test_parquet_array_nulls_double.arr_prim SCRIPT []
POSTHOOK: Lineage: test_parquet_array_nulls_double.id SCRIPT []
PREHOOK: query: SELECT * FROM test_parquet_array_nulls_double
PREHOOK: type: QUERY
PREHOOK: Input: default@test_parquet_array_nulls_double
#### A masked pattern was here ####
POSTHOOK: query: SELECT * FROM test_parquet_array_nulls_double
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_parquet_array_nulls_double
#### A masked pattern was here ####
1 [null,null]
2 NULL
3 [3.3,null]
4 [4.4,5.5]
PREHOOK: query: CREATE TABLE test_parquet_array_nulls_varchar (
id INT,
arr_prim ARRAY<VARCHAR(20)>
) STORED AS PARQUET
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@test_parquet_array_nulls_varchar
POSTHOOK: query: CREATE TABLE test_parquet_array_nulls_varchar (
id INT,
arr_prim ARRAY<VARCHAR(20)>
) STORED AS PARQUET
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@test_parquet_array_nulls_varchar
PREHOOK: query: INSERT INTO test_parquet_array_nulls_varchar
SELECT 1, array(CAST(NULL AS VARCHAR(20)), CAST(NULL AS VARCHAR(20)))
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@test_parquet_array_nulls_varchar
POSTHOOK: query: INSERT INTO test_parquet_array_nulls_varchar
SELECT 1, array(CAST(NULL AS VARCHAR(20)), CAST(NULL AS VARCHAR(20)))
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@test_parquet_array_nulls_varchar
POSTHOOK: Lineage: test_parquet_array_nulls_varchar.arr_prim EXPRESSION []
POSTHOOK: Lineage: test_parquet_array_nulls_varchar.id SIMPLE []
PREHOOK: query: INSERT INTO test_parquet_array_nulls_varchar
SELECT 2, CAST(NULL AS ARRAY<VARCHAR(20)>)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@test_parquet_array_nulls_varchar
POSTHOOK: query: INSERT INTO test_parquet_array_nulls_varchar
SELECT 2, CAST(NULL AS ARRAY<VARCHAR(20)>)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@test_parquet_array_nulls_varchar
POSTHOOK: Lineage: test_parquet_array_nulls_varchar.arr_prim SIMPLE []
POSTHOOK: Lineage: test_parquet_array_nulls_varchar.id SIMPLE []
PREHOOK: query: SELECT * FROM test_parquet_array_nulls_varchar
PREHOOK: type: QUERY
PREHOOK: Input: default@test_parquet_array_nulls_varchar
#### A masked pattern was here ####
POSTHOOK: query: SELECT * FROM test_parquet_array_nulls_varchar
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_parquet_array_nulls_varchar
#### A masked pattern was here ####
1 [null,null]
2 NULL
PREHOOK: query: CREATE TABLE test_parquet_array_nulls_float (
id INT,
arr_prim ARRAY<FLOAT>
) STORED AS PARQUET
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@test_parquet_array_nulls_float
POSTHOOK: query: CREATE TABLE test_parquet_array_nulls_float (
id INT,
arr_prim ARRAY<FLOAT>
) STORED AS PARQUET
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@test_parquet_array_nulls_float
PREHOOK: query: INSERT INTO test_parquet_array_nulls_float
SELECT 1, array(CAST(NULL AS FLOAT), CAST(NULL AS FLOAT))
UNION ALL
SELECT 2, CAST(NULL AS ARRAY<FLOAT>)
UNION ALL
SELECT 3, array(CAST(3.3 AS FLOAT), CAST(NULL AS FLOAT))
UNION ALL
SELECT 4, array(CAST(4.4 AS FLOAT), CAST(5.5 AS FLOAT))
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@test_parquet_array_nulls_float
POSTHOOK: query: INSERT INTO test_parquet_array_nulls_float
SELECT 1, array(CAST(NULL AS FLOAT), CAST(NULL AS FLOAT))
UNION ALL
SELECT 2, CAST(NULL AS ARRAY<FLOAT>)
UNION ALL
SELECT 3, array(CAST(3.3 AS FLOAT), CAST(NULL AS FLOAT))
UNION ALL
SELECT 4, array(CAST(4.4 AS FLOAT), CAST(5.5 AS FLOAT))
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@test_parquet_array_nulls_float
POSTHOOK: Lineage: test_parquet_array_nulls_float.arr_prim SCRIPT []
POSTHOOK: Lineage: test_parquet_array_nulls_float.id SCRIPT []
PREHOOK: query: SELECT * FROM test_parquet_array_nulls_float
PREHOOK: type: QUERY
PREHOOK: Input: default@test_parquet_array_nulls_float
#### A masked pattern was here ####
POSTHOOK: query: SELECT * FROM test_parquet_array_nulls_float
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_parquet_array_nulls_float
#### A masked pattern was here ####
1 [null,null]
2 NULL
3 [3.3,null]
4 [4.4,5.5]
PREHOOK: query: SELECT * FROM test_parquet_array_nulls_bool
PREHOOK: type: QUERY
PREHOOK: Input: default@test_parquet_array_nulls_bool
#### A masked pattern was here ####
POSTHOOK: query: SELECT * FROM test_parquet_array_nulls_bool
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_parquet_array_nulls_bool
#### A masked pattern was here ####
1 [null,null]
2 NULL
3 [true,null]
4 [true,false]
PREHOOK: query: SELECT * FROM test_parquet_array_nulls_double
PREHOOK: type: QUERY
PREHOOK: Input: default@test_parquet_array_nulls_double
#### A masked pattern was here ####
POSTHOOK: query: SELECT * FROM test_parquet_array_nulls_double
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_parquet_array_nulls_double
#### A masked pattern was here ####
1 [null,null]
2 NULL
3 [3.3,null]
4 [4.4,5.5]
PREHOOK: query: SELECT * FROM test_parquet_array_nulls_varchar
PREHOOK: type: QUERY
PREHOOK: Input: default@test_parquet_array_nulls_varchar
#### A masked pattern was here ####
POSTHOOK: query: SELECT * FROM test_parquet_array_nulls_varchar
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_parquet_array_nulls_varchar
#### A masked pattern was here ####
1 [null,null]
2 NULL
PREHOOK: query: SELECT * FROM test_parquet_array_nulls_float
PREHOOK: type: QUERY
PREHOOK: Input: default@test_parquet_array_nulls_float
#### A masked pattern was here ####
POSTHOOK: query: SELECT * FROM test_parquet_array_nulls_float
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_parquet_array_nulls_float
#### A masked pattern was here ####
1 [null,null]
2 NULL
3 [3.3,null]
4 [4.4,5.5]
Loading
Loading