diff --git a/spark/src/test/resources/sql-tests/expressions/array/routing_arrays_disabled.sql b/spark/src/test/resources/sql-tests/expressions/array/routing_arrays_disabled.sql new file mode 100644 index 0000000000..ece58525d4 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/routing_arrays_disabled.sql @@ -0,0 +1,72 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false +-- Config: spark.comet.exec.strictFloatingPoint=true +-- Config: spark.comet.expression.ArrayContains.allowIncompatible=false +-- Config: spark.comet.expression.SortArray.allowIncompatible=false +-- Config: spark.comet.expression.ArrayExcept.allowIncompatible=false +-- Config: spark.comet.expression.ArrayJoin.allowIncompatible=false +-- Config: spark.comet.expression.Reverse.allowIncompatible=false + +statement +CREATE TABLE routing_arrays(a ARRAY, f ARRAY, x INT, y INT, words ARRAY, sep STRING) USING parquet + +statement +INSERT INTO routing_arrays VALUES (array(2, 1), array(2.0D, 1.0D), 1, 3, array('a', 'b'), '-'), (array(), array(), 3, 1, array(), ''), (NULL, NULL, NULL, NULL, NULL, NULL) + +query expect_native(array_contains) +SELECT array_contains(a, 1) FROM routing_arrays + +query expect_native(sort_array) +SELECT sort_array(a) FROM routing_arrays + +query expect_native(sequence) +SELECT sequence(x, y) FROM routing_arrays + +query expect_native(array_join) +SELECT array_join(words, '-') FROM routing_arrays + +query expect_native(reverse) +SELECT reverse(a) FROM routing_arrays + +query expect_fallback(array_contains: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT array_contains(f, 1.0D) FROM routing_arrays + +query expect_fallback(sort_array: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT sort_array(f) FROM routing_arrays + +query expect_fallback(sort_array: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT sort_array(array(make_dt_interval(x), make_dt_interval(y))) FROM routing_arrays + +query expect_fallback(array_except: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT sort_array(array_except(a, array(1))) FROM routing_arrays + +query expect_fallback(array_except: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT array_except(array(unhex('41')), array(unhex('42'))) + +query expect_fallback(reverse: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT reverse(array(unhex('41'), unhex('42'))) + +query expect_fallback(sequence: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT sequence(x, y + 1) FROM routing_arrays + +query expect_fallback(sequence: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT sequence(DATE '2024-01-01', DATE '2024-01-03') + +query expect_fallback(array_join: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT array_join(words, concat(sep, '')) FROM routing_arrays diff --git a/spark/src/test/resources/sql-tests/expressions/array/routing_arrays_enabled.sql b/spark/src/test/resources/sql-tests/expressions/array/routing_arrays_enabled.sql new file mode 100644 index 0000000000..03d4d3120a --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/routing_arrays_enabled.sql @@ -0,0 +1,72 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.comet.exec.strictFloatingPoint=true +-- Config: spark.comet.expression.ArrayContains.allowIncompatible=false +-- Config: spark.comet.expression.SortArray.allowIncompatible=false +-- Config: spark.comet.expression.ArrayExcept.allowIncompatible=false +-- Config: spark.comet.expression.ArrayJoin.allowIncompatible=false +-- Config: spark.comet.expression.Reverse.allowIncompatible=false + +statement +CREATE TABLE routing_arrays(a ARRAY, f ARRAY, x INT, y INT, words ARRAY, sep STRING) USING parquet + +statement +INSERT INTO routing_arrays VALUES (array(2, 1), array(2.0D, 1.0D), 1, 3, array('a', 'b'), '-'), (array(), array(), 3, 1, array(), ''), (NULL, NULL, NULL, NULL, NULL, NULL) + +query expect_native(array_contains) +SELECT array_contains(a, 1) FROM routing_arrays + +query expect_native(sort_array) +SELECT sort_array(a) FROM routing_arrays + +query expect_native(sequence) +SELECT sequence(x, y) FROM routing_arrays + +query expect_native(array_join) +SELECT array_join(words, '-') FROM routing_arrays + +query expect_native(reverse) +SELECT reverse(a) FROM routing_arrays + +query expect_dispatch(array_contains) +SELECT array_contains(f, 1.0D) FROM routing_arrays + +query expect_dispatch(sort_array) +SELECT sort_array(f) FROM routing_arrays + +query expect_dispatch(sort_array) +SELECT sort_array(array(make_dt_interval(x), make_dt_interval(y))) FROM routing_arrays + +query expect_dispatch(array_except) +SELECT sort_array(array_except(a, array(1))) FROM routing_arrays + +query expect_dispatch(array_except) +SELECT array_except(array(unhex('41')), array(unhex('42'))) + +query expect_dispatch(reverse) +SELECT reverse(array(unhex('41'), unhex('42'))) + +query expect_dispatch(sequence) +SELECT sequence(x, y + 1) FROM routing_arrays + +query expect_dispatch(sequence) +SELECT sequence(DATE '2024-01-01', DATE '2024-01-03') + +query expect_dispatch(array_join) +SELECT array_join(words, concat(sep, '')) FROM routing_arrays diff --git a/spark/src/test/resources/sql-tests/expressions/array/routing_arrays_opt_in.sql b/spark/src/test/resources/sql-tests/expressions/array/routing_arrays_opt_in.sql new file mode 100644 index 0000000000..64038315a8 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/routing_arrays_opt_in.sql @@ -0,0 +1,72 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false +-- Config: spark.comet.exec.strictFloatingPoint=true +-- Config: spark.comet.expression.ArrayContains.allowIncompatible=true +-- Config: spark.comet.expression.SortArray.allowIncompatible=true +-- Config: spark.comet.expression.ArrayExcept.allowIncompatible=true +-- Config: spark.comet.expression.ArrayJoin.allowIncompatible=true +-- Config: spark.comet.expression.Reverse.allowIncompatible=true + +statement +CREATE TABLE routing_arrays(a ARRAY, f ARRAY, x INT, y INT, words ARRAY, sep STRING) USING parquet + +statement +INSERT INTO routing_arrays VALUES (array(2, 1), array(2.0D, 1.0D), 1, 3, array('a', 'b'), '-'), (array(), array(), 3, 1, array(), ''), (NULL, NULL, NULL, NULL, NULL, NULL) + +query expect_native(array_contains) +SELECT array_contains(a, 1) FROM routing_arrays + +query expect_native(sort_array) +SELECT sort_array(a) FROM routing_arrays + +query expect_native(sequence) +SELECT sequence(x, y) FROM routing_arrays + +query expect_native(array_join) +SELECT array_join(words, '-') FROM routing_arrays + +query expect_native(reverse) +SELECT reverse(a) FROM routing_arrays + +query expect_native(array_contains) +SELECT array_contains(f, 1.0D) FROM routing_arrays + +query expect_native(sort_array) +SELECT sort_array(f) FROM routing_arrays + +query expect_fallback(sort_array: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT sort_array(array(make_dt_interval(x), make_dt_interval(y))) FROM routing_arrays + +query expect_native(array_except) +SELECT sort_array(array_except(a, array(1))) FROM routing_arrays + +query expect_fallback(data type not supported) +SELECT array_except(array(unhex('41')), array(unhex('42'))) + +query expect_fallback(child data type not supported) +SELECT reverse(array(unhex('41'), unhex('42'))) + +query expect_fallback(sequence: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT sequence(x, y + 1) FROM routing_arrays + +query expect_fallback(sequence: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT sequence(DATE '2024-01-01', DATE '2024-01-03') + +query expect_native(array_join) +SELECT array_join(words, concat(sep, '')) FROM routing_arrays diff --git a/spark/src/test/resources/sql-tests/expressions/map/routing_map_legacy_disabled.sql b/spark/src/test/resources/sql-tests/expressions/map/routing_map_legacy_disabled.sql new file mode 100644 index 0000000000..a390e18e09 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map/routing_map_legacy_disabled.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false +-- Config: spark.sql.legacy.truncateForEmptyRegexSplit=true +-- Config: spark.sql.mapKeyDedupPolicy=LAST_WIN +-- Config: spark.comet.expression.StringToMap.allowIncompatible=false +-- Config: spark.comet.expression.MapFromEntries.allowIncompatible=false + +statement +CREATE TABLE routing_map_legacy(s STRING, e ARRAY>) USING parquet + +statement +INSERT INTO routing_map_legacy VALUES ('a:1,b:2', array(named_struct('key', 'a', 'value', 1))), (NULL, NULL) + +query expect_fallback(str_to_map: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT str_to_map(s) FROM routing_map_legacy + +query expect_fallback(map_from_entries: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT map_from_entries(e) FROM routing_map_legacy diff --git a/spark/src/test/resources/sql-tests/expressions/map/routing_map_legacy_enabled.sql b/spark/src/test/resources/sql-tests/expressions/map/routing_map_legacy_enabled.sql new file mode 100644 index 0000000000..afbfc95dba --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map/routing_map_legacy_enabled.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.sql.legacy.truncateForEmptyRegexSplit=true +-- Config: spark.sql.mapKeyDedupPolicy=LAST_WIN +-- Config: spark.comet.expression.StringToMap.allowIncompatible=false +-- Config: spark.comet.expression.MapFromEntries.allowIncompatible=false + +statement +CREATE TABLE routing_map_legacy(s STRING, e ARRAY>) USING parquet + +statement +INSERT INTO routing_map_legacy VALUES ('a:1,b:2', array(named_struct('key', 'a', 'value', 1))), (NULL, NULL) + +query expect_dispatch(str_to_map) +SELECT str_to_map(s) FROM routing_map_legacy + +query expect_dispatch(map_from_entries) +SELECT map_from_entries(e) FROM routing_map_legacy diff --git a/spark/src/test/resources/sql-tests/expressions/map/routing_map_legacy_opt_in.sql b/spark/src/test/resources/sql-tests/expressions/map/routing_map_legacy_opt_in.sql new file mode 100644 index 0000000000..91b6ecf1bb --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map/routing_map_legacy_opt_in.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false +-- Config: spark.sql.legacy.truncateForEmptyRegexSplit=true +-- Config: spark.sql.mapKeyDedupPolicy=LAST_WIN +-- Config: spark.comet.expression.StringToMap.allowIncompatible=true +-- Config: spark.comet.expression.MapFromEntries.allowIncompatible=true + +statement +CREATE TABLE routing_map_legacy(s STRING, e ARRAY>) USING parquet + +statement +INSERT INTO routing_map_legacy VALUES ('a:1,b:2', array(named_struct('key', 'a', 'value', 1))), (NULL, NULL) + +query expect_native(str_to_map) +SELECT str_to_map(s) FROM routing_map_legacy + +query expect_native(map_from_entries) +SELECT map_from_entries(e) FROM routing_map_legacy diff --git a/spark/src/test/resources/sql-tests/expressions/map/routing_maps_disabled.sql b/spark/src/test/resources/sql-tests/expressions/map/routing_maps_disabled.sql new file mode 100644 index 0000000000..5a60445012 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map/routing_maps_disabled.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false +-- Config: spark.comet.expression.MapFromEntries.allowIncompatible=false + +statement +CREATE TABLE routing_maps(s STRING, entries ARRAY>, binary_entries ARRAY>) USING parquet + +statement +INSERT INTO routing_maps VALUES ('a:1,b:2', array(named_struct('key', 'a', 'value', 1)), array(named_struct('key', unhex('41'), 'value', 1))), ('', array(), array()), (NULL, NULL, NULL) + +query expect_native(str_to_map) +SELECT str_to_map(s) FROM routing_maps + +query expect_native(map_from_entries) +SELECT map_from_entries(entries) FROM routing_maps + +query expect_fallback(map_from_entries: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT map_from_entries(binary_entries) FROM routing_maps diff --git a/spark/src/test/resources/sql-tests/expressions/map/routing_maps_enabled.sql b/spark/src/test/resources/sql-tests/expressions/map/routing_maps_enabled.sql new file mode 100644 index 0000000000..e5a09d72a2 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map/routing_maps_enabled.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.comet.expression.MapFromEntries.allowIncompatible=false + +statement +CREATE TABLE routing_maps(s STRING, entries ARRAY>, binary_entries ARRAY>) USING parquet + +statement +INSERT INTO routing_maps VALUES ('a:1,b:2', array(named_struct('key', 'a', 'value', 1)), array(named_struct('key', unhex('41'), 'value', 1))), ('', array(), array()), (NULL, NULL, NULL) + +query expect_native(str_to_map) +SELECT str_to_map(s) FROM routing_maps + +query expect_native(map_from_entries) +SELECT map_from_entries(entries) FROM routing_maps + +query expect_dispatch(map_from_entries) +SELECT map_from_entries(binary_entries) FROM routing_maps diff --git a/spark/src/test/resources/sql-tests/expressions/map/routing_maps_opt_in.sql b/spark/src/test/resources/sql-tests/expressions/map/routing_maps_opt_in.sql new file mode 100644 index 0000000000..eb27ca74c4 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map/routing_maps_opt_in.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false +-- Config: spark.comet.expression.MapFromEntries.allowIncompatible=true + +statement +CREATE TABLE routing_maps(s STRING, entries ARRAY>, binary_entries ARRAY>) USING parquet + +statement +INSERT INTO routing_maps VALUES ('a:1,b:2', array(named_struct('key', 'a', 'value', 1)), array(named_struct('key', unhex('41'), 'value', 1))), ('', array(), array()), (NULL, NULL, NULL) + +query expect_native(str_to_map) +SELECT str_to_map(s) FROM routing_maps + +query expect_native(map_from_entries) +SELECT map_from_entries(entries) FROM routing_maps + +query expect_native(map_from_entries) +SELECT map_from_entries(binary_entries) FROM routing_maps