Skip to content

test: add IN list slt coverage for temporal, Decimal128 and Interval types #23875

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
alamb wants to merge 1 commit into
base: main
Choose a base branch
from
+214 −0
Draft

test: add IN list slt coverage for temporal, Decimal128 and Interval types #23875

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
214 changes: 214 additions & 0 deletions datafusion/sqllogictest/test_files/in_list.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,23 @@ minus_one false false false false false false false false
one true false true false true false true false
zero false true false true false true false true

# Seventeen item IN list for the 1-byte types. Lists this long keep the bitmap
# specialization covered as shorter lists gain their own specializations.
query TBB
SELECT
label,
i8 IN (-128, -120, -100, -80, -60, -40, -20, -10, -5, -3, -2, 2, 3, 5, 20, 40, 11),
u8 IN (2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 16, 20, 0, 11, 255)
FROM in_list_ints
ORDER BY label
----
eleven true true
max false true
min true true
minus_one false false
one false false
zero false true

# Cleanup
statement ok
DROP TABLE in_list_ints;
Expand Down Expand Up @@ -329,6 +346,203 @@ Float64 match true true false
Float64 no_match false NULL false
Float64 nulls NULL NULL NULL

# Nine element Float16 IN list. Lists this long keep the bitmap specialization
# covered as shorter lists gain their own specializations.
query TB
SELECT
label,
f16 IN (arrow_cast(1.0, 'Float16'), arrow_cast(2.0, 'Float16'), arrow_cast(3.0, 'Float16'),
arrow_cast(4.0, 'Float16'), arrow_cast(5.0, 'Float16'), arrow_cast(6.0, 'Float16'),
arrow_cast(8.0, 'Float16'), arrow_cast(9.0, 'Float16'), arrow_cast(11.0, 'Float16'))
FROM in_list_floats
ORDER BY label
----
match true
no_match false
nulls NULL

# Cleanup
statement ok
DROP TABLE in_list_floats

####
## Temporal IN List Specializations
####

# Note `arrow_cast(1, 'Time32(Second)')` is not supported (the literal is Int64),
# so temporal values are built via an explicit Int32/Int64 cast first.
statement ok
CREATE TABLE in_list_temporal AS
SELECT
label,
arrow_cast(arrow_cast(value, 'Int32'), 'Date32') AS d32,
arrow_cast(arrow_cast(value, 'Int64'), 'Date64') AS d64,
arrow_cast(arrow_cast(value, 'Int32'), 'Time32(Second)') AS t32s,
arrow_cast(arrow_cast(value, 'Int64'), 'Time64(Nanosecond)') AS t64ns,
arrow_cast(arrow_cast(value, 'Int64'), 'Timestamp(Nanosecond, None)') AS ts_ns,
arrow_cast(arrow_cast(value, 'Int64'), 'Timestamp(Second, Some("UTC"))') AS ts_s_utc,
arrow_cast(arrow_cast(value, 'Int64'), 'Duration(Second)') AS dur_s
FROM (VALUES
('match', 11),
('no_match', 7),
('nulls', NULL)
) AS t(label, value);

# Verify the Arrow types of the columns, since the specializations are chosen by type.
query TTTTTTT
SELECT
arrow_typeof(d32),
arrow_typeof(d64),
arrow_typeof(t32s),
arrow_typeof(t64ns),
arrow_typeof(ts_ns),
arrow_typeof(ts_s_utc),
arrow_typeof(dur_s)
FROM in_list_temporal
LIMIT 1
----
Date32 Date64 Time32(s) Time64(ns) Timestamp(ns) Timestamp(s, "UTC") Duration(s)

# Four element IN lists cover the temporal specializations.
query TBBBBBBB
SELECT
label,
d32 IN (arrow_cast(arrow_cast(3, 'Int32'), 'Date32'), arrow_cast(arrow_cast(4, 'Int32'), 'Date32'), arrow_cast(arrow_cast(5, 'Int32'), 'Date32'), arrow_cast(arrow_cast(11, 'Int32'), 'Date32')),
d64 IN (arrow_cast(arrow_cast(3, 'Int64'), 'Date64'), arrow_cast(arrow_cast(4, 'Int64'), 'Date64'), arrow_cast(arrow_cast(5, 'Int64'), 'Date64'), arrow_cast(arrow_cast(11, 'Int64'), 'Date64')),
t32s IN (arrow_cast(arrow_cast(3, 'Int32'), 'Time32(Second)'), arrow_cast(arrow_cast(4, 'Int32'), 'Time32(Second)'), arrow_cast(arrow_cast(5, 'Int32'), 'Time32(Second)'), arrow_cast(arrow_cast(11, 'Int32'), 'Time32(Second)')),
t64ns IN (arrow_cast(arrow_cast(3, 'Int64'), 'Time64(Nanosecond)'), arrow_cast(arrow_cast(4, 'Int64'), 'Time64(Nanosecond)'), arrow_cast(arrow_cast(5, 'Int64'), 'Time64(Nanosecond)'), arrow_cast(arrow_cast(11, 'Int64'), 'Time64(Nanosecond)')),
ts_ns IN (arrow_cast(arrow_cast(3, 'Int64'), 'Timestamp(Nanosecond, None)'), arrow_cast(arrow_cast(4, 'Int64'), 'Timestamp(Nanosecond, None)'), arrow_cast(arrow_cast(5, 'Int64'), 'Timestamp(Nanosecond, None)'), arrow_cast(arrow_cast(11, 'Int64'), 'Timestamp(Nanosecond, None)')),
ts_s_utc IN (arrow_cast(arrow_cast(3, 'Int64'), 'Timestamp(Second, Some("UTC"))'), arrow_cast(arrow_cast(4, 'Int64'), 'Timestamp(Second, Some("UTC"))'), arrow_cast(arrow_cast(5, 'Int64'), 'Timestamp(Second, Some("UTC"))'), arrow_cast(arrow_cast(11, 'Int64'), 'Timestamp(Second, Some("UTC"))')),
dur_s IN (arrow_cast(arrow_cast(3, 'Int64'), 'Duration(Second)'), arrow_cast(arrow_cast(4, 'Int64'), 'Duration(Second)'), arrow_cast(arrow_cast(5, 'Int64'), 'Duration(Second)'), arrow_cast(arrow_cast(11, 'Int64'), 'Duration(Second)'))
FROM in_list_temporal
ORDER BY label
----
match true true true true true true true
no_match false false false false false false false
nulls NULL NULL NULL NULL NULL NULL NULL

# The same lists with NOT IN.
query TBBBBBBB
SELECT
label,
d32 NOT IN (arrow_cast(arrow_cast(3, 'Int32'), 'Date32'), arrow_cast(arrow_cast(4, 'Int32'), 'Date32'), arrow_cast(arrow_cast(5, 'Int32'), 'Date32'), arrow_cast(arrow_cast(11, 'Int32'), 'Date32')),
d64 NOT IN (arrow_cast(arrow_cast(3, 'Int64'), 'Date64'), arrow_cast(arrow_cast(4, 'Int64'), 'Date64'), arrow_cast(arrow_cast(5, 'Int64'), 'Date64'), arrow_cast(arrow_cast(11, 'Int64'), 'Date64')),
t32s NOT IN (arrow_cast(arrow_cast(3, 'Int32'), 'Time32(Second)'), arrow_cast(arrow_cast(4, 'Int32'), 'Time32(Second)'), arrow_cast(arrow_cast(5, 'Int32'), 'Time32(Second)'), arrow_cast(arrow_cast(11, 'Int32'), 'Time32(Second)')),
t64ns NOT IN (arrow_cast(arrow_cast(3, 'Int64'), 'Time64(Nanosecond)'), arrow_cast(arrow_cast(4, 'Int64'), 'Time64(Nanosecond)'), arrow_cast(arrow_cast(5, 'Int64'), 'Time64(Nanosecond)'), arrow_cast(arrow_cast(11, 'Int64'), 'Time64(Nanosecond)')),
ts_ns NOT IN (arrow_cast(arrow_cast(3, 'Int64'), 'Timestamp(Nanosecond, None)'), arrow_cast(arrow_cast(4, 'Int64'), 'Timestamp(Nanosecond, None)'), arrow_cast(arrow_cast(5, 'Int64'), 'Timestamp(Nanosecond, None)'), arrow_cast(arrow_cast(11, 'Int64'), 'Timestamp(Nanosecond, None)')),
ts_s_utc NOT IN (arrow_cast(arrow_cast(3, 'Int64'), 'Timestamp(Second, Some("UTC"))'), arrow_cast(arrow_cast(4, 'Int64'), 'Timestamp(Second, Some("UTC"))'), arrow_cast(arrow_cast(5, 'Int64'), 'Timestamp(Second, Some("UTC"))'), arrow_cast(arrow_cast(11, 'Int64'), 'Timestamp(Second, Some("UTC"))')),
dur_s NOT IN (arrow_cast(arrow_cast(3, 'Int64'), 'Duration(Second)'), arrow_cast(arrow_cast(4, 'Int64'), 'Duration(Second)'), arrow_cast(arrow_cast(5, 'Int64'), 'Duration(Second)'), arrow_cast(arrow_cast(11, 'Int64'), 'Duration(Second)'))
FROM in_list_temporal
ORDER BY label
----
match false false false false false false false
no_match true true true true true true true
nulls NULL NULL NULL NULL NULL NULL NULL

# Null IN list values return true for matches and NULL for non-matches.
query TBBBBBBB
SELECT
label,
d32 IN (NULL, arrow_cast(arrow_cast(3, 'Int32'), 'Date32'), arrow_cast(arrow_cast(4, 'Int32'), 'Date32'), arrow_cast(arrow_cast(11, 'Int32'), 'Date32')),
d64 IN (NULL, arrow_cast(arrow_cast(3, 'Int64'), 'Date64'), arrow_cast(arrow_cast(4, 'Int64'), 'Date64'), arrow_cast(arrow_cast(11, 'Int64'), 'Date64')),
t32s IN (NULL, arrow_cast(arrow_cast(3, 'Int32'), 'Time32(Second)'), arrow_cast(arrow_cast(4, 'Int32'), 'Time32(Second)'), arrow_cast(arrow_cast(11, 'Int32'), 'Time32(Second)')),
t64ns IN (NULL, arrow_cast(arrow_cast(3, 'Int64'), 'Time64(Nanosecond)'), arrow_cast(arrow_cast(4, 'Int64'), 'Time64(Nanosecond)'), arrow_cast(arrow_cast(11, 'Int64'), 'Time64(Nanosecond)')),
ts_ns IN (NULL, arrow_cast(arrow_cast(3, 'Int64'), 'Timestamp(Nanosecond, None)'), arrow_cast(arrow_cast(4, 'Int64'), 'Timestamp(Nanosecond, None)'), arrow_cast(arrow_cast(11, 'Int64'), 'Timestamp(Nanosecond, None)')),
ts_s_utc IN (NULL, arrow_cast(arrow_cast(3, 'Int64'), 'Timestamp(Second, Some("UTC"))'), arrow_cast(arrow_cast(4, 'Int64'), 'Timestamp(Second, Some("UTC"))'), arrow_cast(arrow_cast(11, 'Int64'), 'Timestamp(Second, Some("UTC"))')),
dur_s IN (NULL, arrow_cast(arrow_cast(3, 'Int64'), 'Duration(Second)'), arrow_cast(arrow_cast(4, 'Int64'), 'Duration(Second)'), arrow_cast(arrow_cast(11, 'Int64'), 'Duration(Second)'))
FROM in_list_temporal
ORDER BY label
----
match true true true true true true true
no_match NULL NULL NULL NULL NULL NULL NULL
nulls NULL NULL NULL NULL NULL NULL NULL

# A NULL in the list keeps short lists on the IN list path rather than rewriting
# them to an OR chain, which covers IN lists with a single non-null value.
query TBBBBBBB
SELECT
label,
d32 IN (NULL, arrow_cast(arrow_cast(11, 'Int32'), 'Date32')),
d64 IN (NULL, arrow_cast(arrow_cast(11, 'Int64'), 'Date64')),
t32s IN (NULL, arrow_cast(arrow_cast(11, 'Int32'), 'Time32(Second)')),
t64ns IN (NULL, arrow_cast(arrow_cast(11, 'Int64'), 'Time64(Nanosecond)')),
ts_ns IN (NULL, arrow_cast(arrow_cast(11, 'Int64'), 'Timestamp(Nanosecond, None)')),
ts_s_utc IN (NULL, arrow_cast(arrow_cast(11, 'Int64'), 'Timestamp(Second, Some("UTC"))')),
dur_s IN (NULL, arrow_cast(arrow_cast(11, 'Int64'), 'Duration(Second)'))
FROM in_list_temporal
ORDER BY label
----
match true true true true true true true
no_match NULL NULL NULL NULL NULL NULL NULL
nulls NULL NULL NULL NULL NULL NULL NULL

# Cleanup
statement ok
DROP TABLE in_list_temporal

####
## Decimal128 and Interval IN List Specializations
####

statement ok
CREATE TABLE in_list_wide AS
SELECT * FROM (VALUES
('match', arrow_cast(11, 'Decimal128(10, 2)'), INTERVAL '11 months'),
('no_match', arrow_cast(7, 'Decimal128(10, 2)'), INTERVAL '7 months'),
('nulls', NULL, NULL)
) AS t(label, d128, imdn);

query TT
SELECT arrow_typeof(d128), arrow_typeof(imdn) FROM in_list_wide LIMIT 1
----
Decimal128(10, 2) Interval(MonthDayNano)

# Four non-null values and five non-null values must agree. These sizes sit on
# either side of the 16-byte specialization threshold.
query TBBBB
SELECT
label,
d128 IN (arrow_cast(3, 'Decimal128(10, 2)'), arrow_cast(4, 'Decimal128(10, 2)'), arrow_cast(5, 'Decimal128(10, 2)'), arrow_cast(11, 'Decimal128(10, 2)')),
d128 IN (arrow_cast(3, 'Decimal128(10, 2)'), arrow_cast(4, 'Decimal128(10, 2)'), arrow_cast(5, 'Decimal128(10, 2)'), arrow_cast(11, 'Decimal128(10, 2)'), arrow_cast(13, 'Decimal128(10, 2)')),
imdn IN (INTERVAL '3 months', INTERVAL '4 months', INTERVAL '5 months', INTERVAL '11 months'),
imdn IN (INTERVAL '3 months', INTERVAL '4 months', INTERVAL '5 months', INTERVAL '11 months', INTERVAL '13 months')
FROM in_list_wide
ORDER BY label
----
match true true true true
no_match false false false false
nulls NULL NULL NULL NULL

# The same lists with NOT IN.
query TBBBB
SELECT
label,
d128 NOT IN (arrow_cast(3, 'Decimal128(10, 2)'), arrow_cast(4, 'Decimal128(10, 2)'), arrow_cast(5, 'Decimal128(10, 2)'), arrow_cast(11, 'Decimal128(10, 2)')),
d128 NOT IN (arrow_cast(3, 'Decimal128(10, 2)'), arrow_cast(4, 'Decimal128(10, 2)'), arrow_cast(5, 'Decimal128(10, 2)'), arrow_cast(11, 'Decimal128(10, 2)'), arrow_cast(13, 'Decimal128(10, 2)')),
imdn NOT IN (INTERVAL '3 months', INTERVAL '4 months', INTERVAL '5 months', INTERVAL '11 months'),
imdn NOT IN (INTERVAL '3 months', INTERVAL '4 months', INTERVAL '5 months', INTERVAL '11 months', INTERVAL '13 months')
FROM in_list_wide
ORDER BY label
----
match false false false false
no_match true true true true
nulls NULL NULL NULL NULL

# Null IN list values, including short lists with a single non-null value.
query TBBBB
SELECT
label,
d128 IN (NULL, arrow_cast(3, 'Decimal128(10, 2)'), arrow_cast(4, 'Decimal128(10, 2)'), arrow_cast(11, 'Decimal128(10, 2)')),
d128 IN (NULL, arrow_cast(11, 'Decimal128(10, 2)')),
imdn IN (NULL, INTERVAL '3 months', INTERVAL '4 months', INTERVAL '11 months'),
imdn IN (NULL, INTERVAL '11 months')
FROM in_list_wide
ORDER BY label
----
match true true true true
no_match NULL NULL NULL NULL
nulls NULL NULL NULL NULL

# Cleanup
statement ok
DROP TABLE in_list_wide
Loading