From a45cff7ddd3f3480369e0af269df54988ef36388 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:16:50 -0500 Subject: [PATCH 1/7] fix: `from_unixtime` should respect `datafusion.execution.time_zone` `from_unixtime(expr)` (the single argument form) hardcoded `Timestamp(Second, None)`, so it ignored the session time zone while `now()` and the `to_timestamp*` family in the same session correctly returned timestamps in `datafusion.execution.time_zone`: ```sql SET datafusion.execution.time_zone = 'America/Denver'; SELECT arrow_typeof(from_unixtime(1704110400)), from_unixtime(1704110400); -- Timestamp(s) | 2024-01-01T12:00:00 <-- timezone naive ``` Give `FromUnixtimeFunc` the same `timezone` field + `new_with_config()` / `with_updated_config()` treatment that `NowFunc` and the `to_timestamp*` functions already use, and register it with `make_udf_function_with_config!` so the session config reaches it both at registration time and on `SET`/`RESET`. The two argument form `from_unixtime(expr, 'tz')` is unchanged: an explicit timezone still wins over the session time zone. With the session time zone unset (the default) the result is still `Timestamp(Second, None)`, so existing behaviour is preserved. `to_unixtime` is not affected: it returns `Int64` epoch seconds, and it already threads `config_options` into `ToTimestampSecondsFunc` for its string parsing path. `FromUnixtimeFunc::new()` is deprecated in favour of `new_with_config()`, mirroring what was done for the `to_timestamp*` constructors. Closes https://github.com/apache/datafusion/issues/12892 Co-Authored-By: Claude Opus 5 --- .../functions/src/datetime/from_unixtime.rs | 153 ++++++++++++++++-- datafusion/functions/src/datetime/mod.rs | 6 +- datafusion/sql/src/unparser/expr.rs | 2 +- .../test_files/from_unixtime_timezone.slt | 97 +++++++++++ .../source/user-guide/sql/scalar_functions.md | 19 ++- 5 files changed, 260 insertions(+), 17 deletions(-) create mode 100644 datafusion/sqllogictest/test_files/from_unixtime_timezone.slt diff --git a/datafusion/functions/src/datetime/from_unixtime.rs b/datafusion/functions/src/datetime/from_unixtime.rs index 85494f3abff7..ff042db1bd07 100644 --- a/datafusion/functions/src/datetime/from_unixtime.rs +++ b/datafusion/functions/src/datetime/from_unixtime.rs @@ -20,18 +20,26 @@ use std::sync::Arc; use arrow::datatypes::DataType::{Int64, Timestamp, Utf8}; use arrow::datatypes::TimeUnit::Second; use arrow::datatypes::{DataType, Field, FieldRef}; +use datafusion_common::config::ConfigOptions; use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; use datafusion_expr::TypeSignature::Exact; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use datafusion_expr::{ - ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, - Signature, Volatility, + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, + ScalarUDFImpl, Signature, Volatility, }; use datafusion_macros::user_doc; #[user_doc( doc_section(label = "Time and Date Functions"), - description = "Converts an integer to RFC3339 timestamp format (`YYYY-MM-DDT00:00:00.000000000Z`). Integers and unsigned integers are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`) return the corresponding timestamp.", + description = r#" +Converts an integer to RFC3339 timestamp format (`YYYY-MM-DDT00:00:00.000000000Z`). +Integers and unsigned integers are interpreted as seconds since the unix epoch +(`1970-01-01T00:00:00Z`) return the corresponding timestamp. + +If the optional `timezone` argument is omitted, the timestamp is returned in the +session time zone (`datafusion.execution.time_zone`), which is unset (i.e. +timezone-naive) by default."#, syntax_example = "from_unixtime(expression[, timezone])", sql_example = r#"```sql > select from_unixtime(1599572549, 'America/New_York'); @@ -40,31 +48,59 @@ use datafusion_macros::user_doc; +-----------------------------------------------------------+ | 2020-09-08T09:42:29-04:00 | +-----------------------------------------------------------+ + +-- Without an explicit timezone the session time zone is used +> SET datafusion.execution.time_zone = 'America/New_York'; +> select from_unixtime(1599572549); ++----------------------------------+ +| from_unixtime(Int64(1599572549)) | ++----------------------------------+ +| 2020-09-08T09:42:29-04:00 | ++----------------------------------+ ```"#, standard_argument(name = "expression",), argument( name = "timezone", - description = "Optional timezone to use when converting the integer to a timestamp. If not provided, the default timezone is UTC." + description = "Optional timezone to use when converting the integer to a timestamp. If not provided, the session time zone (`datafusion.execution.time_zone`) is used, which is unset (timezone-naive) by default." ) )] #[derive(Debug, PartialEq, Eq, Hash)] pub struct FromUnixtimeFunc { signature: Signature, + /// Timezone from `datafusion.execution.time_zone`, used by the + /// single-argument form. The two-argument form always uses the timezone + /// given explicitly as the second argument. + timezone: Option>, } impl Default for FromUnixtimeFunc { fn default() -> Self { - Self::new() + Self::new_with_config(&ConfigOptions::default()) } } impl FromUnixtimeFunc { + #[deprecated(since = "55.0.0", note = "use `new_with_config` instead")] + /// Deprecated constructor retained for backwards compatibility. + /// + /// Prefer [`FromUnixtimeFunc::new_with_config`], which picks up the session + /// time zone from [`ConfigOptions`]. This helper mirrors the canonical + /// default (no timezone) provided by `ConfigOptions::default()`. pub fn new() -> Self { + Self::new_with_config(&ConfigOptions::default()) + } + + pub fn new_with_config(config: &ConfigOptions) -> Self { Self { signature: Signature::one_of( vec![Exact(vec![Int64, Utf8]), Exact(vec![Int64])], Volatility::Immutable, ), + timezone: config + .execution + .time_zone + .as_ref() + .map(|tz| Arc::from(tz.as_str())), } } } @@ -78,12 +114,19 @@ impl ScalarUDFImpl for FromUnixtimeFunc { &self.signature } + fn with_updated_config(&self, config: &ConfigOptions) -> Option { + Some(Self::new_with_config(config).into()) + } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { // Length check handled in the signature debug_assert!(matches!(args.scalar_arguments.len(), 1 | 2)); if args.scalar_arguments.len() == 1 { - Ok(Field::new(self.name(), Timestamp(Second, None), true).into()) + Ok( + Field::new(self.name(), Timestamp(Second, self.timezone.clone()), true) + .into(), + ) } else { args.scalar_arguments[1] .and_then(|sv| { @@ -133,7 +176,7 @@ impl ScalarUDFImpl for FromUnixtimeFunc { } match len { - 1 => args[0].cast_to(&Timestamp(Second, None), None), + 1 => args[0].cast_to(&Timestamp(Second, self.timezone.clone()), None), 2 => match &args[1] { ColumnarValue::Scalar(ScalarValue::Utf8(Some(tz))) => args[0] .cast_to(&Timestamp(Second, Some(Arc::from(tz.to_string()))), None), @@ -175,11 +218,13 @@ impl ScalarUDFImpl for FromUnixtimeFunc { mod test { use crate::datetime::from_unixtime::FromUnixtimeFunc; use arrow::datatypes::TimeUnit::Second; - use arrow::datatypes::{DataType, Field}; + use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::ScalarValue; use datafusion_common::ScalarValue::Int64; use datafusion_common::config::ConfigOptions; - use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl}; + use datafusion_expr::{ + ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, + }; use std::sync::Arc; #[test] @@ -192,7 +237,7 @@ mod test { return_field: Field::new("f", DataType::Timestamp(Second, None), true).into(), config_options: Arc::new(ConfigOptions::default()), }; - let result = FromUnixtimeFunc::new().invoke_with_args(args).unwrap(); + let result = FromUnixtimeFunc::default().invoke_with_args(args).unwrap(); match result { ColumnarValue::Scalar(ScalarValue::TimestampSecond(Some(sec), None)) => { @@ -202,6 +247,92 @@ mod test { } } + /// The single argument form reports (and produces) the session time zone + /// configured via `datafusion.execution.time_zone`. + #[test] + fn test_session_timezone_is_used_without_explicit_timezone() { + let mut options = ConfigOptions::default(); + options.execution.time_zone = Some("America/Denver".to_string()); + + let func = FromUnixtimeFunc::new_with_config(&options); + + let arg_field: FieldRef = Field::new("a", DataType::Int64, true).into(); + let scalar_arguments = vec![None]; + let return_field = func + .return_field_from_args(ReturnFieldArgs { + arg_fields: std::slice::from_ref(&arg_field), + scalar_arguments: &scalar_arguments, + }) + .unwrap(); + assert_eq!( + return_field.data_type(), + &DataType::Timestamp(Second, Some(Arc::from("America/Denver"))) + ); + + let args = ScalarFunctionArgs { + args: vec![ColumnarValue::Scalar(Int64(Some(1729900800)))], + arg_fields: vec![arg_field], + number_rows: 1, + return_field, + config_options: Arc::new(options), + }; + let result = func.invoke_with_args(args).unwrap(); + + match result { + ColumnarValue::Scalar(ScalarValue::TimestampSecond(Some(sec), Some(tz))) => { + assert_eq!(sec, 1729900800); + assert_eq!(tz.as_ref(), "America/Denver"); + } + other => panic!("Expected timezone aware scalar value, got {other:?}"), + } + } + + /// An explicit second argument wins over the session time zone. + #[test] + fn test_explicit_timezone_overrides_session_timezone() { + let mut options = ConfigOptions::default(); + options.execution.time_zone = Some("America/Denver".to_string()); + + let func = FromUnixtimeFunc::new_with_config(&options); + + let arg_fields: Vec = vec![ + Field::new("a", DataType::Int64, true).into(), + Field::new("b", DataType::Utf8, true).into(), + ]; + let tz_arg = ScalarValue::Utf8(Some("+08:00".to_string())); + let scalar_arguments = vec![None, Some(&tz_arg)]; + let return_field = func + .return_field_from_args(ReturnFieldArgs { + arg_fields: &arg_fields, + scalar_arguments: &scalar_arguments, + }) + .unwrap(); + assert_eq!( + return_field.data_type(), + &DataType::Timestamp(Second, Some(Arc::from("+08:00"))) + ); + + let args = ScalarFunctionArgs { + args: vec![ + ColumnarValue::Scalar(Int64(Some(1729900800))), + ColumnarValue::Scalar(tz_arg.clone()), + ], + arg_fields, + number_rows: 1, + return_field, + config_options: Arc::new(options), + }; + let result = func.invoke_with_args(args).unwrap(); + + match result { + ColumnarValue::Scalar(ScalarValue::TimestampSecond(Some(sec), Some(tz))) => { + assert_eq!(sec, 1729900800); + assert_eq!(tz.as_ref(), "+08:00"); + } + other => panic!("Expected timezone aware scalar value, got {other:?}"), + } + } + #[test] fn test_with_timezone() { let arg_fields = vec![ @@ -225,7 +356,7 @@ mod test { .into(), config_options: Arc::new(ConfigOptions::default()), }; - let result = FromUnixtimeFunc::new().invoke_with_args(args).unwrap(); + let result = FromUnixtimeFunc::default().invoke_with_args(args).unwrap(); match result { ColumnarValue::Scalar(ScalarValue::TimestampSecond(Some(sec), Some(tz))) => { diff --git a/datafusion/functions/src/datetime/mod.rs b/datafusion/functions/src/datetime/mod.rs index 39b9453295df..1188851d4249 100644 --- a/datafusion/functions/src/datetime/mod.rs +++ b/datafusion/functions/src/datetime/mod.rs @@ -47,12 +47,12 @@ make_udf_function!(date_part::DatePartFunc, date_part); make_udf_function!(date_trunc::DateTruncFunc, date_trunc); make_udf_function!(make_date::MakeDateFunc, make_date); make_udf_function!(make_time::MakeTimeFunc, make_time); -make_udf_function!(from_unixtime::FromUnixtimeFunc, from_unixtime); make_udf_function!(to_char::ToCharFunc, to_char); make_udf_function!(to_date::ToDateFunc, to_date); make_udf_function!(to_local_time::ToLocalTimeFunc, to_local_time); make_udf_function!(to_time::ToTimeFunc, to_time); make_udf_function!(to_unixtime::ToUnixtimeFunc, to_unixtime); +make_udf_function_with_config!(from_unixtime::FromUnixtimeFunc, from_unixtime); make_udf_function_with_config!(to_timestamp::ToTimestampFunc, to_timestamp); make_udf_function_with_config!( to_timestamp::ToTimestampSecondsFunc, @@ -80,7 +80,7 @@ pub mod expr_fn { ),( from_unixtime, "converts an integer to RFC3339 timestamp format string", - unixtime + @config unixtime ),( date_bin, "coerces an arbitrary timestamp to the start of the nearest specified interval", @@ -281,7 +281,7 @@ pub fn functions() -> Vec> { date_bin(), date_part(), date_trunc(), - from_unixtime(), + from_unixtime(&config), make_date(), make_time(), now(&config), diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs index cd3ac1f3a455..77a5e0969389 100644 --- a/datafusion/sql/src/unparser/expr.rs +++ b/datafusion/sql/src/unparser/expr.rs @@ -3470,7 +3470,7 @@ mod tests { ] { let unparser = Unparser::new(dialect.as_ref()); let expr = Expr::ScalarFunction(ScalarFunction { - func: Arc::new(ScalarUDF::from(FromUnixtimeFunc::new())), + func: Arc::new(ScalarUDF::from(FromUnixtimeFunc::default())), args: vec![col("date_col")], }); diff --git a/datafusion/sqllogictest/test_files/from_unixtime_timezone.slt b/datafusion/sqllogictest/test_files/from_unixtime_timezone.slt new file mode 100644 index 000000000000..2af8c902fe7c --- /dev/null +++ b/datafusion/sqllogictest/test_files/from_unixtime_timezone.slt @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +########## +## from_unixtime timezone tests +## +## https://github.com/apache/datafusion/issues/12892 +########## + +## Make sure we start from the default (unset) session time zone +statement ok +RESET datafusion.execution.time_zone + +## Test 1: session time zone unset -> timezone naive, as before +query TP +SELECT arrow_typeof(from_unixtime(1704110400)), from_unixtime(1704110400); +---- +Timestamp(s) 2024-01-01T12:00:00 + +## Test 2: fixed offset session time zone +statement ok +SET datafusion.execution.time_zone = '+08:00'; + +query TP +SELECT arrow_typeof(from_unixtime(1704110400)), from_unixtime(1704110400); +---- +Timestamp(s, "+08:00") 2024-01-01T20:00:00+08:00 + +## The reproducer from the issue: to_unixtime/from_unixtime must round trip +query I +SELECT to_unixtime(from_unixtime(1704110400)); +---- +1704110400 + +## Test 3: named IANA session time zone +statement ok +SET datafusion.execution.time_zone = 'America/Denver'; + +query TP +SELECT arrow_typeof(from_unixtime(1704110400)), from_unixtime(1704110400); +---- +Timestamp(s, "America/Denver") 2024-01-01T05:00:00-07:00 + +## Test 4: an explicit timezone argument wins over the session time zone +query TP +SELECT + arrow_typeof(from_unixtime(1704110400, 'America/New_York')), + from_unixtime(1704110400, 'America/New_York'); +---- +Timestamp(s, "America/New_York") 2024-01-01T07:00:00-05:00 + +## Test 5: array (not constant folded) input carries the session time zone too +query TP +SELECT arrow_typeof(from_unixtime(v)), from_unixtime(v) FROM (VALUES (1704110400), (0)) AS t(v); +---- +Timestamp(s, "America/Denver") 2024-01-01T05:00:00-07:00 +Timestamp(s, "America/Denver") 1969-12-31T17:00:00-07:00 + +## Test 6: NULL input stays NULL, but keeps the session time zone in its type +query TP +SELECT arrow_typeof(from_unixtime(NULL)), from_unixtime(NULL); +---- +Timestamp(s, "America/Denver") NULL + +query TP +SELECT arrow_typeof(from_unixtime(arrow_cast(NULL, 'Int64'), 'America/New_York')), from_unixtime(arrow_cast(NULL, 'Int64'), 'America/New_York'); +---- +Timestamp(s, "America/New_York") NULL + +## Test 7: the underlying instant is unchanged - only the displayed time zone differs +query B +SELECT from_unixtime(1704110400) = from_unixtime(1704110400, 'America/New_York'); +---- +true + +## Test 8: RESET restores the timezone naive behaviour +statement ok +RESET datafusion.execution.time_zone + +query TP +SELECT arrow_typeof(from_unixtime(1704110400)), from_unixtime(1704110400); +---- +Timestamp(s) 2024-01-01T12:00:00 diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index bdc8efb1bd15..e610044563b0 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -2659,7 +2659,13 @@ _Alias of [date_trunc](#date_trunc)._ ### `from_unixtime` -Converts an integer to RFC3339 timestamp format (`YYYY-MM-DDT00:00:00.000000000Z`). Integers and unsigned integers are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`) return the corresponding timestamp. +Converts an integer to RFC3339 timestamp format (`YYYY-MM-DDT00:00:00.000000000Z`). +Integers and unsigned integers are interpreted as seconds since the unix epoch +(`1970-01-01T00:00:00Z`) return the corresponding timestamp. + +If the optional `timezone` argument is omitted, the timestamp is returned in the +session time zone (`datafusion.execution.time_zone`), which is unset (i.e. +timezone-naive) by default. ```sql from_unixtime(expression[, timezone]) @@ -2668,7 +2674,7 @@ from_unixtime(expression[, timezone]) #### Arguments - **expression**: The expression to operate on. Can be a constant, column, or function, and any combination of operators. -- **timezone**: Optional timezone to use when converting the integer to a timestamp. If not provided, the default timezone is UTC. +- **timezone**: Optional timezone to use when converting the integer to a timestamp. If not provided, the session time zone (`datafusion.execution.time_zone`) is used, which is unset (timezone-naive) by default. #### Example @@ -2679,6 +2685,15 @@ from_unixtime(expression[, timezone]) +-----------------------------------------------------------+ | 2020-09-08T09:42:29-04:00 | +-----------------------------------------------------------+ + +-- Without an explicit timezone the session time zone is used +> SET datafusion.execution.time_zone = 'America/New_York'; +> select from_unixtime(1599572549); ++----------------------------------+ +| from_unixtime(Int64(1599572549)) | ++----------------------------------+ +| 2020-09-08T09:42:29-04:00 | ++----------------------------------+ ``` ### `make_date` From 3b6dea544a75a2b0bc3e0e855db5611ffe18ebb7 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:25:33 -0500 Subject: [PATCH 2/7] Deprecate FromUnixtimeFunc::new() since 56.0.0, not 55.0.0 55.0.0 is already tagged and released, so a new deprecation on main belongs in the next release. Main already carries three other `since = "56.0.0"` deprecations. Co-Authored-By: Claude Opus 5 --- datafusion/functions/src/datetime/from_unixtime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/functions/src/datetime/from_unixtime.rs b/datafusion/functions/src/datetime/from_unixtime.rs index ff042db1bd07..34396f317b20 100644 --- a/datafusion/functions/src/datetime/from_unixtime.rs +++ b/datafusion/functions/src/datetime/from_unixtime.rs @@ -80,7 +80,7 @@ impl Default for FromUnixtimeFunc { } impl FromUnixtimeFunc { - #[deprecated(since = "55.0.0", note = "use `new_with_config` instead")] + #[deprecated(since = "56.0.0", note = "use `new_with_config` instead")] /// Deprecated constructor retained for backwards compatibility. /// /// Prefer [`FromUnixtimeFunc::new_with_config`], which picks up the session From bda12f107f79560c9ea986cff5cc3c13720236ef Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:17:36 -0500 Subject: [PATCH 3/7] docs: fix the `from_unixtime` description The description predates this PR and was wrong in both halves: `from_unixtime` returns an Arrow `Timestamp(Second, ...)`, not an RFC3339 string with nanosecond precision, and the trailing `Z` contradicted the example right below it, which shows a `-04:00` offset. Describe what the function actually returns and regenerate `docs/source/user-guide/sql/scalar_functions.md`. Co-Authored-By: Claude Opus 5 --- datafusion/functions/src/datetime/from_unixtime.rs | 6 +++--- docs/source/user-guide/sql/scalar_functions.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datafusion/functions/src/datetime/from_unixtime.rs b/datafusion/functions/src/datetime/from_unixtime.rs index 34396f317b20..a97c1b0ddca5 100644 --- a/datafusion/functions/src/datetime/from_unixtime.rs +++ b/datafusion/functions/src/datetime/from_unixtime.rs @@ -33,9 +33,9 @@ use datafusion_macros::user_doc; #[user_doc( doc_section(label = "Time and Date Functions"), description = r#" -Converts an integer to RFC3339 timestamp format (`YYYY-MM-DDT00:00:00.000000000Z`). -Integers and unsigned integers are interpreted as seconds since the unix epoch -(`1970-01-01T00:00:00Z`) return the corresponding timestamp. +Converts an integer to a timestamp with second precision (`Timestamp(Second)`). +The integer is interpreted as the number of seconds since the unix epoch +(`1970-01-01T00:00:00Z`). If the optional `timezone` argument is omitted, the timestamp is returned in the session time zone (`datafusion.execution.time_zone`), which is unset (i.e. diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index e610044563b0..5b51bf72e9c9 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -2659,9 +2659,9 @@ _Alias of [date_trunc](#date_trunc)._ ### `from_unixtime` -Converts an integer to RFC3339 timestamp format (`YYYY-MM-DDT00:00:00.000000000Z`). -Integers and unsigned integers are interpreted as seconds since the unix epoch -(`1970-01-01T00:00:00Z`) return the corresponding timestamp. +Converts an integer to a timestamp with second precision (`Timestamp(Second)`). +The integer is interpreted as the number of seconds since the unix epoch +(`1970-01-01T00:00:00Z`). If the optional `timezone` argument is omitted, the timestamp is returned in the session time zone (`datafusion.execution.time_zone`), which is unset (i.e. From 78764476dff7dc565da826fdd8d53fe56a85fcd2 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:17:48 -0500 Subject: [PATCH 4/7] fix: reject `from_unixtime` values whose local date and time is out of range A `Timestamp(Second, Some(tz))` is *rendered* by shifting the UTC instant by the zone's offset, and Arrow does that with `chrono`'s `DateTime::naive_local`, which panics when the shifted value falls outside `NaiveDateTime`'s range: SELECT from_unixtime(-8334601211039, 'America/New_York'); -- thread 'main' panicked: Local time out of range for `NaiveDateTime` The panic fires when the value is *formatted*, not when it is produced, so it surfaces far away from `from_unixtime`. The two argument form has always been able to reach it (https://github.com/apache/datafusion/issues/16594). The previous commits on this branch let the one argument form reach it too, because it now applies the session time zone. Check the bound in `from_unixtime` and return a `DataFusionError` instead of producing a value that panics downstream. Values inside the range that is representable in every time zone (the overwhelmingly common case) take a fast path that only looks at the array's min and max, so the per-value offset resolution is confined to inputs near the limits. Timezone naive results are never shifted and keep the full `NaiveDateTime` range, exactly as before. Values that are out of range in UTC as well are left to the cast, which already reports them. Co-Authored-By: Claude Opus 5 --- .../functions/src/datetime/from_unixtime.rs | 322 +++++++++++++++++- .../test_files/from_unixtime_timezone.slt | 64 ++++ 2 files changed, 379 insertions(+), 7 deletions(-) diff --git a/datafusion/functions/src/datetime/from_unixtime.rs b/datafusion/functions/src/datetime/from_unixtime.rs index a97c1b0ddca5..254f19e5f4cb 100644 --- a/datafusion/functions/src/datetime/from_unixtime.rs +++ b/datafusion/functions/src/datetime/from_unixtime.rs @@ -17,9 +17,14 @@ use std::sync::Arc; +use arrow::array::AsArray; +use arrow::array::timezone::Tz; +use arrow::compute::kernels::aggregate::{max, min}; use arrow::datatypes::DataType::{Int64, Timestamp, Utf8}; +use arrow::datatypes::Int64Type; use arrow::datatypes::TimeUnit::Second; use arrow::datatypes::{DataType, Field, FieldRef}; +use chrono::{DateTime, NaiveDateTime, Offset, TimeDelta, TimeZone}; use datafusion_common::config::ConfigOptions; use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; use datafusion_expr::TypeSignature::Exact; @@ -175,20 +180,25 @@ impl ScalarUDFImpl for FromUnixtimeFunc { ); } - match len { - 1 => args[0].cast_to(&Timestamp(Second, self.timezone.clone()), None), + let timezone = match len { + 1 => self.timezone.clone(), 2 => match &args[1] { - ColumnarValue::Scalar(ScalarValue::Utf8(Some(tz))) => args[0] - .cast_to(&Timestamp(Second, Some(Arc::from(tz.to_string()))), None), + ColumnarValue::Scalar(ScalarValue::Utf8(Some(tz))) => { + Some(Arc::::from(tz.as_str())) + } _ => { - exec_err!( + return exec_err!( "Unsupported data type {} for function from_unixtime", args[1].data_type() - ) + ); } }, _ => unreachable!(), - } + }; + + validate_local_datetime_range(&args[0], timezone.as_deref())?; + + args[0].cast_to(&Timestamp(Second, timezone), None) } fn output_ordering(&self, inputs: &[ExprProperties]) -> Result { @@ -214,11 +224,113 @@ impl ScalarUDFImpl for FromUnixtimeFunc { } } +/// An upper bound on the magnitude of any time zone's UTC offset, rounded up to +/// a whole day. Real offsets never exceed 15 hours (including the historical +/// local mean time offsets used before a zone's first transition). +const MAX_TIMEZONE_OFFSET_SECONDS: i64 = 24 * 60 * 60; + +/// The range of `Timestamp(Second)` values whose local date and time is +/// representable in *every* time zone. +/// +/// Values inside this range never need a per-value check: shifting them by any +/// possible UTC offset keeps them inside [`NaiveDateTime`]'s range. +fn always_representable_range() -> (i64, i64) { + ( + NaiveDateTime::MIN.and_utc().timestamp() + MAX_TIMEZONE_OFFSET_SECONDS, + NaiveDateTime::MAX.and_utc().timestamp() - MAX_TIMEZONE_OFFSET_SECONDS, + ) +} + +/// Returns an error if any value in `values` has no representable local date +/// and time in `timezone`. +/// +/// A `Timestamp(Second, Some(tz))` is *rendered* by shifting the UTC instant by +/// the zone's offset. Arrow does that with `DateTime::naive_local`, which +/// **panics** when the shifted value falls outside [`NaiveDateTime`]'s range, +/// so such values have to be rejected before they are produced: the panic would +/// otherwise happen far away from `from_unixtime`, when the value is formatted. +/// +/// See . +fn validate_local_datetime_range( + values: &ColumnarValue, + timezone: Option<&str>, +) -> Result<()> { + // A timezone naive timestamp is never shifted, so it can only be out of + // range if the UTC instant itself is, which the cast already rejects. + let Some(timezone) = timezone else { + return Ok(()); + }; + + let (safe_min, safe_max) = always_representable_range(); + + let (value_min, value_max) = match values { + ColumnarValue::Scalar(ScalarValue::Int64(Some(value))) => (*value, *value), + // NULL (and anything else the cast will reject) needs no check. + ColumnarValue::Scalar(_) => return Ok(()), + ColumnarValue::Array(array) => { + let array = array.as_primitive::(); + match (min(array), max(array)) { + (Some(value_min), Some(value_max)) => (value_min, value_max), + // All null. + _ => return Ok(()), + } + } + }; + + // Fast path: no value can overflow, whatever the zone's offset is. + if value_min >= safe_min && value_max <= safe_max { + return Ok(()); + } + + let Ok(tz) = timezone.parse::() else { + // An unparseable time zone is reported by the cast below. + return Ok(()); + }; + + let check = |seconds: i64| -> Result<()> { + if seconds >= safe_min && seconds <= safe_max { + return Ok(()); + } + let Some(utc) = DateTime::from_timestamp(seconds, 0) else { + // Not representable as an instant at all: the cast reports this. + return Ok(()); + }; + let utc = utc.naive_utc(); + let offset = tz.offset_from_utc_datetime(&utc).fix().local_minus_utc(); + if utc + .checked_add_signed(TimeDelta::seconds(i64::from(offset))) + .is_none() + { + return exec_err!( + "Cannot convert {seconds} to a timestamp in timezone \"{timezone}\" \ + for function from_unixtime: the local date and time is outside the \ + supported range" + ); + } + Ok(()) + }; + + match values { + ColumnarValue::Scalar(_) => check(value_min), + ColumnarValue::Array(array) => { + // Only the values near the limits do any real work; `check` returns + // immediately for everything inside the always representable range. + for value in array.as_primitive::().iter().flatten() { + check(value)?; + } + Ok(()) + } + } +} + #[cfg(test)] mod test { use crate::datetime::from_unixtime::FromUnixtimeFunc; + use arrow::array::{ArrayRef, Int64Array}; use arrow::datatypes::TimeUnit::Second; use arrow::datatypes::{DataType, Field, FieldRef}; + use arrow::util::display::{ArrayFormatter, FormatOptions}; + use datafusion_common::Result; use datafusion_common::ScalarValue; use datafusion_common::ScalarValue::Int64; use datafusion_common::config::ConfigOptions; @@ -227,6 +339,202 @@ mod test { }; use std::sync::Arc; + /// The last second since the epoch that `chrono::NaiveDateTime` can + /// represent (`+262142-12-31T23:59:59`). + const MAX_UTC_SECONDS: i64 = 8210266876799; + /// The first second since the epoch that `chrono::NaiveDateTime` can + /// represent (`-262143-01-01T00:00:00`). + const MIN_UTC_SECONDS: i64 = -8334601228800; + + /// Invoke the single argument form with `datafusion.execution.time_zone` + /// set to `timezone`. + fn from_unixtime_session_tz( + seconds: i64, + timezone: Option<&str>, + ) -> Result { + let mut options = ConfigOptions::default(); + options.execution.time_zone = timezone.map(str::to_string); + let func = FromUnixtimeFunc::new_with_config(&options); + + let arg_field: FieldRef = Field::new("a", DataType::Int64, true).into(); + let return_field = Field::new( + "f", + DataType::Timestamp(Second, timezone.map(Arc::from)), + true, + ) + .into(); + func.invoke_with_args(ScalarFunctionArgs { + args: vec![ColumnarValue::Scalar(Int64(Some(seconds)))], + arg_fields: vec![arg_field], + number_rows: 1, + return_field, + config_options: Arc::new(options), + }) + } + + /// Invoke the two argument form with an explicit `timezone`. + fn from_unixtime_explicit_tz( + values: ColumnarValue, + timezone: &str, + ) -> Result { + let number_rows = match &values { + ColumnarValue::Array(array) => array.len(), + ColumnarValue::Scalar(_) => 1, + }; + let arg_fields: Vec = vec![ + Field::new("a", DataType::Int64, true).into(), + Field::new("b", DataType::Utf8, true).into(), + ]; + let return_field = Field::new( + "f", + DataType::Timestamp(Second, Some(Arc::from(timezone))), + true, + ) + .into(); + FromUnixtimeFunc::default().invoke_with_args(ScalarFunctionArgs { + args: vec![ + values, + ColumnarValue::Scalar(ScalarValue::Utf8(Some(timezone.to_string()))), + ], + arg_fields, + number_rows, + return_field, + config_options: Arc::new(ConfigOptions::default()), + }) + } + + /// Render the result the way a client would. + /// + /// The out of range local date and time panics when the value is + /// *formatted*, not when it is produced, so every bounds test has to go + /// through a formatter to be meaningful. + fn display(value: ColumnarValue) -> String { + let array = value.to_array(1).unwrap(); + let options = FormatOptions::default(); + let formatter = ArrayFormatter::try_new(array.as_ref(), &options).unwrap(); + formatter.value(0).to_string() + } + + /// The single argument form used to silently produce values that panicked + /// downstream once a session time zone was set. + /// + /// `America/New_York` is at -04:56:02 (local mean time) that far in the + /// past, so the local date and time runs out one offset earlier than the + /// UTC instant does. + #[test] + fn test_named_timezone_lower_bound() { + let last_ok = MIN_UTC_SECONDS + 17762; + assert_eq!(last_ok, -8334601211038); + + let value = from_unixtime_session_tz(last_ok, Some("America/New_York")).unwrap(); + assert_eq!(display(value), "-262143-01-01T00:00:00-04:56"); + + let err = from_unixtime_session_tz(last_ok - 1, Some("America/New_York")) + .expect_err("expected an out of range error, not a value that panics"); + assert!( + err.message().contains("outside the supported range"), + "unexpected error: {err}" + ); + } + + /// The same bound in the other direction: a zone that is *ahead* of UTC + /// runs out of local date and time before the UTC instant does. + /// + /// This is the two argument form, which panicked on `main` as well + /// (). + #[test] + fn test_named_timezone_upper_bound() { + // +09:00, with no daylight saving. + let last_ok = MAX_UTC_SECONDS - 32400; + + let value = from_unixtime_explicit_tz( + ColumnarValue::Scalar(Int64(Some(last_ok))), + "Asia/Tokyo", + ) + .unwrap(); + assert_eq!(display(value), "+262142-12-31T23:59:59+09:00"); + + let err = from_unixtime_explicit_tz( + ColumnarValue::Scalar(Int64(Some(last_ok + 1))), + "Asia/Tokyo", + ) + .expect_err("expected an out of range error, not a value that panics"); + assert!( + err.message().contains("outside the supported range"), + "unexpected error: {err}" + ); + } + + /// Fixed offset zones hit exactly the same bound. + #[test] + fn test_fixed_offset_upper_bound() { + let last_ok = MAX_UTC_SECONDS - 28800; + + let value = from_unixtime_session_tz(last_ok, Some("+08:00")).unwrap(); + assert_eq!(display(value), "+262142-12-31T23:59:59+08:00"); + + let err = from_unixtime_session_tz(last_ok + 1, Some("+08:00")) + .expect_err("expected an out of range error, not a value that panics"); + assert!( + err.message().contains("outside the supported range"), + "unexpected error: {err}" + ); + } + + #[test] + fn test_fixed_offset_lower_bound() { + let last_ok = MIN_UTC_SECONDS + 28800; + + let value = from_unixtime_explicit_tz( + ColumnarValue::Scalar(Int64(Some(last_ok))), + "-08:00", + ) + .unwrap(); + assert_eq!(display(value), "-262143-01-01T00:00:00-08:00"); + + let err = from_unixtime_explicit_tz( + ColumnarValue::Scalar(Int64(Some(last_ok - 1))), + "-08:00", + ) + .expect_err("expected an out of range error, not a value that panics"); + assert!( + err.message().contains("outside the supported range"), + "unexpected error: {err}" + ); + } + + /// Array input is checked too, not just the constant folded scalar case. + #[test] + fn test_array_input_is_bounds_checked() { + let array: ArrayRef = + Arc::new(Int64Array::from(vec![Some(0), None, Some(MIN_UTC_SECONDS)])); + let err = + from_unixtime_explicit_tz(ColumnarValue::Array(array), "America/New_York") + .expect_err("expected an out of range error, not a value that panics"); + assert!( + err.message().contains("outside the supported range"), + "unexpected error: {err}" + ); + + // In range values still work. + let array: ArrayRef = Arc::new(Int64Array::from(vec![Some(0), None])); + let value = + from_unixtime_explicit_tz(ColumnarValue::Array(array), "America/New_York") + .unwrap(); + assert_eq!(display(value), "1969-12-31T19:00:00-05:00"); + } + + /// Without a time zone the value is never shifted, so the full + /// `NaiveDateTime` range stays usable, exactly as before this bound check. + #[test] + fn test_timezone_naive_extremes_are_still_accepted() { + let value = from_unixtime_session_tz(MIN_UTC_SECONDS, None).unwrap(); + assert_eq!(display(value), "-262143-01-01T00:00:00"); + + let value = from_unixtime_session_tz(MAX_UTC_SECONDS, None).unwrap(); + assert_eq!(display(value), "+262142-12-31T23:59:59"); + } + #[test] fn test_without_timezone() { let arg_field = Arc::new(Field::new("a", DataType::Int64, true)); diff --git a/datafusion/sqllogictest/test_files/from_unixtime_timezone.slt b/datafusion/sqllogictest/test_files/from_unixtime_timezone.slt index 2af8c902fe7c..5ea50dac9c62 100644 --- a/datafusion/sqllogictest/test_files/from_unixtime_timezone.slt +++ b/datafusion/sqllogictest/test_files/from_unixtime_timezone.slt @@ -95,3 +95,67 @@ query TP SELECT arrow_typeof(from_unixtime(1704110400)), from_unixtime(1704110400); ---- Timestamp(s) 2024-01-01T12:00:00 + +########## +## Out of range values +## +## A `Timestamp(Second, Some(tz))` is rendered by shifting the UTC instant by +## the zone's offset, and that shifted value has to stay inside the range +## chrono's `NaiveDateTime` can represent, otherwise formatting the value +## panics. +## +## https://github.com/apache/datafusion/issues/16594 +########## + +## The last value `America/New_York` can represent (its offset is -04:56 that +## far in the past, so the local time runs out before the UTC instant does) +query P +SELECT from_unixtime(-8334601211038, 'America/New_York'); +---- +-262143-01-01T00:00:00-04:56 + +## One second earlier is an error, not a panic +query error DataFusion error: Execution error: Cannot convert \-8334601211039 to a timestamp in timezone "America/New_York" for function from_unixtime: the local date and time is outside the supported range +SELECT from_unixtime(-8334601211039, 'America/New_York'); + +## The same bound applies to the session time zone form +statement ok +SET datafusion.execution.time_zone = 'America/New_York'; + +query P +SELECT from_unixtime(-8334601211038); +---- +-262143-01-01T00:00:00-04:56 + +query error DataFusion error: Execution error: Cannot convert \-8334601211039 to a timestamp in timezone "America/New_York" for function from_unixtime: the local date and time is outside the supported range +SELECT from_unixtime(-8334601211039); + +## ... and to array (not constant folded) input +query error DataFusion error: Execution error: Cannot convert \-8334601211039 to a timestamp in timezone "America/New_York" for function from_unixtime: the local date and time is outside the supported range +SELECT from_unixtime(v) FROM (VALUES (0), (-8334601211039)) AS t(v); + +## A zone that is ahead of UTC runs out at the other end +statement ok +SET datafusion.execution.time_zone = '+08:00'; + +query P +SELECT from_unixtime(8210266847999); +---- ++262142-12-31T23:59:59+08:00 + +query error DataFusion error: Execution error: Cannot convert 8210266848000 to a timestamp in timezone "\+08:00" for function from_unixtime: the local date and time is outside the supported range +SELECT from_unixtime(8210266848000); + +statement ok +RESET datafusion.execution.time_zone + +## Without a time zone the full range stays usable +query P +SELECT from_unixtime(-8334601228800); +---- +-262143-01-01T00:00:00 + +query P +SELECT from_unixtime(8210266876799); +---- ++262142-12-31T23:59:59 From e8cb6a699c26f958ff17ddede1c71917c48501e8 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:21:09 -0500 Subject: [PATCH 5/7] Fix a typo flagged by the spell-check job `unparseable` -> `unparsable`, in a comment added by the previous commit. Co-Authored-By: Claude Opus 5 --- datafusion/functions/src/datetime/from_unixtime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/functions/src/datetime/from_unixtime.rs b/datafusion/functions/src/datetime/from_unixtime.rs index 254f19e5f4cb..8af4553f5c34 100644 --- a/datafusion/functions/src/datetime/from_unixtime.rs +++ b/datafusion/functions/src/datetime/from_unixtime.rs @@ -283,7 +283,7 @@ fn validate_local_datetime_range( } let Ok(tz) = timezone.parse::() else { - // An unparseable time zone is reported by the cast below. + // An unparsable time zone is reported by the cast below. return Ok(()); }; From fb72121aef5d69343836c2b5b2ce68a640d3cc8c Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:01:17 -0500 Subject: [PATCH 6/7] Correct the from_unixtime expr_fn doc string `mod.rs` still described `from_unixtime` as converting "an integer to RFC3339 timestamp format string". The function returns an Arrow `Timestamp(Second, ...)`, not a string, and not at nanosecond resolution. This is the same wrong claim that an earlier commit corrected in the `user_doc` block. The `expr_fn` copy was missed. Co-Authored-By: Claude Opus 5 --- datafusion/functions/src/datetime/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/functions/src/datetime/mod.rs b/datafusion/functions/src/datetime/mod.rs index 1188851d4249..9246d1d25d97 100644 --- a/datafusion/functions/src/datetime/mod.rs +++ b/datafusion/functions/src/datetime/mod.rs @@ -79,7 +79,7 @@ pub mod expr_fn { "returns current UTC time as a Time64 value", ),( from_unixtime, - "converts an integer to RFC3339 timestamp format string", + "converts an integer of epoch seconds to a second-precision timestamp", @config unixtime ),( date_bin, From 469a32abd3826fe0acc9476cde69de6747f1a4de Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 11 Sep 2026 09:57:27 -0500 Subject: [PATCH 7/7] test: cover a DST summer instant for from_unixtime in America/Denver The named-zone test only checked a January instant, which renders `-07:00`. That result is equally consistent with a DST-aware zone and with an implementation that wrongly treated America/Denver as a fixed `-07:00` offset, so it did not prove DST handling. Adds a July instant, 2024-07-01T12:00:00Z, under the same session zone. It must render `2024-07-01T06:00:00-06:00`, which only a DST-aware zone produces. Expected output generated by running the query; changing it to `-07:00` makes the file fail. Suggested in review by @kosiew. Co-Authored-By: Claude Opus 5 --- .../sqllogictest/test_files/from_unixtime_timezone.slt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/datafusion/sqllogictest/test_files/from_unixtime_timezone.slt b/datafusion/sqllogictest/test_files/from_unixtime_timezone.slt index 5ea50dac9c62..a295fe48ef18 100644 --- a/datafusion/sqllogictest/test_files/from_unixtime_timezone.slt +++ b/datafusion/sqllogictest/test_files/from_unixtime_timezone.slt @@ -55,6 +55,13 @@ SELECT arrow_typeof(from_unixtime(1704110400)), from_unixtime(1704110400); ---- Timestamp(s, "America/Denver") 2024-01-01T05:00:00-07:00 +# January alone (`-07:00`) cannot tell a DST-aware zone from a fixed `-07:00` +# offset. A July instant must render `-06:00`, which only a DST-aware zone does. +query TP +SELECT arrow_typeof(from_unixtime(1719835200)), from_unixtime(1719835200); +---- +Timestamp(s, "America/Denver") 2024-07-01T06:00:00-06:00 + ## Test 4: an explicit timezone argument wins over the session time zone query TP SELECT