From eb23655d11c940e1899ca861a469b14a5932c748 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:10:19 -0500 Subject: [PATCH 1/4] test: characterize `AT TIME ZONE` on a timezone-aware timestamp Records DataFusion's current behaviour for ` AT TIME ZONE zone`, including the reproducer from apache/datafusion#12218, DST transitions on a real multi-row column, chaining, fixed offsets and precision handling. PostgreSQL returns a timezone-*naive* `timestamp` here, holding the wall clock in `zone`; DataFusion returns a timezone-aware value that merely relabels the display zone. These expectations are flipped in the next commit. Co-Authored-By: Claude Opus 5 --- .../test_files/datetime/timestamps.slt | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index d73bc6eb06de8..de6e599d026d5 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -4049,6 +4049,123 @@ SELECT '2000-12-01 04:04:12' AT TIME ZONE 'America/New York'; statement error SELECT '2023-03-12 02:00:00' AT TIME ZONE 'EDT'; +########## +## AT TIME ZONE applied to a timezone-*aware* timestamp +## +## https://github.com/apache/datafusion/issues/12218 +## +## PostgreSQL (and DuckDB) semantics are asymmetric: +## +## * ` AT TIME ZONE zone` reads the value as a wall clock +## in `zone` and returns the matching tz-*aware* instant, and +## * ` AT TIME ZONE zone` returns the wall clock that the +## instant has in `zone`, as a tz-*naive* timestamp. +## +## The expectations below record DataFusion's behaviour *before* the fix for +## #12218: the tz-aware case wrongly stays tz-aware, which only relabels the +## display zone. The following commit flips them to the PostgreSQL results. +########## + +statement ok +SET datafusion.execution.time_zone = 'UTC'; + +statement ok +CREATE TABLE at_tz_t AS +SELECT + '2024-01-01T12:00:00Z'::timestamptz AS tstz, + '2024-01-01 12:00:00'::timestamp AS tsn; + +# The tz-naive column is unaffected by this issue: noon-in-Denver is the same +# instant PostgreSQL reports (`2024-01-01 19:00:00+00`). +query TP +SELECT arrow_typeof(tsn AT TIME ZONE 'America/Denver'), tsn AT TIME ZONE 'America/Denver' FROM at_tz_t; +---- +Timestamp(ns, "America/Denver") 2024-01-01T12:00:00-07:00 + +# The tz-aware column: PostgreSQL returns `timestamp` (naive) `2024-01-01 05:00:00`. +query TP +SELECT arrow_typeof(tstz AT TIME ZONE 'America/Denver'), tstz AT TIME ZONE 'America/Denver' FROM at_tz_t; +---- +Timestamp(ns, "America/Denver") 2024-01-01T05:00:00-07:00 + +# The exact reproducer from #12218. PostgreSQL returns `2024-01-01 05:00:00`. +query P +SELECT (tstz AT TIME ZONE 'America/Denver')::timestamp FROM at_tz_t; +---- +2024-01-01T12:00:00 + +# Chained. PostgreSQL: `timestamptz` `2024-01-01 04:00:00+00`, i.e. the Denver +# wall clock (05:00) re-read as a Brussels wall clock. +query TP +SELECT + arrow_typeof(tstz AT TIME ZONE 'America/Denver' AT TIME ZONE 'Europe/Brussels'), + tstz AT TIME ZONE 'America/Denver' AT TIME ZONE 'Europe/Brussels' +FROM at_tz_t; +---- +Timestamp(ns, "Europe/Brussels") 2024-01-01T13:00:00+01:00 + +# Fixed offset. Note DataFusion follows the arrow/ISO-8601 sign convention here +# (`+05:30` is 5h30m *east* of UTC), while PostgreSQL applies the POSIX +# convention to offsets spelled as strings. That difference is pre-existing and +# is not what #12218 is about; only the result *type* changes below. +query TP +SELECT arrow_typeof(tstz AT TIME ZONE '+05:30'), tstz AT TIME ZONE '+05:30' FROM at_tz_t; +---- +Timestamp(ns, "+05:30") 2024-01-01T17:30:00+05:30 + +# `AT TIME ZONE` currently forces Nanosecond precision regardless of the input. +query T +SELECT arrow_typeof(arrow_cast('2024-01-01T12:00:00Z', 'Timestamp(Microsecond, Some("UTC"))') AT TIME ZONE 'America/Denver'); +---- +Timestamp(ns, "America/Denver") + +query T +SELECT arrow_typeof(arrow_cast('2024-01-01T12:00:00', 'Timestamp(Microsecond, None)') AT TIME ZONE 'America/Denver'); +---- +Timestamp(ns, "America/Denver") + +# A real (multi-row) tz-aware column, spanning both DST transitions in +# America/Denver. The wall clocks below all agree with PostgreSQL; it is only +# the type that is wrong today. +statement ok +CREATE TABLE at_tz_dst(ts timestamptz) AS VALUES + (arrow_cast('2024-01-01T12:00:00Z', 'Timestamp(Nanosecond, Some("UTC"))')), + (arrow_cast('2024-03-10T08:59:00Z', 'Timestamp(Nanosecond, Some("UTC"))')), + (arrow_cast('2024-03-10T09:00:00Z', 'Timestamp(Nanosecond, Some("UTC"))')), + (arrow_cast('2024-07-01T12:00:00Z', 'Timestamp(Nanosecond, Some("UTC"))')), + (arrow_cast('2024-11-03T07:59:00Z', 'Timestamp(Nanosecond, Some("UTC"))')), + (arrow_cast('2024-11-03T08:00:00Z', 'Timestamp(Nanosecond, Some("UTC"))')); + +query PPT +SELECT + ts, + ts AT TIME ZONE 'America/Denver' AS denver, + arrow_typeof(ts AT TIME ZONE 'America/Denver') AS denver_type +FROM at_tz_dst ORDER BY ts; +---- +2024-01-01T12:00:00Z 2024-01-01T05:00:00-07:00 Timestamp(ns, "America/Denver") +2024-03-10T08:59:00Z 2024-03-10T01:59:00-07:00 Timestamp(ns, "America/Denver") +2024-03-10T09:00:00Z 2024-03-10T03:00:00-06:00 Timestamp(ns, "America/Denver") +2024-07-01T12:00:00Z 2024-07-01T06:00:00-06:00 Timestamp(ns, "America/Denver") +2024-11-03T07:59:00Z 2024-11-03T01:59:00-06:00 Timestamp(ns, "America/Denver") +2024-11-03T08:00:00Z 2024-11-03T01:00:00-07:00 Timestamp(ns, "America/Denver") + +# `now()` is tz-aware whenever `datafusion.execution.time_zone` is set. +query T +SELECT arrow_typeof(now() AT TIME ZONE 'America/Denver'); +---- +Timestamp(ns, "America/Denver") + +statement ok +DROP TABLE at_tz_dst; + +statement ok +DROP TABLE at_tz_t; + +statement ok +RESET datafusion.execution.time_zone; + + # Test current_time without parentheses query B select current_time = current_time; From 8e702a8a9e686be3a66a442c94caa5c60d40b81a Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:25:28 -0500 Subject: [PATCH 2/4] fix: `AT TIME ZONE` on a tz-aware timestamp returns a naive timestamp `expr AT TIME ZONE 'tz'` was unconditionally lowered to `CAST(expr AS Timestamp(Nanosecond, Some(tz)))`. That is right for a timezone-naive input -- arrow's `Timestamp(_, None) -> Timestamp(_, Some(tz))` cast reads the naive value as local time in `tz` -- but wrong for a timezone-aware one, where it merely relabels the display zone. PostgreSQL (and DuckDB) make the operator asymmetric: `timestamptz AT TIME ZONE zone` returns a `timestamp` (naive) holding the wall clock in `zone`. Because DataFusion kept the value timezone-aware, casting the result to `::timestamp` produced the UTC wall clock instead of the zone's, which is what apache/datafusion#12218 reports. The SQL planner now types the input and branches on it. The naive case is unchanged. The aware case relabels the instant into `tz` with the same cast and then drops the timezone while keeping the displayed value, which is exactly `to_local_time`. `datafusion-sql` must not depend on `datafusion-functions`, so that second half goes through a new `ExprPlanner::plan_at_time_zone` hook -- the same shape as `plan_extract` lowering `EXTRACT` to `date_part` -- implemented by `DatetimeFunctionPlanner`. Sessions without it get a clear planning error instead of the old silent mislowering. `AT TIME ZONE` also no longer forces `Nanosecond`: it keeps the input's `TimeUnit` when the input is a timestamp. Closes https://github.com/apache/datafusion/issues/12218 Co-Authored-By: Claude Opus 5 --- datafusion/expr/src/planner.rs | 20 ++++ datafusion/functions/src/datetime/planner.rs | 16 +++ datafusion/sql/src/expr/mod.rs | 105 +++++++++++++++--- datafusion/sql/tests/sql_integration.rs | 33 ++++++ .../test_files/datetime/timestamps.slt | 35 +++--- 5 files changed, 172 insertions(+), 37 deletions(-) diff --git a/datafusion/expr/src/planner.rs b/datafusion/expr/src/planner.rs index 7aaf3a98cbe5d..c196894d77023 100644 --- a/datafusion/expr/src/planner.rs +++ b/datafusion/expr/src/planner.rs @@ -218,6 +218,26 @@ pub trait ExprPlanner: Debug + Send + Sync { Ok(PlannerResult::Original(args)) } + /// Plan ` AT TIME ZONE ''` when `` is **already + /// timezone-aware**. + /// + /// Following PostgreSQL, ` AT TIME ZONE zone` returns + /// the wall clock the instant has in `zone`, as a timezone-*naive* + /// timestamp. (The timezone-*naive* case is the plain + /// `CAST( AS Timestamp(unit, Some(tz)))` that the SQL planner + /// handles on its own and never reaches this method.) + /// + /// `args` holds a single expression: the input already cast to + /// `Timestamp(unit, Some(tz))`. That cast preserves the instant and only + /// relabels the timezone, so all an implementation has to do is drop the + /// timezone while keeping the displayed value — exactly what + /// `to_local_time` does. + /// + /// Returns original expression arguments if not possible + fn plan_at_time_zone(&self, args: Vec) -> Result>> { + Ok(PlannerResult::Original(args)) + } + /// Plans a struct literal, such as `{'field1' : expr1, 'field2' : expr2, ...}` /// /// This function takes a vector of expressions and a boolean flag diff --git a/datafusion/functions/src/datetime/planner.rs b/datafusion/functions/src/datetime/planner.rs index f2b8ef9d1d310..c28d33199f76b 100644 --- a/datafusion/functions/src/datetime/planner.rs +++ b/datafusion/functions/src/datetime/planner.rs @@ -32,4 +32,20 @@ impl ExprPlanner for DatetimeFunctionPlanner { ScalarFunction::new_udf(crate::datetime::date_part(), args), ))) } + + /// ` AT TIME ZONE 'tz'` returns the wall clock in `tz` + /// as a timezone-naive timestamp, matching PostgreSQL. + /// + /// The SQL planner hands us the input already cast to + /// `Timestamp(unit, Some(tz))` — an instant-preserving relabel — so the + /// remaining step is to strip the timezone while keeping the displayed + /// value, which is `to_local_time`. + fn plan_at_time_zone( + &self, + args: Vec, + ) -> datafusion_common::Result>> { + Ok(PlannerResult::Planned(Expr::ScalarFunction( + ScalarFunction::new_udf(crate::datetime::to_local_time(), args), + ))) + } } diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index f1661028ed051..524277fd3fb95 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -16,6 +16,7 @@ // under the License. use std::ops::ControlFlow; +use std::sync::Arc; use arrow::datatypes::{DataType, TimeUnit}; use datafusion_expr::planner::{ @@ -659,24 +660,12 @@ impl SqlToRel<'_, S> { SQLExpr::AtTimeZone { timestamp, time_zone, - } => Ok(Expr::Cast(Cast::new( - Box::new(self.sql_expr_to_logical_expr_internal( - *timestamp, - schema, - planner_context, - )?), - match *time_zone { - SQLExpr::Value(ValueWithSpan { - value: Value::SingleQuotedString(s), - span: _, - }) => DataType::Timestamp(TimeUnit::Nanosecond, Some(s.into())), - _ => { - return not_impl_err!( - "Unsupported ast node in sqltorel: {time_zone:?}" - ); - } - }, - ))), + } => self.sql_at_time_zone_to_expr( + *timestamp, + *time_zone, + schema, + planner_context, + ), SQLExpr::Dictionary(fields) => { self.try_plan_dictionary_literal(fields, schema, planner_context) } @@ -814,6 +803,86 @@ impl SqlToRel<'_, S> { } } + /// Plan ` AT TIME ZONE ''`. + /// + /// The meaning of `AT TIME ZONE` depends on whether its input carries a + /// timezone, and it always returns the *other* kind of timestamp. This + /// follows PostgreSQL (and DuckDB): + /// + /// * a timezone-**naive** input is read as a wall clock in `tz`, and the + /// result is the corresponding timezone-**aware** instant. That is a + /// plain `CAST(expr AS Timestamp(unit, Some(tz)))`, because arrow's + /// `Timestamp(_, None) -> Timestamp(_, Some(tz))` cast interprets the + /// naive value as local time in `tz`. + /// * a timezone-**aware** input is an instant, and the result is the wall + /// clock that instant has in `tz`, as a timezone-**naive** timestamp. + /// The same cast is still the first half of that (casting between two + /// aware types preserves the instant and only relabels the zone); the + /// second half — dropping the zone while keeping the displayed value — + /// is delegated to [`ExprPlanner::plan_at_time_zone`], which + /// `datafusion-functions` implements with `to_local_time`. + /// + /// Anything that is not a timestamp (a string literal, for instance) takes + /// the naive path, since a `CAST` to a timezone-aware timestamp is the + /// natural reading of `AT TIME ZONE` for it. + /// + /// [`ExprPlanner::plan_at_time_zone`]: datafusion_expr::planner::ExprPlanner::plan_at_time_zone + fn sql_at_time_zone_to_expr( + &self, + timestamp: SQLExpr, + time_zone: SQLExpr, + schema: &DFSchema, + planner_context: &mut PlannerContext, + ) -> Result { + let tz: Arc = match time_zone { + SQLExpr::Value(ValueWithSpan { + value: Value::SingleQuotedString(s), + span: _, + }) => s.into(), + _ => { + return not_impl_err!("Unsupported ast node in sqltorel: {time_zone:?}"); + } + }; + + let expr = + self.sql_expr_to_logical_expr_internal(timestamp, schema, planner_context)?; + + // `AT TIME ZONE` does not change the precision of its input, so keep + // the input's `TimeUnit` when it has one. + let (unit, input_is_tz_aware) = match expr.get_type(schema)? { + DataType::Timestamp(unit, tz) => (unit, tz.is_some()), + _ => (TimeUnit::Nanosecond, false), + }; + + // Instant-preserving relabel into `tz` for an aware input; local-time + // interpretation for a naive one. + let relabeled = Expr::Cast(Cast::new( + Box::new(expr), + DataType::Timestamp(unit, Some(tz)), + )); + + if !input_is_tz_aware { + return Ok(relabeled); + } + + let mut args = vec![relabeled]; + for planner in self.context_provider.get_expr_planners() { + match planner.plan_at_time_zone(args)? { + PlannerResult::Planned(expr) => return Ok(expr), + PlannerResult::Original(original) => { + args = original; + } + } + } + + plan_err!( + "AT TIME ZONE on a timezone-aware timestamp is not supported by any \ + ExprPlanner. It needs the `to_local_time` function; register \ + `datafusion_functions::datetime` (or its `DatetimeFunctionPlanner`) \ + with the session" + ) + } + fn sql_position_to_expr( &self, substr_expr: SQLExpr, diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs index 00103bfd9f56a..35070a22b18f1 100644 --- a/datafusion/sql/tests/sql_integration.rs +++ b/datafusion/sql/tests/sql_integration.rs @@ -3805,6 +3805,39 @@ fn plan_merge_into_rejects_invalid_actions_and_structure( ); } +/// A timezone-naive input is read as a wall clock in the target timezone, so it +/// lowers to a plain `CAST` and preserves the input's `TimeUnit`. +#[test] +fn plan_at_time_zone_on_naive_timestamp() { + let plan = + logical_plan("SELECT birth_date AT TIME ZONE 'America/Denver' FROM person") + .unwrap(); + assert_snapshot!( + plan, + @r#" + Projection: CAST(person.birth_date AS Timestamp(ns, "America/Denver")) + TableScan: person + "# + ); +} + +/// A timezone-aware input needs `to_local_time`, which is provided by +/// `datafusion-functions`' `DatetimeFunctionPlanner`. The mock context provider +/// used by these tests does not register it, so planning must fail with a clear +/// message rather than silently falling back to the naive lowering. +#[test] +fn plan_at_time_zone_on_tz_aware_timestamp_without_planner() { + let err = logical_plan( + "SELECT (birth_date AT TIME ZONE 'UTC') AT TIME ZONE 'America/Denver' FROM person", + ) + .unwrap_err(); + assert!( + err.strip_backtrace() + .contains("AT TIME ZONE on a timezone-aware timestamp"), + "unexpected error: {err}" + ); +} + fn logical_plan(sql: &str) -> Result { logical_plan_with_options(sql, ParserOptions::default()) } diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index de6e599d026d5..cc827a287294a 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -4061,9 +4061,7 @@ SELECT '2023-03-12 02:00:00' AT TIME ZONE 'EDT'; ## * ` AT TIME ZONE zone` returns the wall clock that the ## instant has in `zone`, as a tz-*naive* timestamp. ## -## The expectations below record DataFusion's behaviour *before* the fix for -## #12218: the tz-aware case wrongly stays tz-aware, which only relabels the -## display zone. The following commit flips them to the PostgreSQL results. +## Every result below was checked against PostgreSQL 17. ########## statement ok @@ -4086,13 +4084,13 @@ Timestamp(ns, "America/Denver") 2024-01-01T12:00:00-07:00 query TP SELECT arrow_typeof(tstz AT TIME ZONE 'America/Denver'), tstz AT TIME ZONE 'America/Denver' FROM at_tz_t; ---- -Timestamp(ns, "America/Denver") 2024-01-01T05:00:00-07:00 +Timestamp(ns) 2024-01-01T05:00:00 # The exact reproducer from #12218. PostgreSQL returns `2024-01-01 05:00:00`. query P SELECT (tstz AT TIME ZONE 'America/Denver')::timestamp FROM at_tz_t; ---- -2024-01-01T12:00:00 +2024-01-01T05:00:00 # Chained. PostgreSQL: `timestamptz` `2024-01-01 04:00:00+00`, i.e. the Denver # wall clock (05:00) re-read as a Brussels wall clock. @@ -4102,7 +4100,7 @@ SELECT tstz AT TIME ZONE 'America/Denver' AT TIME ZONE 'Europe/Brussels' FROM at_tz_t; ---- -Timestamp(ns, "Europe/Brussels") 2024-01-01T13:00:00+01:00 +Timestamp(ns, "Europe/Brussels") 2024-01-01T05:00:00+01:00 # Fixed offset. Note DataFusion follows the arrow/ISO-8601 sign convention here # (`+05:30` is 5h30m *east* of UTC), while PostgreSQL applies the POSIX @@ -4111,22 +4109,21 @@ Timestamp(ns, "Europe/Brussels") 2024-01-01T13:00:00+01:00 query TP SELECT arrow_typeof(tstz AT TIME ZONE '+05:30'), tstz AT TIME ZONE '+05:30' FROM at_tz_t; ---- -Timestamp(ns, "+05:30") 2024-01-01T17:30:00+05:30 +Timestamp(ns) 2024-01-01T17:30:00 -# `AT TIME ZONE` currently forces Nanosecond precision regardless of the input. +# `AT TIME ZONE` preserves the precision of its input. query T SELECT arrow_typeof(arrow_cast('2024-01-01T12:00:00Z', 'Timestamp(Microsecond, Some("UTC"))') AT TIME ZONE 'America/Denver'); ---- -Timestamp(ns, "America/Denver") +Timestamp(µs) query T SELECT arrow_typeof(arrow_cast('2024-01-01T12:00:00', 'Timestamp(Microsecond, None)') AT TIME ZONE 'America/Denver'); ---- -Timestamp(ns, "America/Denver") +Timestamp(µs, "America/Denver") # A real (multi-row) tz-aware column, spanning both DST transitions in -# America/Denver. The wall clocks below all agree with PostgreSQL; it is only -# the type that is wrong today. +# America/Denver. statement ok CREATE TABLE at_tz_dst(ts timestamptz) AS VALUES (arrow_cast('2024-01-01T12:00:00Z', 'Timestamp(Nanosecond, Some("UTC"))')), @@ -4143,18 +4140,18 @@ SELECT arrow_typeof(ts AT TIME ZONE 'America/Denver') AS denver_type FROM at_tz_dst ORDER BY ts; ---- -2024-01-01T12:00:00Z 2024-01-01T05:00:00-07:00 Timestamp(ns, "America/Denver") -2024-03-10T08:59:00Z 2024-03-10T01:59:00-07:00 Timestamp(ns, "America/Denver") -2024-03-10T09:00:00Z 2024-03-10T03:00:00-06:00 Timestamp(ns, "America/Denver") -2024-07-01T12:00:00Z 2024-07-01T06:00:00-06:00 Timestamp(ns, "America/Denver") -2024-11-03T07:59:00Z 2024-11-03T01:59:00-06:00 Timestamp(ns, "America/Denver") -2024-11-03T08:00:00Z 2024-11-03T01:00:00-07:00 Timestamp(ns, "America/Denver") +2024-01-01T12:00:00Z 2024-01-01T05:00:00 Timestamp(ns) +2024-03-10T08:59:00Z 2024-03-10T01:59:00 Timestamp(ns) +2024-03-10T09:00:00Z 2024-03-10T03:00:00 Timestamp(ns) +2024-07-01T12:00:00Z 2024-07-01T06:00:00 Timestamp(ns) +2024-11-03T07:59:00Z 2024-11-03T01:59:00 Timestamp(ns) +2024-11-03T08:00:00Z 2024-11-03T01:00:00 Timestamp(ns) # `now()` is tz-aware whenever `datafusion.execution.time_zone` is set. query T SELECT arrow_typeof(now() AT TIME ZONE 'America/Denver'); ---- -Timestamp(ns, "America/Denver") +Timestamp(ns) statement ok DROP TABLE at_tz_dst; From 3152108065b7bfe27b3ecbaab56e98587378d895 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:25:33 -0500 Subject: [PATCH 3/4] docs: document `AT TIME ZONE` semantics Adds an `AT TIME ZONE` entry to the SQL operators guide covering both directions (naive -> aware, aware -> naive), precision preservation, chaining and the ISO-vs-POSIX sign convention for fixed-offset strings. `to_local_time`'s own examples all apply `AT TIME ZONE` to timezone-*naive* values, so they are unaffected by the semantic change; its description now says so, and notes that wrapping an already timezone-aware `AT TIME ZONE` in `to_local_time` is redundant. Co-Authored-By: Claude Opus 5 --- .../functions/src/datetime/to_local_time.rs | 6 +- docs/source/user-guide/sql/operators.md | 62 +++++++++++++++++++ .../source/user-guide/sql/scalar_functions.md | 4 ++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/datafusion/functions/src/datetime/to_local_time.rs b/datafusion/functions/src/datetime/to_local_time.rs index 973afa549e0b2..ad49638c0d906 100644 --- a/datafusion/functions/src/datetime/to_local_time.rs +++ b/datafusion/functions/src/datetime/to_local_time.rs @@ -44,7 +44,11 @@ use datafusion_macros::user_doc; /// while keep the display value of the timestamp the same. #[user_doc( doc_section(label = "Time and Date Functions"), - description = "Converts a timestamp with a timezone to a timestamp without a timezone (with no offset or timezone information). This function handles daylight saving time changes.", + description = r#"Converts a timestamp with a timezone to a timestamp without a timezone (with no offset or timezone information). This function handles daylight saving time changes. + +A timestamp that already has no timezone is returned unchanged. + +`AT TIME ZONE` applied to a timezone-*aware* timestamp is defined in terms of this function: it relabels the instant into the target timezone and then applies `to_local_time`. Wrapping such an expression in `to_local_time` is therefore redundant. The examples below instead apply `AT TIME ZONE` to timezone-*naive* values, which is the form that produces a timezone-aware timestamp for `to_local_time` to strip."#, syntax_example = "to_local_time(expression)", sql_example = r#"```sql > SELECT to_local_time('2024-04-01T00:00:20Z'::timestamp); diff --git a/docs/source/user-guide/sql/operators.md b/docs/source/user-guide/sql/operators.md index b63f552396211..c6432b7d9add9 100644 --- a/docs/source/user-guide/sql/operators.md +++ b/docs/source/user-guide/sql/operators.md @@ -507,6 +507,7 @@ Bitwise Shift Left - [|| (string concatenation)](#op_str_cat) - [@> (array contains)](#op_arr_contains) - [<@ (array is contained by)](#op_arr_contained_by) +- [AT TIME ZONE](#op_at_time_zone) (op_str_cat)= @@ -553,6 +554,67 @@ Array Is Contained By +-------------------------------------------------------------------------+ ``` +(op_at_time_zone)= + +### `AT TIME ZONE` + +Converts between timezone-naive and timezone-aware timestamps, following +PostgreSQL. `AT TIME ZONE` always returns the _other_ kind of timestamp from +the one it is given: + +| Input type | Result type | Meaning | +| ---------------------------------------- | ---------------------------------------- | ------------------------------------------------------------------- | +| `Timestamp(unit, None)` (timezone-naive) | `Timestamp(unit, Some(tz))` (tz-aware) | Read the value as a wall clock in `tz`; the result is that instant. | +| `Timestamp(unit, Some(_))` (tz-aware) | `Timestamp(unit, None)` (timezone-naive) | Return the wall clock that instant has in `tz`. | + +A timezone-naive timestamp is a wall clock with no instant attached, so +`AT TIME ZONE` pins it to one: + +```sql +> SET datafusion.execution.time_zone = 'UTC'; +> SELECT + '2024-01-01 12:00:00'::timestamp AT TIME ZONE 'America/Denver' AS instant, + arrow_typeof('2024-01-01 12:00:00'::timestamp AT TIME ZONE 'America/Denver') AS type; ++---------------------------+---------------------------------+ +| instant | type | ++---------------------------+---------------------------------+ +| 2024-01-01T12:00:00-07:00 | Timestamp(ns, "America/Denver") | ++---------------------------+---------------------------------+ +``` + +A timezone-aware timestamp already is an instant, so `AT TIME ZONE` reads off +its wall clock in `tz` and drops the timezone. Daylight saving time is taken +into account: + +```sql +> SELECT + '2024-01-01T12:00:00Z'::timestamptz AT TIME ZONE 'America/Denver' AS wall_clock, + arrow_typeof('2024-01-01T12:00:00Z'::timestamptz AT TIME ZONE 'America/Denver') AS type; ++---------------------+---------------+ +| wall_clock | type | ++---------------------+---------------+ +| 2024-01-01T05:00:00 | Timestamp(ns) | ++---------------------+---------------+ +``` + +The second form is equivalent to [`to_local_time`] applied to the timestamp +after it has been relabelled into `tz`, and requires `to_local_time` to be +registered with the session. + +Because the two forms return different types, applying `AT TIME ZONE` twice +returns a timezone-aware timestamp again: the wall clock produced by the first +application is re-read as a local time in the second timezone. + +`AT TIME ZONE` never changes the precision (`TimeUnit`) of its input. Inputs +that are not timestamps at all (a string literal, for example) are cast to +`Timestamp(Nanosecond, Some(tz))`. + +Note that a timezone written as a fixed offset string follows the ISO 8601 +convention, so `'+05:30'` is 5 hours 30 minutes _east_ of UTC. PostgreSQL +applies the opposite (POSIX) convention to offsets spelled this way. + +[`to_local_time`]: scalar_functions.md#to_local_time + ## Literals Use single quotes for literal values. For example, the string `foo bar` is diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index bdc8efb1bd15d..f183f547b4f82 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -2855,6 +2855,10 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo Converts a timestamp with a timezone to a timestamp without a timezone (with no offset or timezone information). This function handles daylight saving time changes. +A timestamp that already has no timezone is returned unchanged. + +`AT TIME ZONE` applied to a timezone-_aware_ timestamp is defined in terms of this function: it relabels the instant into the target timezone and then applies `to_local_time`. Wrapping such an expression in `to_local_time` is therefore redundant. The examples below instead apply `AT TIME ZONE` to timezone-_naive_ values, which is the form that produces a timezone-aware timestamp for `to_local_time` to strip. + ```sql to_local_time(expression) ``` From 3e42f8b1029a341484749df54b60a74a5fe4164d Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:15:00 -0500 Subject: [PATCH 4/4] test: pin the CASE limitation in AT TIME ZONE dispatch Review of this PR found that the fix does not reach every expression. The type branch calls `Expr::get_type` in the SQL planner, which runs before the type coercion analyzer. For a `CASE`, `get_type` reports the first non-null `THEN` arm and ignores coercion. So these two have the same coerced input type, `Timestamp(ns, "UTC")`, and give different answers: CASE WHEN b THEN naive ELSE aware END AT TIME ZONE 'America/Denver' -> Timestamp(ns, "America/Denver") 2024-01-01T05:00:00-07:00 CASE WHEN b THEN aware ELSE naive END AT TIME ZONE 'America/Denver' -> Timestamp(ns) 2024-01-01T05:00:00 PostgreSQL gives the naive `2024-01-01 05:00:00` for both. `coalesce` is not affected, because `verify_function_arguments` coerces before the planner sees the type. A correct fix has to dispatch after coercion, which is a larger change than this PR. Pinning the behaviour so it is visible rather than silent. Co-Authored-By: Claude Opus 5 --- .../test_files/datetime/timestamps.slt | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index cc827a287294a..1a4fc41561f37 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -5643,3 +5643,41 @@ query P SELECT date_bin(NULL, TIMESTAMP '2023-01-01 12:30:00', TIMESTAMP '2023-01-01 12:00:00') ---- NULL + +# KNOWN LIMITATION: `AT TIME ZONE` chooses its behaviour from the type that +# `Expr::get_type` reports in the SQL planner, which runs BEFORE the type +# coercion analyzer. For a `CASE` expression, `get_type` reports the type of the +# first non-null `THEN` arm and ignores coercion. So the two queries below have +# the same coerced input type, `Timestamp(ns, "UTC")`, but give different +# results: the first still takes the timezone-naive path and keeps the old +# aware-in aware-out shape. +# +# PostgreSQL gives `2024-01-01 05:00:00` as a naive timestamp for both. +# A correct fix has to dispatch after coercion. Tracked here so the behaviour is +# visible rather than silent. +statement ok +CREATE TABLE at_tz_case AS +SELECT arrow_cast('2024-01-01T12:00:00Z', 'Timestamp(Nanosecond, Some("UTC"))') AS aware, + arrow_cast('2024-01-01T12:00:00', 'Timestamp(Nanosecond, None)') AS naive, + true AS b + +# both CASE expressions have the same type +query TT +SELECT arrow_typeof(CASE WHEN b THEN naive ELSE aware END), + arrow_typeof(CASE WHEN b THEN aware ELSE naive END) +FROM at_tz_case +---- +Timestamp(ns, "UTC") Timestamp(ns, "UTC") + +# but AT TIME ZONE does not agree on them +query TPTP +SELECT arrow_typeof((CASE WHEN b THEN naive ELSE aware END) AT TIME ZONE 'America/Denver'), + (CASE WHEN b THEN naive ELSE aware END) AT TIME ZONE 'America/Denver', + arrow_typeof((CASE WHEN b THEN aware ELSE naive END) AT TIME ZONE 'America/Denver'), + (CASE WHEN b THEN aware ELSE naive END) AT TIME ZONE 'America/Denver' +FROM at_tz_case +---- +Timestamp(ns, "America/Denver") 2024-01-01T05:00:00-07:00 Timestamp(ns) 2024-01-01T05:00:00 + +statement ok +DROP TABLE at_tz_case