diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fb786b..9f743cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ v0.3.1 (Unreleased) --- +### Query translation +* **`toStartOf*` date-time functions** via `EF.Functions`: `ToStartOfYear`, `ToStartOfQuarter`, `ToStartOfMonth`, `ToStartOfWeek` (with optional week `mode`), `ToStartOfDay`, `ToStartOfHour`, `ToStartOfMinute`, `ToStartOfSecond`, the fixed buckets `ToStartOfFiveMinutes` / `ToStartOfTenMinutes` / `ToStartOfFifteenMinutes`, and the general `ToStartOfInterval(source, value, unit)`. Each maps to the matching ClickHouse function and works in `GROUP BY`. Return types follow ClickHouse: the calendar buckets (`Year`/`Quarter`/`Month`/`Week`) return `Date`, the day/hour/minute buckets return `DateTime`, and `ToStartOfSecond` returns `DateTime64`. All accept `DateTime`/`DateTime64` columns, and the plain truncation functions also accept `DateOnly`; `ToStartOfInterval` requires a `DateTime`/`DateTime64` column on older ClickHouse, which rejects a `DateOnly` (Date/Date32) source with `Illegal type Date32 of 1st argument` for every unit; recent versions accept it. `ToStartOfInterval` takes a `ClickHouseInterval` unit (`Second`…`Year`) and emits `toStartOfInterval(source, toInterval(value))`; the unit must be a constant so it can be translated. The default `Date`/`DateTime` result types only span 1970–2149/2106, so ClickHouse narrows out-of-range values — enable `enable_extended_results_for_datetime_functions` (e.g. `set_enable_extended_results_for_datetime_functions=1` in the connection string) for range-preserving `Date32`/`DateTime64` results. + ### Bug fixes * `Sum`/`SumAsync` over a `double` or `float` column no longer throws `InvalidCastException`. EF Core wraps a top-level aggregate so the empty case returns `0`, supplying that fallback as a boxed `Int32` carrying the `Float64`/`Float32` mapping; the literal generators now convert rather than unbox. The `Float32` read path also converts, since ClickHouse widens `sum(Float32)` to `Float64` (which the driver's `GetFloat()` refuses to downcast). ([#46](https://github.com/ClickHouse/ClickHouse.EntityFrameworkCore/issues/46)) * **SummingMergeTree with multiple sum columns**: `HasSummingMergeTreeEngine("A", "B")` now generates valid DDL (`SummingMergeTree((A, B))`). Previously it emitted a comma-separated argument list (`SummingMergeTree(A, B)`), which ClickHouse rejects with `NUMBER_OF_ARGUMENTS_DOESNT_MATCH`. Single-column and no-column usage are unaffected. diff --git a/README.md b/README.md index 1247051..e2f60db 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,33 @@ This provider is in active development. It supports **LINQ queries**, **inserts* `Math.Abs`, `Floor`, `Ceiling`, `Round`, `Truncate`, `Pow`, `Sqrt`, `Cbrt`, `Exp`, `Log`, `Log2`, `Log10`, `Sign`, `Sin`, `Cos`, `Tan`, `Asin`, `Acos`, `Atan`, `Atan2`, `RadiansToDegrees`, `DegreesToRadians`, `IsNaN`, `IsInfinity`, `IsFinite`, `IsPositiveInfinity`, `IsNegativeInfinity` — with both `Math` and `MathF` overloads. +### Date/Time Functions + +The ClickHouse `toStartOf*` family is exposed through `EF.Functions`, so you can bucket and truncate timestamps directly in queries, including in `GROUP BY`: + +`ToStartOfYear`, `ToStartOfQuarter`, `ToStartOfMonth`, `ToStartOfWeek` (with an optional ClickHouse week `mode`), `ToStartOfDay`, `ToStartOfHour`, `ToStartOfMinute`, `ToStartOfSecond`, `ToStartOfFiveMinutes`, `ToStartOfTenMinutes`, `ToStartOfFifteenMinutes`, and the general `ToStartOfInterval(source, value, unit)`. + +```csharp +// Truncate to the start of the month +var monthly = await ctx.Events + .Select(e => EF.Functions.ToStartOfMonth(e.Timestamp)) + .ToListAsync(); + +// Bucket into 15-minute intervals and count per bucket +var buckets = await ctx.Events + .GroupBy(e => EF.Functions.ToStartOfInterval(e.Timestamp, 15, ClickHouseInterval.Minute)) + .Select(g => new { Bucket = g.Key, Count = g.Count() }) + .ToListAsync(); +``` + +`ToStartOfInterval` takes a `ClickHouseInterval` unit (`Second`, `Minute`, `Hour`, `Day`, `Week`, `Month`, `Quarter`, `Year`) — from the `ClickHouse.EntityFrameworkCore.Metadata` namespace — and emits `toStartOfInterval(source, toInterval(value))`. The unit must be a constant. + +Input and return types follow ClickHouse. The calendar buckets (`ToStartOfYear`/`Quarter`/`Month`/`Week`) return `Date`; `ToStartOfDay` and the hour/minute buckets return `DateTime`; `ToStartOfSecond` returns `DateTime64`. They all accept `DateTime` and `DateTime64` columns, and the plain truncation functions also accept `DateOnly` (Date/Date32). `ToStartOfInterval` is the exception: older ClickHouse rejects a `DateOnly` (Date/Date32) source with `Illegal type Date32 of 1st argument` while recent versions accept it. Prefer a `DateTime`/`DateTime64` column for interval bucketing. + +> **Date range:** those default result types (`Date`, `DateTime`) only span 1970–2149/2106, so ClickHouse **narrows values outside that window** — a pre-1970 date is clamped to the epoch (calendar buckets) or wraps around (sub-day/interval buckets). To preserve the full range, enable [`enable_extended_results_for_datetime_functions`](https://clickhouse.com/docs/operations/settings/settings#enable_extended_results_for_datetime_functions) for your session — e.g. add `set_enable_extended_results_for_datetime_functions=1` to the connection string — which makes ClickHouse return `Date32`/`DateTime64` instead. + +`ToStartOfWeek` defaults to ClickHouse week mode `0` (Sunday-based); pass a `mode` to change it. + ### INSERT via SaveChanges `SaveChanges` supports INSERT operations using the driver's native `InsertBinaryAsync` API — RowBinary encoding with GZip compression, far more efficient than parameterized SQL. diff --git a/RELEASENOTES.md b/RELEASENOTES.md index e0043a5..bb96fc6 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,5 +1,8 @@ v0.3.1 (Unreleased) --- +### Query translation +* **`toStartOf*` date-time functions** are now translatable through `EF.Functions`, covering the full family: `ToStartOfYear`, `ToStartOfQuarter`, `ToStartOfMonth`, `ToStartOfWeek` (with an optional ClickHouse week `mode`), `ToStartOfDay`, `ToStartOfHour`, `ToStartOfMinute`, `ToStartOfSecond`, the fixed buckets `ToStartOfFiveMinutes` / `ToStartOfTenMinutes` / `ToStartOfFifteenMinutes`, and the general-purpose `ToStartOfInterval(source, value, unit)`. They compose in `GROUP BY` for time bucketing. Input and return types follow ClickHouse: the calendar buckets (`ToStartOfYear`/`Quarter`/`Month`/`Week`) return `Date`, `ToStartOfDay` and the hour/minute buckets return `DateTime`, and `ToStartOfSecond` returns `DateTime64`. All accept `DateTime`/`DateTime64` columns, and the plain truncation functions also accept `DateOnly` (Date/Date32); `ToStartOfInterval` is the exception — older ClickHouse rejects a `DateOnly` (Date/Date32) source with `Illegal type Date32 of 1st argument`, while recent versions accept it, so prefer a `DateTime`/`DateTime64` column for interval bucketing. `ToStartOfInterval` uses a `ClickHouseInterval` enum for the unit and is emitted as `toStartOfInterval(source, toInterval(value))`. Because the default result types (`Date`/`DateTime`) only span 1970–2149/2106, ClickHouse narrows out-of-range values (pre-1970 clamps to the epoch or wraps around); enable `enable_extended_results_for_datetime_functions` in your session (e.g. `set_enable_extended_results_for_datetime_functions=1` in the connection string) to get range-preserving `Date32`/`DateTime64` results. + ### Bug fixes * Summing a `double` or `float` column (`.SumAsync(x => x.Value)`) no longer throws `InvalidCastException`. Two ClickHouse-specific mismatches were biting: EF Core hands the float literal generator a boxed `Int32` `0` as the empty-result fallback, and ClickHouse widens `sum(Float32)` to `Float64` so the driver couldn't read it back as a `float`. Both the literal generation and the `Float32` read path now convert instead of hard-casting. ([#46](https://github.com/ClickHouse/ClickHouse.EntityFrameworkCore/issues/46)) (Thanks to @HotTotem!) * **SummingMergeTree with multiple sum columns** now produces valid DDL. Configuring more than one sum column (`HasSummingMergeTreeEngine("A", "B")`) previously emitted `SummingMergeTree(A, B)`, which ClickHouse rejects with `NUMBER_OF_ARGUMENTS_DOESNT_MATCH` — the engine takes a single optional parameter that must be a tuple of columns. Multiple columns are now wrapped in a tuple (`SummingMergeTree((A, B))`); single-column and no-column usage are unchanged. diff --git a/src/EFCore.ClickHouse/Extensions/ClickHouseDateTimeDbFunctionsExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHouseDateTimeDbFunctionsExtensions.cs new file mode 100644 index 0000000..f02a91e --- /dev/null +++ b/src/EFCore.ClickHouse/Extensions/ClickHouseDateTimeDbFunctionsExtensions.cs @@ -0,0 +1,152 @@ +using ClickHouse.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Diagnostics; + +namespace Microsoft.EntityFrameworkCore; + +/// +/// ClickHouse-specific extension methods for the toStartOf* family of +/// date-time functions. Each method is translated to the corresponding ClickHouse SQL function and cannot +/// be evaluated on the client. +/// +public static class ClickHouseDateTimeDbFunctionsExtensions +{ + /// + /// Rounds a date or date with time down to the first day of the year. + /// Maps to ClickHouse: toStartOfYear(source). + /// + /// The instance. + /// The date or date-time value to truncate. + [DbFunction("toStartOfYear")] + public static T ToStartOfYear(this DbFunctions _, T source) => + throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(ToStartOfYear))); + + /// + /// Rounds a date or date with time down to the first day of the quarter. + /// Maps to ClickHouse: toStartOfQuarter(source). + /// + /// The instance. + /// The date or date-time value to truncate. + [DbFunction("toStartOfQuarter")] + public static T ToStartOfQuarter(this DbFunctions _, T source) => + throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(ToStartOfQuarter))); + + /// + /// Rounds a date or date with time down to the first day of the month. + /// Maps to ClickHouse: toStartOfMonth(source). + /// + /// The instance. + /// The date or date-time value to truncate. + [DbFunction("toStartOfMonth")] + public static T ToStartOfMonth(this DbFunctions _, T source) => + throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(ToStartOfMonth))); + + /// + /// Rounds a date or date with time down to the start of the week. The default ClickHouse week mode is + /// 0, which treats Sunday as the first day of the week; use the mode overload to change this. + /// Maps to ClickHouse: toStartOfWeek(source). + /// + /// The instance. + /// The date or date-time value to truncate. + [DbFunction("toStartOfWeek")] + public static T ToStartOfWeek(this DbFunctions _, T source) => + throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(ToStartOfWeek))); + + /// + /// Rounds a date or date with time down to the start of the week using the specified week mode. + /// Maps to ClickHouse: toStartOfWeek(source, mode). + /// + /// The instance. + /// The date or date-time value to truncate. + /// The ClickHouse week mode (0-9) that determines the first day of the week. + [DbFunction("toStartOfWeek")] + public static T ToStartOfWeek(this DbFunctions _, T source, byte mode) => + throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(ToStartOfWeek))); + + /// + /// Rounds a date with time down to the start of the day. + /// Maps to ClickHouse: toStartOfDay(source). + /// + /// The instance. + /// The date-time value to truncate. + [DbFunction("toStartOfDay")] + public static T ToStartOfDay(this DbFunctions _, T source) => + throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(ToStartOfDay))); + + /// + /// Rounds a date with time down to the start of the hour. + /// Maps to ClickHouse: toStartOfHour(source). + /// + /// The instance. + /// The date-time value to truncate. + [DbFunction("toStartOfHour")] + public static T ToStartOfHour(this DbFunctions _, T source) => + throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(ToStartOfHour))); + + /// + /// Rounds a date with time down to the start of the minute. + /// Maps to ClickHouse: toStartOfMinute(source). + /// + /// The instance. + /// The date-time value to truncate. + [DbFunction("toStartOfMinute")] + public static T ToStartOfMinute(this DbFunctions _, T source) => + throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(ToStartOfMinute))); + + /// + /// Rounds a date with time down to the start of the second. + /// Maps to ClickHouse: toStartOfSecond(source). + /// + /// ClickHouse toStartOfSecond requires a DateTime64 argument. + /// The instance. + /// The date-time value to truncate. Must map to a ClickHouse DateTime64 column. + [DbFunction("toStartOfSecond")] + public static T ToStartOfSecond(this DbFunctions _, T source) => + throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(ToStartOfSecond))); + + /// + /// Rounds a date with time down to the start of the five-minute interval. + /// Maps to ClickHouse: toStartOfFiveMinutes(source). + /// + /// The instance. + /// The date-time value to truncate. + [DbFunction("toStartOfFiveMinutes")] + public static T ToStartOfFiveMinutes(this DbFunctions _, T source) => + throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(ToStartOfFiveMinutes))); + + /// + /// Rounds a date with time down to the start of the ten-minute interval. + /// Maps to ClickHouse: toStartOfTenMinutes(source). + /// + /// The instance. + /// The date-time value to truncate. + [DbFunction("toStartOfTenMinutes")] + public static T ToStartOfTenMinutes(this DbFunctions _, T source) => + throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(ToStartOfTenMinutes))); + + /// + /// Rounds a date with time down to the start of the fifteen-minute interval. + /// Maps to ClickHouse: toStartOfFifteenMinutes(source). + /// + /// The instance. + /// The date-time value to truncate. + [DbFunction("toStartOfFifteenMinutes")] + public static T ToStartOfFifteenMinutes(this DbFunctions _, T source) => + throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(ToStartOfFifteenMinutes))); + + /// + /// Rounds a date or date with time down to the start of the specified interval. + /// Maps to ClickHouse: toStartOfInterval(source, INTERVAL value unit), emitted as + /// toStartOfInterval(source, toInterval<unit>(value)). + /// + /// The instance. + /// + /// The value to truncate. Prefer a date-time source: older ClickHouse rejects a date-only + /// (Date/Date32) source for toStartOfInterval with + /// Illegal type Date32 of 1st argument while recent versions accept it. + /// + /// The number of interval units in each bucket. + /// The interval unit. Must be a constant so it can be translated to SQL. + [DbFunction("toStartOfInterval")] + public static T ToStartOfInterval(this DbFunctions _, T source, int value, ClickHouseInterval unit) => + throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(ToStartOfInterval))); +} diff --git a/src/EFCore.ClickHouse/Metadata/ClickHouseInterval.cs b/src/EFCore.ClickHouse/Metadata/ClickHouseInterval.cs new file mode 100644 index 0000000..8afaaa9 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/ClickHouseInterval.cs @@ -0,0 +1,33 @@ +namespace ClickHouse.EntityFrameworkCore.Metadata; + +/// +/// Identifies the interval unit used by EF.Functions.ToStartOfInterval, which maps to the +/// ClickHouse toStartOfInterval(t, INTERVAL n unit) function. Each value corresponds to a +/// ClickHouse toInterval* helper (for example emits toIntervalMinute). +/// +public enum ClickHouseInterval +{ + /// Second interval (toIntervalSecond). + Second, + + /// Minute interval (toIntervalMinute). + Minute, + + /// Hour interval (toIntervalHour). + Hour, + + /// Day interval (toIntervalDay). + Day, + + /// Week interval (toIntervalWeek). + Week, + + /// Month interval (toIntervalMonth). + Month, + + /// Quarter interval (toIntervalQuarter). + Quarter, + + /// Year interval (toIntervalYear). + Year +} diff --git a/src/EFCore.ClickHouse/Query/ExpressionTranslators/Internal/ClickHouseDateTimeMethodTranslator.cs b/src/EFCore.ClickHouse/Query/ExpressionTranslators/Internal/ClickHouseDateTimeMethodTranslator.cs new file mode 100644 index 0000000..50c11df --- /dev/null +++ b/src/EFCore.ClickHouse/Query/ExpressionTranslators/Internal/ClickHouseDateTimeMethodTranslator.cs @@ -0,0 +1,168 @@ +using System.Reflection; +using ClickHouse.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Query; +using Microsoft.EntityFrameworkCore.Query.SqlExpressions; +using DateTimeDbFunctions = Microsoft.EntityFrameworkCore.ClickHouseDateTimeDbFunctionsExtensions; + +namespace ClickHouse.EntityFrameworkCore.Query.ExpressionTranslators.Internal; + +/// +/// Translates the EF.Functions.ToStartOf* extension methods +/// () to their ClickHouse SQL functions. +/// +public class ClickHouseDateTimeMethodTranslator : IMethodCallTranslator +{ + private readonly ISqlExpressionFactory _sqlExpressionFactory; + + /// + /// Maps the generic method definitions that take only the source value (and, for + /// ToStartOfWeek(source, mode), an extra scalar argument) directly to a ClickHouse function name. + /// is handled separately. + /// + private static readonly Dictionary SupportedMethods; + + /// Maps values to the ClickHouse toInterval* helper function. + private static readonly Dictionary IntervalFunctions = new() + { + [ClickHouseInterval.Second] = "toIntervalSecond", + [ClickHouseInterval.Minute] = "toIntervalMinute", + [ClickHouseInterval.Hour] = "toIntervalHour", + [ClickHouseInterval.Day] = "toIntervalDay", + [ClickHouseInterval.Week] = "toIntervalWeek", + [ClickHouseInterval.Month] = "toIntervalMonth", + [ClickHouseInterval.Quarter] = "toIntervalQuarter", + [ClickHouseInterval.Year] = "toIntervalYear", + }; + + private static readonly MethodInfo ToStartOfIntervalMethod; + + static ClickHouseDateTimeMethodTranslator() + { + var type = typeof(DateTimeDbFunctions); + SupportedMethods = new Dictionary(); + + // Source-only methods: (DbFunctions, T) -> functionName + void RegisterSourceOnly(string methodName, string sqlFunction) + { + var method = type.GetMethods().FirstOrDefault(m => + { + if (m.Name != methodName || !m.IsGenericMethod) + { + return false; + } + + var parameters = m.GetParameters(); + return parameters.Length == 2 + && parameters[0].ParameterType == typeof(DbFunctions) + && parameters[1].ParameterType.IsGenericParameter; + }) ?? throw new InvalidOperationException($"Method {methodName} with strict signature not found."); + + SupportedMethods.Add(method, sqlFunction); + } + + RegisterSourceOnly(nameof(DateTimeDbFunctions.ToStartOfYear), "toStartOfYear"); + RegisterSourceOnly(nameof(DateTimeDbFunctions.ToStartOfQuarter), "toStartOfQuarter"); + RegisterSourceOnly(nameof(DateTimeDbFunctions.ToStartOfMonth), "toStartOfMonth"); + RegisterSourceOnly(nameof(DateTimeDbFunctions.ToStartOfWeek), "toStartOfWeek"); + RegisterSourceOnly(nameof(DateTimeDbFunctions.ToStartOfDay), "toStartOfDay"); + RegisterSourceOnly(nameof(DateTimeDbFunctions.ToStartOfHour), "toStartOfHour"); + RegisterSourceOnly(nameof(DateTimeDbFunctions.ToStartOfMinute), "toStartOfMinute"); + RegisterSourceOnly(nameof(DateTimeDbFunctions.ToStartOfSecond), "toStartOfSecond"); + RegisterSourceOnly(nameof(DateTimeDbFunctions.ToStartOfFiveMinutes), "toStartOfFiveMinutes"); + RegisterSourceOnly(nameof(DateTimeDbFunctions.ToStartOfTenMinutes), "toStartOfTenMinutes"); + RegisterSourceOnly(nameof(DateTimeDbFunctions.ToStartOfFifteenMinutes), "toStartOfFifteenMinutes"); + + // ToStartOfWeek(source, byte mode) -> toStartOfWeek + var weekWithMode = type.GetMethods().FirstOrDefault(m => + { + if (m.Name != nameof(DateTimeDbFunctions.ToStartOfWeek) || !m.IsGenericMethod) + { + return false; + } + + var parameters = m.GetParameters(); + return parameters.Length == 3 + && parameters[0].ParameterType == typeof(DbFunctions) + && parameters[1].ParameterType.IsGenericParameter + && parameters[2].ParameterType == typeof(byte); + }) ?? throw new InvalidOperationException("Method ToStartOfWeek(source, mode) with strict signature not found."); + SupportedMethods.Add(weekWithMode, "toStartOfWeek"); + + ToStartOfIntervalMethod = type.GetMethods().FirstOrDefault(m => + { + if (m.Name != nameof(DateTimeDbFunctions.ToStartOfInterval) || !m.IsGenericMethod) + { + return false; + } + + var parameters = m.GetParameters(); + return parameters.Length == 4 + && parameters[0].ParameterType == typeof(DbFunctions) + && parameters[1].ParameterType.IsGenericParameter + && parameters[2].ParameterType == typeof(int) + && parameters[3].ParameterType == typeof(ClickHouseInterval); + }) ?? throw new InvalidOperationException("Method ToStartOfInterval with strict signature not found."); + } + + public ClickHouseDateTimeMethodTranslator(ISqlExpressionFactory sqlExpressionFactory) + { + _sqlExpressionFactory = sqlExpressionFactory; + } + + public SqlExpression? Translate( + SqlExpression? instance, + MethodInfo method, + IReadOnlyList arguments, + IDiagnosticsLogger logger) + { + var genericMethod = method.IsGenericMethod ? method.GetGenericMethodDefinition() : method; + + if (SupportedMethods.TryGetValue(genericMethod, out var function)) + { + // arguments[0] is the DbFunctions receiver; the source is arguments[1]. + var sqlArguments = arguments.Skip(1).ToList(); + var source = sqlArguments[0]; + + return _sqlExpressionFactory.Function( + name: function, + arguments: sqlArguments, + nullable: true, + // Only the source value propagates nullability; the optional week mode is a constant. + argumentsPropagateNullability: sqlArguments.Select((_, i) => i == 0), + returnType: method.ReturnType, + typeMapping: source.TypeMapping); + } + + if (genericMethod == ToStartOfIntervalMethod) + { + var source = arguments[1]; + var value = arguments[2]; + + // The interval unit must be a compile-time constant so it can be mapped to a toInterval* function. + if (arguments[3] is not SqlConstantExpression { Value: ClickHouseInterval unit } + || !IntervalFunctions.TryGetValue(unit, out var intervalFunction)) + { + return null; + } + + var interval = _sqlExpressionFactory.Function( + name: intervalFunction, + arguments: [value], + nullable: true, + argumentsPropagateNullability: [false], + returnType: value.Type); + + return _sqlExpressionFactory.Function( + name: "toStartOfInterval", + arguments: [source, interval], + nullable: true, + argumentsPropagateNullability: [true, false], + returnType: method.ReturnType, + typeMapping: source.TypeMapping); + } + + return null; + } +} diff --git a/src/EFCore.ClickHouse/Query/ExpressionTranslators/Internal/ClickHouseMethodCallTranslatorProvider.cs b/src/EFCore.ClickHouse/Query/ExpressionTranslators/Internal/ClickHouseMethodCallTranslatorProvider.cs index 1d1fdec..b6c61fa 100644 --- a/src/EFCore.ClickHouse/Query/ExpressionTranslators/Internal/ClickHouseMethodCallTranslatorProvider.cs +++ b/src/EFCore.ClickHouse/Query/ExpressionTranslators/Internal/ClickHouseMethodCallTranslatorProvider.cs @@ -20,6 +20,7 @@ public ClickHouseMethodCallTranslatorProvider( new ClickHouseMathMethodTranslator(sqlExpressionFactory, typeMappingSource), new ClickHouseJsonNodeTranslator(sqlExpressionFactory, typeMappingSource), new ClickHouseJsonDbFunctionsTranslator(sqlExpressionFactory), + new ClickHouseDateTimeMethodTranslator(sqlExpressionFactory), ]); } } diff --git a/src/EFCore.ClickHouse/Query/Internal/ClickHouseEvaluatableExpressionFilter.cs b/src/EFCore.ClickHouse/Query/Internal/ClickHouseEvaluatableExpressionFilter.cs index b869bd4..3cda25a 100644 --- a/src/EFCore.ClickHouse/Query/Internal/ClickHouseEvaluatableExpressionFilter.cs +++ b/src/EFCore.ClickHouse/Query/Internal/ClickHouseEvaluatableExpressionFilter.cs @@ -19,6 +19,8 @@ public ClickHouseEvaluatableExpressionFilter( { MethodCallExpression methodCallExpression when methodCallExpression.Method.DeclaringType == typeof(ClickHouseJsonDbFunctionsExtensions) => false, + MethodCallExpression methodCallExpression when methodCallExpression.Method.DeclaringType == + typeof(ClickHouseDateTimeDbFunctionsExtensions) => false, NewExpression newExpression => !newExpression.Type.IsAssignableTo(typeof(ITuple)), _ => base.IsEvaluatableExpression(expression, model) }; diff --git a/test/EFCore.ClickHouse.Tests/DateTimeFunctionsTranslationTests.cs b/test/EFCore.ClickHouse.Tests/DateTimeFunctionsTranslationTests.cs new file mode 100644 index 0000000..96122df --- /dev/null +++ b/test/EFCore.ClickHouse.Tests/DateTimeFunctionsTranslationTests.cs @@ -0,0 +1,398 @@ +using ClickHouse.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore; +using Xunit; + +namespace EFCore.ClickHouse.Tests; + +public class DateTimeEntity +{ + public long Id { get; set; } + + /// Mapped to ClickHouse DateTime. + public DateTime Timestamp { get; set; } + + /// Mapped to ClickHouse DateTime64(3). + public DateTime Timestamp64 { get; set; } + + /// Mapped to ClickHouse Date32. + public DateOnly Date { get; set; } +} + +public class DateTimeDbContext : DbContext +{ + public DbSet Events => Set(); + + private readonly string _connectionString; + + public DateTimeDbContext(string connectionString) + { + _connectionString = connectionString; + } + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + { + optionsBuilder.UseClickHouse(_connectionString); + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("datetime_functions_test"); + entity.HasKey(e => e.Id); + entity.Property(e => e.Id).HasColumnName("id"); + entity.Property(e => e.Timestamp).HasColumnName("ts"); + entity.Property(e => e.Timestamp64).HasColumnName("ts64").HasColumnType("DateTime64(3)"); + entity.Property(e => e.Date).HasColumnName("d"); + }); + } +} + +public class DateTimeFixture : IAsyncLifetime +{ + public string ConnectionString { get; private set; } = string.Empty; + + // Reference instant used for all assertions: Monday 2026-08-10 13:47:32.500. + public static readonly DateTime Instant = new(2026, 8, 10, 13, 47, 32); + + public async Task InitializeAsync() + { + ConnectionString = await SharedContainer.GetConnectionStringAsync(); + + using var connection = new global::ClickHouse.Driver.ADO.ClickHouseConnection(ConnectionString); + await connection.OpenAsync(); + + using var createCmd = connection.CreateCommand(); + createCmd.CommandText = """ + CREATE TABLE datetime_functions_test ( + id Int64, + ts DateTime, + ts64 DateTime64(3), + d Date32 + ) ENGINE = MergeTree() + ORDER BY id + """; + await createCmd.ExecuteNonQueryAsync(); + + using var insertCmd = connection.CreateCommand(); + // Row 1 is inside the legacy Date/DateTime window (1970-2106). Row 2 is pre-1970 (only Date32 + // and DateTime64 can hold it; ts is a placeholder) — used to verify how the toStartOf* result + // types narrow the range by ClickHouse default. + insertCmd.CommandText = """ + INSERT INTO datetime_functions_test (id, ts, ts64, d) VALUES + (1, '2026-08-10 13:47:32', '2026-08-10 13:47:32.500', '2026-08-10'), + (2, '1970-01-01 00:00:00', '1920-05-15 12:34:56.500', '1920-05-15') + """; + await insertCmd.ExecuteNonQueryAsync(); + } + + public Task DisposeAsync() => Task.CompletedTask; +} + +public class DateTimeFunctionsTranslationTest : IClassFixture +{ + private readonly DateTimeFixture _fixture; + + public DateTimeFunctionsTranslationTest(DateTimeFixture fixture) + { + _fixture = fixture; + } + + private async Task SelectSingleAsync(Func, IQueryable> selector) + { + await using var context = new DateTimeDbContext(_fixture.ConnectionString); + return await selector(context.Events.AsNoTracking().Where(e => e.Id == 1)).SingleAsync(); + } + + [Fact] + public async Task ToStartOfYear_truncates_to_first_of_year() + { + var result = await SelectSingleAsync(q => q.Select(e => EF.Functions.ToStartOfYear(e.Timestamp))); + Assert.Equal(new DateTime(2026, 1, 1), result); + } + + [Fact] + public async Task ToStartOfQuarter_truncates_to_first_of_quarter() + { + var result = await SelectSingleAsync(q => q.Select(e => EF.Functions.ToStartOfQuarter(e.Timestamp))); + Assert.Equal(new DateTime(2026, 7, 1), result); + } + + [Fact] + public async Task ToStartOfMonth_truncates_to_first_of_month() + { + var result = await SelectSingleAsync(q => q.Select(e => EF.Functions.ToStartOfMonth(e.Timestamp))); + Assert.Equal(new DateTime(2026, 8, 1), result); + } + + [Fact] + public async Task ToStartOfWeek_default_mode_starts_on_sunday() + { + // 2026-08-10 is a Monday; default mode 0 (Sunday-based) rolls back to 2026-08-09. + var result = await SelectSingleAsync(q => q.Select(e => EF.Functions.ToStartOfWeek(e.Timestamp))); + Assert.Equal(new DateTime(2026, 8, 9), result); + } + + [Fact] + public async Task ToStartOfWeek_mode_one_starts_on_monday() + { + // Mode 1 (Monday-based); 2026-08-10 is itself a Monday. + var result = await SelectSingleAsync(q => q.Select(e => EF.Functions.ToStartOfWeek(e.Timestamp, 1))); + Assert.Equal(new DateTime(2026, 8, 10), result); + } + + [Fact] + public async Task ToStartOfDay_truncates_to_midnight() + { + var result = await SelectSingleAsync(q => q.Select(e => EF.Functions.ToStartOfDay(e.Timestamp))); + Assert.Equal(new DateTime(2026, 8, 10, 0, 0, 0), result); + } + + [Fact] + public async Task ToStartOfHour_truncates_to_hour() + { + var result = await SelectSingleAsync(q => q.Select(e => EF.Functions.ToStartOfHour(e.Timestamp))); + Assert.Equal(new DateTime(2026, 8, 10, 13, 0, 0), result); + } + + [Fact] + public async Task ToStartOfMinute_truncates_to_minute() + { + var result = await SelectSingleAsync(q => q.Select(e => EF.Functions.ToStartOfMinute(e.Timestamp))); + Assert.Equal(new DateTime(2026, 8, 10, 13, 47, 0), result); + } + + [Fact] + public async Task ToStartOfSecond_truncates_subsecond() + { + // toStartOfSecond requires a DateTime64 argument. + var result = await SelectSingleAsync(q => q.Select(e => EF.Functions.ToStartOfSecond(e.Timestamp64))); + Assert.Equal(new DateTime(2026, 8, 10, 13, 47, 32), result); + } + + [Fact] + public async Task ToStartOfFiveMinutes_truncates_to_five_minute_bucket() + { + var result = await SelectSingleAsync(q => q.Select(e => EF.Functions.ToStartOfFiveMinutes(e.Timestamp))); + Assert.Equal(new DateTime(2026, 8, 10, 13, 45, 0), result); + } + + [Fact] + public async Task ToStartOfTenMinutes_truncates_to_ten_minute_bucket() + { + var result = await SelectSingleAsync(q => q.Select(e => EF.Functions.ToStartOfTenMinutes(e.Timestamp))); + Assert.Equal(new DateTime(2026, 8, 10, 13, 40, 0), result); + } + + [Fact] + public async Task ToStartOfFifteenMinutes_truncates_to_fifteen_minute_bucket() + { + var result = await SelectSingleAsync(q => q.Select(e => EF.Functions.ToStartOfFifteenMinutes(e.Timestamp))); + Assert.Equal(new DateTime(2026, 8, 10, 13, 45, 0), result); + } + + [Fact] + public async Task ToStartOfInterval_with_minute_unit_truncates_to_bucket() + { + var result = await SelectSingleAsync( + q => q.Select(e => EF.Functions.ToStartOfInterval(e.Timestamp, 15, ClickHouseInterval.Minute))); + Assert.Equal(new DateTime(2026, 8, 10, 13, 45, 0), result); + } + + [Fact] + public async Task ToStartOfInterval_with_hour_unit_truncates_to_bucket() + { + var result = await SelectSingleAsync( + q => q.Select(e => EF.Functions.ToStartOfInterval(e.Timestamp, 1, ClickHouseInterval.Hour))); + Assert.Equal(new DateTime(2026, 8, 10, 13, 0, 0), result); + } + + [Fact] + public async Task ToStartOfInterval_on_date_with_month_unit_truncates_to_month() + { + var result = await SelectSingleAsync( + q => q.Select(e => EF.Functions.ToStartOfInterval(e.Date, 1, ClickHouseInterval.Month))); + Assert.Equal(new DateOnly(2026, 8, 1), result); + } + + // --- Out-of-legacy-range (pre-1970) behavior -------------------------------------------------- + // By ClickHouse default, the calendar buckets return Date and the sub-day/interval buckets return + // DateTime — neither of which can represent dates before 1970. Values outside that window are + // therefore narrowed (Date clamps to the epoch; DateTime wraps around). Row 2 is a 1920 value. + // Enabling enable_extended_results_for_datetime_functions makes ClickHouse return the wider + // Date32/DateTime64 types instead, preserving the full range — verified below. + + [Fact] + public async Task ToStartOfMonth_on_pre_1970_date_is_narrowed_to_epoch_by_default() + { + await using var context = new DateTimeDbContext(_fixture.ConnectionString); + + // toStartOfMonth returns Date (min 1970-01-01), so the 1920 input is clamped to the epoch. + var result = await context.Events.AsNoTracking().Where(e => e.Id == 2) + .Select(e => EF.Functions.ToStartOfMonth(e.Date)) + .SingleAsync(); + + Assert.Equal(new DateOnly(1970, 1, 1), result); + } + + [Fact] + public async Task ToStartOfInterval_on_pre_1970_datetime64_is_not_preserved_by_default() + { + await using var context = new DateTimeDbContext(_fixture.ConnectionString); + + // toStartOfInterval returns DateTime (1970-2106) by default, so a 1920 value is not preserved. + var result = await context.Events.AsNoTracking().Where(e => e.Id == 2) + .Select(e => EF.Functions.ToStartOfInterval(e.Timestamp64, 15, ClickHouseInterval.Minute)) + .SingleAsync(); + + Assert.NotEqual(1920, result.Year); + } + + [Fact] + public async Task ToStartOf_with_extended_results_setting_preserves_pre_1970_values() + { + // The driver applies set_* connection-string parameters as query settings. Enabling + // enable_extended_results_for_datetime_functions makes the functions return Date32/DateTime64, + // preserving the full range. + var connectionString = + _fixture.ConnectionString + ";set_enable_extended_results_for_datetime_functions=1"; + await using var context = new DateTimeDbContext(connectionString); + + var month = await context.Events.AsNoTracking().Where(e => e.Id == 2) + .Select(e => EF.Functions.ToStartOfMonth(e.Date)) + .SingleAsync(); + Assert.Equal(new DateOnly(1920, 5, 1), month); + + var bucket = await context.Events.AsNoTracking().Where(e => e.Id == 2) + .Select(e => EF.Functions.ToStartOfInterval(e.Timestamp64, 15, ClickHouseInterval.Minute)) + .SingleAsync(); + Assert.Equal(new DateTime(1920, 5, 15, 12, 30, 0), bucket); + } + + [Fact] + public async Task ToStartOf_functions_work_in_group_by() + { + await using var context = new DateTimeDbContext(_fixture.ConnectionString); + + var buckets = await context.Events + .AsNoTracking() + .Where(e => e.Id == 1) + .GroupBy(e => EF.Functions.ToStartOfMonth(e.Timestamp)) + .Select(g => new { Month = g.Key, Count = g.Count() }) + .ToListAsync(); + + Assert.Single(buckets); + Assert.Equal(new DateTime(2026, 8, 1), buckets[0].Month); + Assert.Equal(1, buckets[0].Count); + } + + [Fact] + public void ToStartOfInterval_translates_to_expected_sql() + { + using var context = new DateTimeDbContext(_fixture.ConnectionString); + + var sql = context.Events + .Select(e => EF.Functions.ToStartOfInterval(e.Timestamp, 15, ClickHouseInterval.Minute)) + .ToQueryString(); + + Assert.Contains("toStartOfInterval", sql); + Assert.Contains("toIntervalMinute", sql); + } + + [Fact] + public void ToStartOf_functions_translate_to_expected_sql() + { + using var context = new DateTimeDbContext(_fixture.ConnectionString); + + var sql = context.Events + .Select(e => new + { + Year = EF.Functions.ToStartOfYear(e.Timestamp), + Month = EF.Functions.ToStartOfMonth(e.Timestamp), + Day = EF.Functions.ToStartOfDay(e.Timestamp), + FifteenMinutes = EF.Functions.ToStartOfFifteenMinutes(e.Timestamp) + }) + .ToQueryString(); + + Assert.Contains("toStartOfYear", sql); + Assert.Contains("toStartOfMonth", sql); + Assert.Contains("toStartOfDay", sql); + Assert.Contains("toStartOfFifteenMinutes", sql); + } + + [Fact] + public async Task ToStartOfInterval_with_non_constant_unit_is_not_translatable() + { + await using var context = new DateTimeDbContext(_fixture.ConnectionString); + + // A captured variable is parameterized by EF, so the unit is not a SqlConstantExpression and the + // translator returns null; the call also cannot be client-evaluated, so the query fails to translate. + var unit = ClickHouseInterval.Minute; + + await Assert.ThrowsAsync(() => + context.Events + .AsNoTracking() + .Select(e => EF.Functions.ToStartOfInterval(e.Timestamp, 15, unit)) + .ToListAsync()); + } + + [Fact] + public void ToStartOf_functions_throw_on_client_evaluation() + { + var t = DateTime.UnixEpoch; + Assert.Throws(() => EF.Functions.ToStartOfYear(t)); + Assert.Throws(() => EF.Functions.ToStartOfQuarter(t)); + Assert.Throws(() => EF.Functions.ToStartOfMonth(t)); + Assert.Throws(() => EF.Functions.ToStartOfWeek(t)); + Assert.Throws(() => EF.Functions.ToStartOfWeek(t, 1)); + Assert.Throws(() => EF.Functions.ToStartOfDay(t)); + Assert.Throws(() => EF.Functions.ToStartOfHour(t)); + Assert.Throws(() => EF.Functions.ToStartOfMinute(t)); + Assert.Throws(() => EF.Functions.ToStartOfSecond(t)); + Assert.Throws(() => EF.Functions.ToStartOfFiveMinutes(t)); + Assert.Throws(() => EF.Functions.ToStartOfTenMinutes(t)); + Assert.Throws(() => EF.Functions.ToStartOfFifteenMinutes(t)); + Assert.Throws( + () => EF.Functions.ToStartOfInterval(t, 15, ClickHouseInterval.Minute)); + } +} + +/// +/// Translation-only tests that need no ClickHouse container (they call ToQueryString(), which +/// compiles the query without connecting). +/// +public class DateTimeFunctionsTranslationOfflineTest +{ + private sealed class OfflineContext : DbContext + { + public DbSet Events => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseClickHouse("Host=localhost;Protocol=http;Port=8123;Database=test"); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + => modelBuilder.Entity(entity => + { + entity.ToTable("datetime_functions_test"); + entity.HasKey(e => e.Id); + entity.Property(e => e.Id).HasColumnName("id"); + entity.Property(e => e.Timestamp).HasColumnName("ts"); + }); + } + + [Fact] + public void ToStartOf_with_constant_argument_is_evaluated_server_side() + { + using var context = new OfflineContext(); + + // A constant/captured argument makes the whole call a client-evaluation candidate; the + // evaluatable-expression filter forces it to be translated server-side instead. + var when = new DateTime(2026, 8, 10, 13, 47, 0); + + var sql = context.Events + .Select(e => new { e.Id, Bucket = EF.Functions.ToStartOfMonth(when) }) + .ToQueryString(); + + Assert.Contains("toStartOfMonth", sql); + } +}