From 5bf65e109b25c48a09f46ff99cf0b22f7cf78db9 Mon Sep 17 00:00:00 2001 From: poma12390 Date: Thu, 30 Oct 2025 15:52:51 +0300 Subject: [PATCH 01/11] create linq2db --- slo/src/Linq2db.Slo/Linq2db.Slo.csproj | 19 ++++++ slo/src/Linq2db.Slo/Program.cs | 3 + slo/src/Linq2db.Slo/SloLinq2DbContext.cs | 83 ++++++++++++++++++++++++ slo/src/src.sln | 6 ++ 4 files changed, 111 insertions(+) create mode 100644 slo/src/Linq2db.Slo/Linq2db.Slo.csproj create mode 100644 slo/src/Linq2db.Slo/Program.cs create mode 100644 slo/src/Linq2db.Slo/SloLinq2DbContext.cs diff --git a/slo/src/Linq2db.Slo/Linq2db.Slo.csproj b/slo/src/Linq2db.Slo/Linq2db.Slo.csproj new file mode 100644 index 00000000..b0d2b8dc --- /dev/null +++ b/slo/src/Linq2db.Slo/Linq2db.Slo.csproj @@ -0,0 +1,19 @@ + + + + Exe + net8.0 + enable + enable + Linq2db + + + + + + + + + + + diff --git a/slo/src/Linq2db.Slo/Program.cs b/slo/src/Linq2db.Slo/Program.cs new file mode 100644 index 00000000..8407a665 --- /dev/null +++ b/slo/src/Linq2db.Slo/Program.cs @@ -0,0 +1,3 @@ +using Internal; + +await Cli.Run(new Linq2db.SloTableContext(), args); \ No newline at end of file diff --git a/slo/src/Linq2db.Slo/SloLinq2DbContext.cs b/slo/src/Linq2db.Slo/SloLinq2DbContext.cs new file mode 100644 index 00000000..47d699d3 --- /dev/null +++ b/slo/src/Linq2db.Slo/SloLinq2DbContext.cs @@ -0,0 +1,83 @@ +using System; +using System.Threading.Tasks; +using Internal; +using LinqToDB; +using LinqToDB.Data; // <= вот это нужно для DataConnection + +namespace Linq2db; + +public sealed class SloTableContext : SloTableContext +{ + protected override string Job => "Linq2DB"; + + public sealed class Linq2dbClient + { + private readonly string _connectionString; + + public Linq2dbClient(string connectionString) + { + _connectionString = connectionString; + } + + public DataConnection Open() => new DataConnection("YDB", _connectionString); + } + + protected override Linq2dbClient CreateClient(Config config) + => new Linq2dbClient(config.ConnectionString); + + protected override async Task Create(Linq2dbClient client, int operationTimeout) + { + await using var db = client.Open(); + db.CommandTimeout = operationTimeout; + + // 1) CREATE TABLE + await db.ExecuteAsync($@" + CREATE TABLE `{SloTable.Name}` ( + Guid Uuid, + Id Int32, + PayloadStr Text, + PayloadDouble Double, + PayloadTimestamp Timestamp, + PRIMARY KEY (Guid, Id) + ); + "); + + await db.ExecuteAsync(SloTable.Options); + } + + protected override async Task Save(Linq2dbClient client, SloTable row, int writeTimeout) + { + await using var db = client.Open(); + db.CommandTimeout = writeTimeout; + + await db.ExecuteAsync($@" + UPSERT INTO `{SloTable.Name}` (Guid, Id, PayloadStr, PayloadDouble, PayloadTimestamp) + VALUES ({row.Guid}, {row.Id}, {row.PayloadStr}, {row.PayloadDouble}, {row.PayloadTimestamp}); + "); + + return 1; + } + + protected override async Task Select(Linq2dbClient client, (Guid Guid, int Id) key, int readTimeout) + { + await using var db = client.Open(); + db.CommandTimeout = readTimeout; + + var exists = await db.ExecuteAsync($@" + SELECT COUNT(*) FROM `{SloTable.Name}` WHERE Guid = {key.Guid} AND Id = {key.Id}; + "); + + return exists > 0 ? 1 : null; + } + + protected override async Task SelectCount(Linq2dbClient client) + { + await using var db = client.Open(); + + var maxId = await db.ExecuteAsync($@" + SELECT MAX(Id) FROM `{SloTable.Name}`; + "); + + return maxId ?? 0; + } +} diff --git a/slo/src/src.sln b/slo/src/src.sln index 68acb0f7..b6cdb57f 100644 --- a/slo/src/src.sln +++ b/slo/src/src.sln @@ -13,6 +13,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EF", "EF\EF.csproj", "{291A EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AdoNet.Dapper", "Dapper\AdoNet.Dapper.csproj", "{A6B9B4F1-4C7C-42C1-A212-B71A9B0D67F7}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Linq2db.Slo", "Linq2db.Slo\Linq2db.Slo.csproj", "{59758BC9-E53B-46C8-84AC-62670DD559D4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -39,6 +41,10 @@ Global {A6B9B4F1-4C7C-42C1-A212-B71A9B0D67F7}.Debug|Any CPU.Build.0 = Debug|Any CPU {A6B9B4F1-4C7C-42C1-A212-B71A9B0D67F7}.Release|Any CPU.ActiveCfg = Release|Any CPU {A6B9B4F1-4C7C-42C1-A212-B71A9B0D67F7}.Release|Any CPU.Build.0 = Release|Any CPU + {59758BC9-E53B-46C8-84AC-62670DD559D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {59758BC9-E53B-46C8-84AC-62670DD559D4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {59758BC9-E53B-46C8-84AC-62670DD559D4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {59758BC9-E53B-46C8-84AC-62670DD559D4}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 4b580b9af91dca59c8a35c7578a06cfb98c4014e Mon Sep 17 00:00:00 2001 From: poma12390 Date: Thu, 30 Oct 2025 15:54:35 +0300 Subject: [PATCH 02/11] add main logic --- src/Linq2db.Ydb/src/IYdbSpecificQueryable.cs | 6 + src/Linq2db.Ydb/src/IYdbSpecificTable.cs | 7 + .../Translation/YdbMemberTranslator.cs | 410 +++++++++++ src/Linq2db.Ydb/src/Internal/YdbBulkCopy.cs | 217 ++++++ .../src/Internal/YdbDataProvider.cs | 490 ++++++++++++ .../src/Internal/YdbMappingSchema.cs | 695 ++++++++++++++++++ .../src/Internal/YdbProviderAdapter.cs | 337 +++++++++ .../src/Internal/YdbRetryPolicy.cs | 144 ++++ .../src/Internal/YdbSchemaProvider.cs | 513 +++++++++++++ .../src/Internal/YdbSpecificQueryable.cs | 12 + .../src/Internal/YdbSpecificTable.cs | 13 + src/Linq2db.Ydb/src/Internal/YdbSqlBuilder.cs | 396 ++++++++++ .../YdbSqlExpressionConvertVisitor.cs | 166 +++++ .../src/Internal/YdbSqlOptimizer.cs | 234 ++++++ .../Internal/YdbTransientExceptionDetector.cs | 116 +++ src/Linq2db.Ydb/src/Linq2db.csproj | 31 + src/Linq2db.Ydb/src/YdbHints.cs | 112 +++ src/Linq2db.Ydb/src/YdbHints.generated.cs | 77 ++ src/Linq2db.Ydb/src/YdbHints.tt | 61 ++ src/Linq2db.Ydb/src/YdbOptions.cs | 55 ++ src/Linq2db.Ydb/src/YdbSpecificExtensions.cs | 44 ++ src/Linq2db.Ydb/src/YdbTools.Registration.cs | 15 + src/Linq2db.Ydb/src/YdbTools.cs | 76 ++ .../test/Linq2db.Tests/Linq2db.Tests.csproj | 21 + .../test/Linq2db.Tests/YdbTests.cs | 286 +++++++ src/YdbSdk.sln | 22 + 26 files changed, 4556 insertions(+) create mode 100644 src/Linq2db.Ydb/src/IYdbSpecificQueryable.cs create mode 100644 src/Linq2db.Ydb/src/IYdbSpecificTable.cs create mode 100644 src/Linq2db.Ydb/src/Internal/Translation/YdbMemberTranslator.cs create mode 100644 src/Linq2db.Ydb/src/Internal/YdbBulkCopy.cs create mode 100644 src/Linq2db.Ydb/src/Internal/YdbDataProvider.cs create mode 100644 src/Linq2db.Ydb/src/Internal/YdbMappingSchema.cs create mode 100644 src/Linq2db.Ydb/src/Internal/YdbProviderAdapter.cs create mode 100644 src/Linq2db.Ydb/src/Internal/YdbRetryPolicy.cs create mode 100644 src/Linq2db.Ydb/src/Internal/YdbSchemaProvider.cs create mode 100644 src/Linq2db.Ydb/src/Internal/YdbSpecificQueryable.cs create mode 100644 src/Linq2db.Ydb/src/Internal/YdbSpecificTable.cs create mode 100644 src/Linq2db.Ydb/src/Internal/YdbSqlBuilder.cs create mode 100644 src/Linq2db.Ydb/src/Internal/YdbSqlExpressionConvertVisitor.cs create mode 100644 src/Linq2db.Ydb/src/Internal/YdbSqlOptimizer.cs create mode 100644 src/Linq2db.Ydb/src/Internal/YdbTransientExceptionDetector.cs create mode 100644 src/Linq2db.Ydb/src/Linq2db.csproj create mode 100644 src/Linq2db.Ydb/src/YdbHints.cs create mode 100644 src/Linq2db.Ydb/src/YdbHints.generated.cs create mode 100644 src/Linq2db.Ydb/src/YdbHints.tt create mode 100644 src/Linq2db.Ydb/src/YdbOptions.cs create mode 100644 src/Linq2db.Ydb/src/YdbSpecificExtensions.cs create mode 100644 src/Linq2db.Ydb/src/YdbTools.Registration.cs create mode 100644 src/Linq2db.Ydb/src/YdbTools.cs create mode 100644 src/Linq2db.Ydb/test/Linq2db.Tests/Linq2db.Tests.csproj create mode 100644 src/Linq2db.Ydb/test/Linq2db.Tests/YdbTests.cs diff --git a/src/Linq2db.Ydb/src/IYdbSpecificQueryable.cs b/src/Linq2db.Ydb/src/IYdbSpecificQueryable.cs new file mode 100644 index 00000000..cd008e43 --- /dev/null +++ b/src/Linq2db.Ydb/src/IYdbSpecificQueryable.cs @@ -0,0 +1,6 @@ +namespace LinqToDB.Internal.DataProvider.Ydb +{ + public interface IYdbSpecificQueryable : IQueryable + { + } +} diff --git a/src/Linq2db.Ydb/src/IYdbSpecificTable.cs b/src/Linq2db.Ydb/src/IYdbSpecificTable.cs new file mode 100644 index 00000000..e1159397 --- /dev/null +++ b/src/Linq2db.Ydb/src/IYdbSpecificTable.cs @@ -0,0 +1,7 @@ +namespace LinqToDB.Internal.DataProvider.Ydb +{ + public interface IYdbSpecificTable : ITable + where TSource : notnull + { + } +} diff --git a/src/Linq2db.Ydb/src/Internal/Translation/YdbMemberTranslator.cs b/src/Linq2db.Ydb/src/Internal/Translation/YdbMemberTranslator.cs new file mode 100644 index 00000000..df36fd1c --- /dev/null +++ b/src/Linq2db.Ydb/src/Internal/Translation/YdbMemberTranslator.cs @@ -0,0 +1,410 @@ +using System.Linq.Expressions; +using LinqToDB.Internal.DataProvider.Translation; +using LinqToDB.Internal.SqlQuery; +using LinqToDB.Linq.Translation; + +namespace LinqToDB.Internal.DataProvider.Ydb.Internal.Translation +{ + public class YdbMemberTranslator : ProviderMemberTranslatorDefault + { + protected override IMemberTranslator CreateStringMemberTranslator() => new StringMemberTranslator(); + + protected override IMemberTranslator CreateDateMemberTranslator() => new DateFunctionsTranslator(); + + protected override IMemberTranslator CreateGuidMemberTranslator() => new GuidMemberTranslator(); + + protected override IMemberTranslator CreateSqlTypesTranslator() => new SqlTypesTranslation(); + + protected override IMemberTranslator CreateMathMemberTranslator() => new MathMemberTranslator(); + + //ConvertToString + //ProcessSqlConvert + //TranslateConvertToBoolean/ProcessConvertToBoolean + //ProcessGetValueOrDefault + + // TODO: we cannot use this implementation as it will generate same UUID for all invocations within single query + //protected override ISqlExpression? TranslateNewGuidMethod(ITranslationContext translationContext, TranslationFlags translationFlags) + //{ + // var factory = translationContext.ExpressionFactory; + + // var timePart = factory.NonPureFunction(factory.GetDbDataType(typeof(Guid)), "RandomUuid", factory.Value(1)); + + // return timePart; + //} + + protected class GuidMemberTranslator : GuidMemberTranslatorBase + { + protected override ISqlExpression? TranslateGuildToString(ITranslationContext translationContext, MethodCallExpression methodCall, ISqlExpression guidExpr, TranslationFlags translationFlags) + { + // Cast({0} as Utf8) + + var factory = translationContext.ExpressionFactory; + var stringDataType = factory.GetDbDataType(typeof(string)).WithDataType(DataType.NVarChar); + + var cast = factory.Cast(guidExpr, stringDataType); + + return cast; + } + } + + protected class StringMemberTranslator : StringMemberTranslatorBase + { + public override ISqlExpression? TranslatePadLeft(ITranslationContext translationContext, MethodCallExpression methodCall, TranslationFlags translationFlags, ISqlExpression value, ISqlExpression padding, ISqlExpression? paddingChar) + { + var factory = translationContext.ExpressionFactory; + var valueTypeString = factory.GetDbDataType(value); + + return paddingChar != null + ? factory.Function(valueTypeString, "String::LeftPad", value, padding, paddingChar) + : factory.Function(valueTypeString, "String::LeftPad", value, padding); + } + + public override ISqlExpression? TranslateReplace(ITranslationContext translationContext, MethodCallExpression methodCall, TranslationFlags translationFlags, ISqlExpression value, ISqlExpression oldValue, ISqlExpression newValue) + { + var factory = translationContext.ExpressionFactory; + var valueTypeString = factory.GetDbDataType(value); + + return factory.Function(valueTypeString, "String::ReplaceAll", value, oldValue, newValue); + } + + protected Expression? TranslateLike(ITranslationContext translationContext, MethodCallExpression methodCall, TranslationFlags translationFlags) + { + var factory = translationContext.ExpressionFactory; + + using var disposable = translationContext.UsingTypeFromExpression(methodCall.Arguments[0], methodCall.Arguments[1]); + + if (!translationContext.TranslateToSqlExpression(methodCall.Arguments[0], out var translatedField)) + return translationContext.CreateErrorExpression(methodCall.Arguments[0], type: methodCall.Type); + + if (!translationContext.TranslateToSqlExpression(methodCall.Arguments[1], out var translatedValue)) + return translationContext.CreateErrorExpression(methodCall.Arguments[1], type: methodCall.Type); + + ISqlExpression? escape = null; + + if (methodCall.Arguments.Count == 3) + { + if (!translationContext.TranslateToSqlExpression(methodCall.Arguments[2], out escape)) + return translationContext.CreateErrorExpression(methodCall.Arguments[2], type: methodCall.Type); + + if (escape is SqlValue { ValueType.DataType: not DataType.Binary } value) + value.ValueType = value.ValueType.WithDataType(DataType.Binary); + } + + var predicate = factory.LikePredicate(translatedField, false, translatedValue, escape); + var searchCondition = factory.SearchCondition().Add(predicate); + + return translationContext.CreatePlaceholder(translationContext.CurrentSelectQuery, searchCondition, methodCall); + } + } + + protected class SqlTypesTranslation : SqlTypesTranslationDefault + { + protected override Expression? ConvertBit(ITranslationContext translationContext, MemberExpression memberExpression, TranslationFlags translationFlags) + { + //return base.ConvertBit(translationContext, memberExpression, translationFlags); + throw new NotImplementedException("55"); + } +#if SUPPORTS_DATEONLY + + protected override Expression? ConvertDateOnly(ITranslationContext translationContext, MemberExpression memberExpression, TranslationFlags translationFlags) + { + //return base.ConvertDateOnly(translationContext, memberExpression, translationFlags); + throw new NotImplementedException("52"); + } +#endif + + // // YDB stores DateTime with microsecond precision in the Timestamp type + // protected override Expression? ConvertDateTime(ITranslationContext translationContext, MemberExpression memberExpression, TranslationFlags translationFlags) + // => MakeSqlTypeExpression(translationContext, memberExpression, t => t.WithDataType(DataType.Timestamp)); + + // protected override Expression? ConvertDateTime2(ITranslationContext translationContext, MemberExpression memberExpression, TranslationFlags translationFlags) + // => MakeSqlTypeExpression(translationContext, memberExpression, t => t.WithDataType(DataType.Timestamp)); + + // protected override Expression? ConvertSmallDateTime(ITranslationContext translationContext, MemberExpression memberExpression, TranslationFlags translationFlags) + // => MakeSqlTypeExpression(translationContext, memberExpression, t => t.WithDataType(DataType.Timestamp)); + + // protected override Expression? ConvertDateTimeOffset(ITranslationContext translationContext, MemberExpression memberExpression, TranslationFlags translationFlags) + // => MakeSqlTypeExpression(translationContext, memberExpression, t => t.WithDataType(DataType.Timestamp)); + } + + protected class DateFunctionsTranslator : DateFunctionsTranslatorBase + { + protected override ISqlExpression? TranslateDateTimeOffsetDatePart(ITranslationContext translationContext, TranslationFlags translationFlag, ISqlExpression dateTimeExpression, Sql.DateParts datepart) + { + //return base.TranslateDateTimeOffsetDatePart(translationContext, translationFlag, dateTimeExpression, datepart); + throw new NotImplementedException("11"); + } + + protected override ISqlExpression? TranslateDateTimeOffsetTruncationToDate(ITranslationContext translationContext, ISqlExpression dateExpression, TranslationFlags translationFlags) + { + //return base.TranslateDateTimeOffsetTruncationToDate(translationContext, dateExpression, translationFlags); + throw new NotImplementedException("10"); + } + + protected override ISqlExpression? TranslateDateTimeOffsetTruncationToTime(ITranslationContext translationContext, ISqlExpression dateExpression, TranslationFlags translationFlags) + { + //return base.TranslateDateTimeOffsetTruncationToTime(translationContext, dateExpression, translationFlags); + throw new NotImplementedException("09"); + } + + protected override ISqlExpression? TranslateDateTimeTruncationToTime(ITranslationContext translationContext, ISqlExpression dateExpression, TranslationFlags translationFlags) + { + var factory = translationContext.ExpressionFactory; + + var type = factory.GetDbDataType(typeof(TimeSpan)).WithDataType(DataType.Interval); + var cast = factory.Function(type, "DateTime::TimeOfDay", dateExpression); + + return cast; + } + + protected override ISqlExpression? TranslateDateTimeDatePart(ITranslationContext translationContext, TranslationFlags translationFlag, ISqlExpression dateTimeExpression, Sql.DateParts datepart) + { + var f = translationContext.ExpressionFactory; + var intType = f.GetDbDataType(typeof(int)); + + // QUARTER = (Month + 2) / 3 + if (datepart == Sql.DateParts.Quarter) + { + var month = f.Function(intType, "DateTime::GetMonth", dateTimeExpression); + var two = f.Value(intType, 2); + var three = f.Value(intType, 3); + return f.Div(intType, f.Add(intType, month, two), three); + } + + string? fn = datepart switch + { + Sql.DateParts.Year => "DateTime::GetYear", + Sql.DateParts.Month => "DateTime::GetMonth", + Sql.DateParts.Day => "DateTime::GetDayOfMonth", + Sql.DateParts.DayOfYear => "DateTime::GetDayOfYear", + Sql.DateParts.Week => "DateTime::GetWeekOfYearIso8601", + Sql.DateParts.Hour => "DateTime::GetHour", + Sql.DateParts.Minute => "DateTime::GetMinute", + Sql.DateParts.Second => "DateTime::GetSecond", + Sql.DateParts.Millisecond => "DateTime::GetMillisecondOfSecond", + Sql.DateParts.WeekDay => "DateTime::GetDayOfWeek", + _ => null + }; + + if (fn == null) + return null; + + var baseExpr = f.Function(intType, fn, dateTimeExpression); + + // Adjust DayOfWeek to match T-SQL format (Sunday=1 ... Saturday=7) + if (datepart == Sql.DateParts.WeekDay) + { + var seven = f.Value(intType, 7); + var one = f.Value(intType, 1); + return f.Add(intType, f.Mod(baseExpr, seven), one); + } + + return baseExpr; + } + + // protected override ISqlExpression? TranslateDateTimeOffsetDatePart( + // ITranslationContext translationContext, TranslationFlags translationFlag, + // ISqlExpression dateTimeExpression, Sql.DateParts datepart) + // => TranslateDateTimeDatePart(translationContext, translationFlag, dateTimeExpression, datepart); + + protected override ISqlExpression? TranslateDateTimeDateAdd( + ITranslationContext translationContext, TranslationFlags translationFlag, + ISqlExpression dateTimeExpression, ISqlExpression increment, Sql.DateParts datepart) + { + var f = translationContext.ExpressionFactory; + var dateType = f.GetDbDataType(dateTimeExpression); + var intType = f.GetDbDataType(typeof(int)); + + if (datepart is Sql.DateParts.Year or Sql.DateParts.Month or Sql.DateParts.Quarter) + { + string shiftFn = datepart switch + { + Sql.DateParts.Year => "DateTime::ShiftYears", + Sql.DateParts.Month => "DateTime::ShiftMonths", + Sql.DateParts.Quarter => "DateTime::ShiftMonths", + _ => throw new InvalidOperationException() + }; + + var shiftArg = increment; + + if (datepart == Sql.DateParts.Quarter) + shiftArg = f.Multiply(intType, increment, 3); + + var shifted = f.Function(f.GetDbDataType(dateTimeExpression), shiftFn, dateTimeExpression, shiftArg); + var makeDateTime = f.Function(dateType, "DateTime::MakeDatetime", shifted); + + return makeDateTime; + } + + string? intervalFn = datepart switch + { + Sql.DateParts.Week => "DateTime::IntervalFromDays", + Sql.DateParts.Day => "DateTime::IntervalFromDays", + Sql.DateParts.Hour => "DateTime::IntervalFromHours", + Sql.DateParts.Minute => "DateTime::IntervalFromMinutes", + Sql.DateParts.Second => "DateTime::IntervalFromSeconds", + Sql.DateParts.Millisecond => "DateTime::IntervalFromMilliseconds", + _ => null + }; + + if (intervalFn == null) + return null; + + if (datepart == Sql.DateParts.Week) + { + increment = f.Multiply(f.GetDbDataType(increment), increment, 7); + } + + var interval = f.Function( + f.GetDbDataType(typeof(TimeSpan)).WithDataType(DataType.Interval), + intervalFn, + increment); + + return f.Add(dateType, dateTimeExpression, interval); + } + + // protected override ISqlExpression? TranslateDateTimeTruncationToDate( + // ITranslationContext translationContext, ISqlExpression dateExpression, TranslationFlags translationFlags) + // { + // var f = translationContext.ExpressionFactory; + // var resType = f.GetDbDataType(typeof(DateTime)).WithDataType(DataType.Date); + // var startDay = f.Function(f.GetDbDataType(dateExpression), "DateTime::StartOfDay", dateExpression); + // return f.Function(resType, "DateTime::MakeDate", startDay); + // } + + // protected override ISqlExpression? TranslateDateTimeOffsetTruncationToDate( + // ITranslationContext translationContext, ISqlExpression dateExpression, TranslationFlags translationFlags) + // => TranslateDateTimeTruncationToDate(translationContext, dateExpression, translationFlags); + + protected override ISqlExpression? TranslateSqlGetDate(ITranslationContext translationContext, TranslationFlags translationFlags) + { + var f = translationContext.ExpressionFactory; + + return f.Function(f.GetDbDataType(typeof(DateTime)), "CurrentUtcTimestamp", ParametersNullabilityType.NotNullable); + } + } + + protected class MathMemberTranslator : MathMemberTranslatorBase + { + protected override ISqlExpression? TranslateMaxMethod(ITranslationContext translationContext, MethodCallExpression methodCall, ISqlExpression xValue, ISqlExpression yValue) + { + var factory = translationContext.ExpressionFactory; + + var valueType = factory.GetDbDataType(xValue); + + var result = factory.Function(valueType, "MAX_OF", xValue, yValue); + + return result; + } + + protected override ISqlExpression? TranslateMinMethod(ITranslationContext translationContext, MethodCallExpression methodCall, ISqlExpression xValue, ISqlExpression yValue) + { + var factory = translationContext.ExpressionFactory; + + var valueType = factory.GetDbDataType(xValue); + + var result = factory.Function(valueType, "MIN_OF", xValue, yValue); + + return result; + } + + protected override ISqlExpression? TranslatePow(ITranslationContext translationContext, MethodCallExpression methodCall, ISqlExpression xValue, ISqlExpression yValue) + { + var factory = translationContext.ExpressionFactory; + + var xType = factory.GetDbDataType(xValue); + var resultType = xType; + + if (xType.DataType is not (DataType.Double or DataType.Single)) + { + xType = factory.GetDbDataType(typeof(double)); + xValue = factory.Cast(xValue, xType); + } + + var yType = factory.GetDbDataType(yValue); + var yValueResult = yValue; + + if (yType.DataType is not (DataType.Double or DataType.Single)) + { + yValueResult = factory.Cast(yValue, xType); + } + + var result = factory.Function(xType, "Math::Pow", xValue, yValueResult); + if (!resultType.EqualsDbOnly(xType)) + { + result = factory.Cast(result, resultType); + } + + return result; + } + + protected override ISqlExpression? TranslateRoundToEven(ITranslationContext translationContext, MethodCallExpression methodCall, ISqlExpression value, ISqlExpression? precision) + { + if (precision != null) + return base.TranslateRoundToEven(translationContext, methodCall, value, precision); + + var factory = translationContext.ExpressionFactory; + + var valueType = factory.GetDbDataType(value); + + var result = factory.Function(valueType, "Math::NearbyInt", value, factory.Fragment("Math::RoundToNearest()")); + + return result; + } + + // /// + // /// Banker's rounding: In YQL, ROUND(value, precision?) already performs to-even rounding for Decimal types. + // /// + // protected override ISqlExpression? TranslateRoundToEven( + // ITranslationContext translationContext, + // MethodCallExpression methodCall, + // ISqlExpression value, + // ISqlExpression? precision) + // { + // var factory = translationContext.ExpressionFactory; + // var valueType = factory.GetDbDataType(value); + + // return precision != null + // ? factory.Function(valueType, "ROUND", value, precision) + // : factory.Function(valueType, "ROUND", value); + // } + + // /// + // /// Away-from-zero rounding: In YQL, ROUND(value, precision?) for Numeric types already uses away-from-zero rounding. + // /// + // protected override ISqlExpression? TranslateRoundAwayFromZero( + // ITranslationContext translationContext, + // MethodCallExpression methodCall, + // ISqlExpression value, + // ISqlExpression? precision) + // { + // var factory = translationContext.ExpressionFactory; + // var valueType = factory.GetDbDataType(value); + + // return precision != null + // ? factory.Function(valueType, "ROUND", value, precision) + // : factory.Function(valueType, "ROUND", value); + // } + + // /// + // /// Exponentiation using the built-in POWER(a, b) function. + // /// + // protected override ISqlExpression? TranslatePow( + // ITranslationContext translationContext, + // MethodCallExpression methodCall, + // ISqlExpression xValue, + // ISqlExpression yValue) + // { + // var factory = translationContext.ExpressionFactory; + // var xType = factory.GetDbDataType(xValue); + // var yType = factory.GetDbDataType(yValue); + + // if (!xType.EqualsDbOnly(yType)) + // yValue = factory.Cast(yValue, xType); + + // return factory.Function(xType, "POWER", xValue, yValue); + // } + } + + } +} diff --git a/src/Linq2db.Ydb/src/Internal/YdbBulkCopy.cs b/src/Linq2db.Ydb/src/Internal/YdbBulkCopy.cs new file mode 100644 index 00000000..ed0e4cfa --- /dev/null +++ b/src/Linq2db.Ydb/src/Internal/YdbBulkCopy.cs @@ -0,0 +1,217 @@ +using System.Collections.ObjectModel; +using System.Data.Common; +using System.Reflection; +using LinqToDB.Async; +using LinqToDB.Data; +using LinqToDB.DataProvider; +using LinqToDB.Internal.Async; +using LinqToDB.Internal.Extensions; +using LinqToDB.Mapping; + +namespace LinqToDB.Internal.DataProvider.Ydb.Internal +{ + /// + /// Provider-specific implementation of BulkCopy for YDB. + /// + public class YdbBulkCopy : BasicBulkCopy + { + readonly YdbDataProvider _provider; + + public YdbBulkCopy(YdbDataProvider dataProvider) + { + _provider = dataProvider; + } + + protected override BulkCopyRowsCopied MultipleRowsCopy( + ITable table, + DataOptions options, + IEnumerable source) + => MultipleRowsCopy1(table, options, source); + + protected override Task MultipleRowsCopyAsync( + ITable table, + DataOptions options, + IEnumerable source, + CancellationToken cancellationToken) + => MultipleRowsCopy1Async(table, options, source, cancellationToken); + + protected override Task MultipleRowsCopyAsync( + ITable table, + DataOptions options, + IAsyncEnumerable source, + CancellationToken cancellationToken) + => MultipleRowsCopy1Async(table, options, source, cancellationToken); + + protected override BulkCopyRowsCopied ProviderSpecificCopy(ITable table, DataOptions options, + IEnumerable source) + { + if (table.TryGetDataConnection(out var dataConnection)) + { + var connection = _provider.TryGetProviderConnection(dataConnection, dataConnection.OpenDbConnection()); + + if (connection != null) + { + return SafeAwaiter.Run(() => ProviderSpecificCopyImplAsync( + dataConnection, + connection, + table, + options, + columns => new BulkCopyReader(dataConnection, columns, source), + default)); + } + } + + return MultipleRowsCopy(table, options, source); + } + + protected override async Task ProviderSpecificCopyAsync(ITable table, + DataOptions options, IAsyncEnumerable source, CancellationToken cancellationToken) + { + if (table.TryGetDataConnection(out var dataConnection)) + { + var connection = _provider.TryGetProviderConnection(dataConnection, + await dataConnection.OpenDbConnectionAsync(cancellationToken).ConfigureAwait(false)); + + if (connection != null) + { + return await ProviderSpecificCopyImplAsync( + dataConnection, + connection, + table, + options, + columns => new BulkCopyReader(dataConnection, columns, source, cancellationToken), + cancellationToken) + .ConfigureAwait(false); + } + } + + return await MultipleRowsCopyAsync(table, options, source, cancellationToken).ConfigureAwait(false); + } + + protected override async Task ProviderSpecificCopyAsync(ITable table, + DataOptions options, IEnumerable source, CancellationToken cancellationToken) + { + if (table.TryGetDataConnection(out var dataConnection)) + { + var connection = _provider.TryGetProviderConnection(dataConnection, + await dataConnection.OpenDbConnectionAsync(cancellationToken).ConfigureAwait(false)); + + if (connection != null) + { + return await ProviderSpecificCopyImplAsync( + dataConnection, + connection, + table, + options, + columns => new BulkCopyReader(dataConnection, columns, source), + cancellationToken) + .ConfigureAwait(false); + } + } + + return await MultipleRowsCopyAsync(table, options, source, cancellationToken).ConfigureAwait(false); + } + + + async Task ProviderSpecificCopyImplAsync( + DataConnection dataConnection, + DbConnection dbConnection, + ITable table, + DataOptions options, + Func, BulkCopyReader> createDataReader, + CancellationToken cancellationToken) + where T : notnull + { + var sqlBuilder = + (YdbSqlBuilder)_provider.CreateSqlBuilder(table.DataContext.MappingSchema, dataConnection.Options); + var ed = table.DataContext.MappingSchema.GetEntityDescriptor(typeof(T), + dataConnection.Options.ConnectionOptions.OnEntityDescriptorCreated); + var columns = ed.Columns + .Where(c => !c.SkipOnInsert || options.BulkCopyOptions.KeepIdentity == true && c.IsIdentity).ToList(); + var command = dataConnection.GetOrCreateCommand(); + + // table name shouldn't be escaped + // TOD: test FQN + var tableName = table.TableName; + var fields = columns.Select(column => column.ColumnName).ToArray(); + + var batchSize = options.BulkCopyOptions.MaxBatchSize ?? 10_000; + + var rd = createDataReader(columns); + await using var _ = rd.ConfigureAwait(false); + + // there is no any options available for this API + var writer = _provider.Adapter.BeginBulkCopy(dbConnection, tableName, fields, cancellationToken); + + var rowsCopied = new BulkCopyRowsCopied(); + var currentCount = 0; + + // for now we will rely on implementation details and reuse row object + var row = new object?[columns.Count]; + while (await rd.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + // provider parameters used as they already have properly types values + // because bulk api doesn't type parameters itself + GetAsParameters(rd, dataConnection, command.CreateParameter, row); + + await writer.AddRowAsync(row).ConfigureAwait(false); + currentCount++; + rowsCopied.RowsCopied++; + + if (options.BulkCopyOptions.NotifyAfter != 0 && + options.BulkCopyOptions.RowsCopiedCallback != null && + rowsCopied.RowsCopied % options.BulkCopyOptions.NotifyAfter == 0) + { + options.BulkCopyOptions.RowsCopiedCallback(rowsCopied); + + if (rowsCopied.Abort) + { + break; + } + } + + if (currentCount >= batchSize) + { + await writer.FlushAsync().ConfigureAwait(false); + currentCount = 0; + } + } + + if (!rowsCopied.Abort) + { + await TraceActionAsync( + dataConnection, + () => + $"INSERT ASYNC BULK {tableName}({string.Join(", ", columns.Select(x => x.ColumnName))}){Environment.NewLine}", + async () => + { + if (currentCount > 0) + await writer.FlushAsync().ConfigureAwait(false); + + return (int)rowsCopied.RowsCopied; + }).ConfigureAwait(false); + } + + if (currentCount > 0 && options.BulkCopyOptions.NotifyAfter != 0 && + options.BulkCopyOptions.RowsCopiedCallback != null) + options.BulkCopyOptions.RowsCopiedCallback(rowsCopied); + + if (table.DataContext.CloseAfterUse) + await table.DataContext.CloseAsync().ConfigureAwait(false); + + return rowsCopied; + } + + // Todo remove and replace for rd.GetAsParameters(command.CreateParameter, row); + private static int GetAsParameters( + BulkCopyReader rd, + DataConnection dataConnection, + Func parameterFactory, + object?[] values) + where T : notnull + { + return rd.GetValues(values); + } + + } +} diff --git a/src/Linq2db.Ydb/src/Internal/YdbDataProvider.cs b/src/Linq2db.Ydb/src/Internal/YdbDataProvider.cs new file mode 100644 index 00000000..1217cfe9 --- /dev/null +++ b/src/Linq2db.Ydb/src/Internal/YdbDataProvider.cs @@ -0,0 +1,490 @@ +using System.Data; +using System.Data.Common; +using System.Globalization; +using System.Text; +using LinqToDB.Data; +using LinqToDB.Internal.DataProvider.Ydb.Internal.Translation; +using LinqToDB.Internal.SqlProvider; +using LinqToDB.Linq.Translation; +using LinqToDB.Mapping; +using LinqToDB.SchemaProvider; + +namespace LinqToDB.Internal.DataProvider.Ydb.Internal +{ + public class YdbDataProvider : DynamicDataProviderBase + { + public YdbDataProvider() + : this("YDB", YdbMappingSchema.Instance) + { + } + + protected YdbDataProvider(string name, MappingSchema mappingSchema) + : base(name, mappingSchema, YdbProviderAdapter.Instance) + { + SqlProviderFlags.IsSubQueryOrderBySupported = true; + SqlProviderFlags.IsDistinctSetOperationsSupported = false; + // only Serializable supported + SqlProviderFlags.DefaultMultiQueryIsolationLevel = IsolationLevel.Serializable; + SqlProviderFlags.RowConstructorSupport = RowFeature.Equality | RowFeature.Comparisons | RowFeature.Between | RowFeature.In | RowFeature.UpdateLiteral; + SqlProviderFlags.SupportsPredicatesComparison = true; + SqlProviderFlags.IsDistinctFromSupported = true; + + // "emulated" using table expressions + SqlProviderFlags.IsCommonTableExpressionsSupported = true; + + // https://github.com/ydb-platform/ydb/issues/11258 + // note that we cannot use LIMIT big_num, X workaround, as we do for CH, because server misbehaves + SqlProviderFlags.IsSkipSupported = false; + SqlProviderFlags.IsSkipSupportedIfTake = true; + + SqlProviderFlags.IsNestedJoinsSupported = false; + + SqlProviderFlags.IsSupportedSimpleCorrelatedSubqueries = true; + SqlProviderFlags.SupportedCorrelatedSubqueriesLevel = 0; + + RegisterYdbReaders(); + + _sqlOptimizer = new YdbSqlOptimizer(SqlProviderFlags); + } + + private void RegisterYdbReaders() + { + // 1) Базовые провайдер-специфичные геттеры через статические методы адаптера + // (предполагается сигнатура: static T GetX(DbDataReader r, int i)) + SetProviderField((r, i) => YdbProviderAdapter.GetBytes(r, i)); + SetProviderField((r, i) => YdbProviderAdapter.GetSByte(r, i)); + SetProviderField((r, i) => YdbProviderAdapter.GetUInt16(r, i)); + SetProviderField((r, i) => YdbProviderAdapter.GetUInt32(r, i)); + SetProviderField((r, i) => YdbProviderAdapter.GetUInt64(r, i)); + SetProviderField((r, i) => YdbProviderAdapter.GetInterval(r, i)); + + // 2) Json-типы: нужно матчить по DataTypeName -> используем SetToType(..., dataTypeName, ...) + SetToType("Json", (r, i) => YdbProviderAdapter.GetJson(r, i)); + SetToType("JsonDocument", (r, i) => YdbProviderAdapter.GetJsonDocument(r, i)); + + // 3) TextReader/Stream без строкового имени метода: через стандартный GetFieldValue() + SetField((r, i) => r.GetFieldValue(i)); + SetField((r, i) => r.GetFieldValue(i)); + + // 4) DateTimeOffset для типов, где провайдер отдает DateTime, но тип называется Timestamp* + // (единственный валидный способ задать DataTypeName — SetToType) + SetToType("Timestamp", + (r, i) => new DateTimeOffset(r.GetDateTime(i), default)); + SetToType("Timestamp64", + (r, i) => new DateTimeOffset(r.GetDateTime(i), default)); + SetToType("Datetime64", + (r, i) => new DateTimeOffset(r.GetDateTime(i), default)); + + // 5) Особенности YSON: провайдер репортит FieldType=String, но в Value лежит byte[] + // Покрываем оба случая: по имени типа и по "обычной String" + SetToType("Yson", (r, i) => Encoding.UTF8.GetString((byte[])r.GetValue(i))); + SetToType("String", (r, i) => Encoding.UTF8.GetString((byte[])r.GetValue(i))); + SetToType("String", + (r, i) => Encoding.UTF8.GetString((byte[])r.GetValue(i))[0]); + + // 6) Приведения к беззнаковым, если провайдер возвращает signed-типы + SetToType((r, i) => unchecked((ushort)r.GetInt16(i))); + SetToType((r, i) => unchecked((uint)r.GetInt32(i))); + SetToType((r, i) => unchecked((ulong)r.GetInt64(i))); + } + public override TableOptions SupportedTableOptions => + TableOptions.IsTemporary | + TableOptions.IsLocalTemporaryStructure | + TableOptions.IsLocalTemporaryData | + TableOptions.CreateIfNotExists | + TableOptions.DropIfExists; + + protected override IMemberTranslator CreateMemberTranslator() => new YdbMemberTranslator(); + + public override ISqlBuilder CreateSqlBuilder(MappingSchema mappingSchema, DataOptions dataOptions) + { + return new YdbSqlBuilder(this, mappingSchema, dataOptions, GetSqlOptimizer(dataOptions), SqlProviderFlags); + } + + private readonly ISqlOptimizer _sqlOptimizer; + public override ISqlOptimizer GetSqlOptimizer(DataOptions dataOptions) => _sqlOptimizer; + + public override ISchemaProvider GetSchemaProvider() => new YdbSchemaProvider(); + + // GetSchemaTable not implemented by provider + public override bool? IsDBNullAllowed(DataOptions options, DbDataReader reader, int idx) => true; + + public override void SetParameter(DataConnection dataConnection, DbParameter parameter, string name, DbDataType dataType, object? value) + { + // handle various provider parameter support features/issues + + // provider doesn't support char type + if (value is char chr) + value = chr.ToString(); + + if (dataType.DataType == DataType.Date && value != null) + { + if (value is DateTime dt) + value = DateOnly.FromDateTime(dt); + else if (value is DateTimeOffset dto) + value = DateOnly.FromDateTime(dto.Date); + else if (value is string s) + value = DateOnly.Parse(s, CultureInfo.InvariantCulture); + } + + switch (dataType.DataType) + { + case DataType.Binary: + case DataType.VarBinary: + { + if (value is string str) + value = Encoding.UTF8.GetBytes(str); + } + + break; + + case DataType.SByte: + { + if (value is byte b) + value = checked((sbyte)b); + else if (value is short s) + value = checked((sbyte)s); + else if (value is ushort us) + value = checked((sbyte)us); + else if (value is int i) + value = checked((sbyte)i); + else if (value is uint u) + value = checked((sbyte)u); + else if (value is ulong ul) + value = checked((sbyte)ul); + else if (value is long l) + value = checked((sbyte)l); + else if (value is float f) + value = checked((sbyte)f); + else if (value is double dbl) + value = checked((sbyte)dbl); + else if (value is decimal d) + value = checked((sbyte)d); + } + + break; + + case DataType.Byte: + { + if (value is bool b) + value = b ? (byte)1 : (byte)0; + else if (value is sbyte sb) + value = checked((byte)sb); + else if (value is ushort us) + value = checked((byte)us); + else if (value is short s) + value = checked((byte)s); + else if (value is uint u) + value = checked((byte)u); + else if (value is int i) + value = checked((byte)i); + else if (value is ulong ul) + value = checked((byte)ul); + else if (value is long l) + value = checked((byte)l); + else if (value is decimal d) + value = checked((byte)d); + else if (value is float f) + value = checked((byte)f); + else if (value is double dbl) + value = checked((byte)dbl); + } + + break; + + case DataType.Int16: + { + if (value is ushort us) + value = checked((short)us); + else if (value is uint u) + value = checked((short)u); + else if (value is int i) + value = checked((short)i); + else if (value is long l) + value = checked((short)l); + else if (value is ulong ul) + value = checked((short)ul); + else if (value is decimal d) + value = checked((short)d); + else if (value is float f) + value = checked((short)f); + else if (value is double dbl) + value = checked((short)dbl); + } + + break; + + case DataType.UInt16: + { + if (value is sbyte sb) + value = checked((ushort)sb); + else if (value is short s) + value = checked((ushort)s); + else if (value is uint u) + value = checked((ushort)u); + else if (value is int i) + value = checked((ushort)i); + else if (value is ulong ul) + value = checked((ushort)ul); + else if (value is long l) + value = checked((ushort)l); + else if (value is decimal d) + value = checked((ushort)d); + else if (value is float f) + value = checked((ushort)f); + else if (value is double dbl) + value = checked((ushort)dbl); + } + + break; + + case DataType.Int32: + { + if (value is uint u) + value = checked((int)u); + else if (value is ulong ul) + value = checked((int)ul); + else if (value is long l) + value = checked((int)l); + else if (value is decimal d) + value = checked((int)d); + else if (value is float f) + value = checked((int)f); + else if (value is double dbl) + value = checked((int)dbl); + } + + break; + + case DataType.UInt32: + { + if (value is ulong ul) + value = checked((uint)ul); + else if (value is sbyte sb) + value = checked((uint)sb); + else if (value is short s) + value = checked((uint)s); + else if (value is int i) + value = checked((uint)i); + else if (value is long l) + value = checked((uint)l); + else if (value is decimal d) + value = checked((uint)d); + else if (value is float f) + value = checked((uint)f); + else if (value is double dbl) + value = checked((uint)dbl); + } + + break; + + case DataType.UInt64: + { + if (value is sbyte sb) + value = checked((ulong)sb); + else if (value is short s) + value = checked((ulong)s); + else if (value is int i) + value = checked((ulong)i); + else if (value is long l) + value = checked((ulong)l); + else if (value is decimal d) + value = checked((ulong)d); + else if (value is float f) + value = checked((ulong)f); + else if (value is double dbl) + value = checked((ulong)dbl); + } + + break; + + case DataType.Int64: + { + if (value is ulong ul) + value = checked((long)ul); + else if (value is decimal d) + value = checked((long)d); + else if (value is float f) + value = checked((long)f); + else if (value is double dbl) + value = checked((long)dbl); + } + + break; + + case DataType.DecFloat: + { + if (value is byte b) + value = (decimal)b; + } + + break; + + case DataType.Single: + { + if (value is byte b) + value = checked((float)b); + else if (value is sbyte sb) + value = checked((float)sb); + else if (value is ushort us) + value = checked((float)us); + else if (value is short s) + value = checked((float)s); + else if (value is uint u) + value = checked((float)u); + else if (value is int i) + value = checked((float)i); + else if (value is ulong ul) + value = checked((float)ul); + else if (value is long l) + value = checked((float)l); + else if (value is decimal d) + value = checked((float)d); + else if (value is double dbl) + value = checked((float)dbl); + } + + break; + + case DataType.Double: + { + if (value is byte b) + value = checked((double)b); + else if (value is sbyte sb) + value = checked((double)sb); + else if (value is ushort us) + value = checked((double)us); + else if (value is short s) + value = checked((double)s); + else if (value is uint u) + value = checked((double)u); + else if (value is int i) + value = checked((double)i); + else if (value is ulong ul) + value = checked((double)ul); + else if (value is long l) + value = checked((double)l); + else if (value is decimal d) + value = checked((double)d); + } + + break; + + case DataType.DateTime: + { + if (value is DateTimeOffset dto) + value = dto.LocalDateTime; + } + + break; + + case DataType.DateTime2: + { + if (value is DateTimeOffset dto) + value = dto.UtcDateTime; + } + + break; + + case DataType.Decimal: + { + if (value is byte b ) value = (decimal)b; + else if (value is sbyte sb ) value = (decimal)sb; + else if (value is ushort us ) value = (decimal)us; + else if (value is short s ) value = (decimal)s; + else if (value is uint u ) value = (decimal)u; + else if (value is int i ) value = (decimal)i; + else if (value is ulong ul ) value = (decimal)ul; + else if (value is long l ) value = (decimal)l; + else if (value is float f ) value = checked((decimal)f); + else if (value is double dbl) value = checked((decimal)dbl); + else if (value is string str) + value = Adapter.MakeDecimalFromString(str, dataType.Precision ?? YdbMappingSchema.DEFAULT_DECIMAL_PRECISION, dataType.Scale ?? YdbMappingSchema.DEFAULT_DECIMAL_SCALE); + } + + break; + } + + base.SetParameter(dataConnection, parameter, name, dataType, value); + } + + protected override void SetParameterType(DataConnection dataConnection, DbParameter parameter, DbDataType dataType) + { + YdbProviderAdapter.YdbDbType? type = null; + + switch (dataType.DataType) + { + case DataType.Date: type = YdbProviderAdapter.YdbDbType.Date; break; + case DataType.DateTime: type = YdbProviderAdapter.YdbDbType.DateTime; break; + case DataType.DateTime2: type = YdbProviderAdapter.YdbDbType.Timestamp; break; + case DataType.Json: type = YdbProviderAdapter.YdbDbType.Json; break; + case DataType.BinaryJson: type = YdbProviderAdapter.YdbDbType.JsonDocument; break; + case DataType.Interval: type = YdbProviderAdapter.YdbDbType.Interval; break; + + case DataType.Decimal: + { + if (dataType.Precision != null) + parameter.Precision = (byte)dataType.Precision.Value; + + if (dataType.Scale != null) + parameter.Scale = (byte)dataType.Scale.Value; + + break; + } + } + + if (type != null) + { + var param = TryGetProviderParameter(dataConnection, parameter); + if (param != null) + { + Adapter.SetDbType(param, type.Value); + return; + } + } + + base.SetParameterType(dataConnection, parameter, dataType); + } + + #region BulkCopy + + public override BulkCopyRowsCopied BulkCopy(DataOptions options, ITable table, IEnumerable source) + { + return new YdbBulkCopy(this).BulkCopy( + options.BulkCopyOptions.BulkCopyType == BulkCopyType.Default ? + options.FindOrDefault(YdbOptions.Default).BulkCopyType : + options.BulkCopyOptions.BulkCopyType, + table, + options, + source); + } + + public override Task BulkCopyAsync(DataOptions options, ITable table, + IEnumerable source, CancellationToken cancellationToken) + { + return new YdbBulkCopy(this).BulkCopyAsync( + options.BulkCopyOptions.BulkCopyType == BulkCopyType.Default ? + options.FindOrDefault(YdbOptions.Default).BulkCopyType : + options.BulkCopyOptions.BulkCopyType, + table, + options, + source, + cancellationToken); + } + + public override Task BulkCopyAsync(DataOptions options, ITable table, + IAsyncEnumerable source, CancellationToken cancellationToken) + { + return new YdbBulkCopy(this).BulkCopyAsync( + options.BulkCopyOptions.BulkCopyType == BulkCopyType.Default ? + options.FindOrDefault(YdbOptions.Default).BulkCopyType : + options.BulkCopyOptions.BulkCopyType, + table, + options, + source, + cancellationToken); + } + + #endregion + } +} diff --git a/src/Linq2db.Ydb/src/Internal/YdbMappingSchema.cs b/src/Linq2db.Ydb/src/Internal/YdbMappingSchema.cs new file mode 100644 index 00000000..0209786d --- /dev/null +++ b/src/Linq2db.Ydb/src/Internal/YdbMappingSchema.cs @@ -0,0 +1,695 @@ +using System.Globalization; +using System.Text; +using LinqToDB.Internal.Common; +using LinqToDB.Internal.Mapping; +using LinqToDB.Mapping; +using LinqToDB.SqlQuery; + +namespace LinqToDB.Internal.DataProvider.Ydb.Internal +{ + public sealed class YdbMappingSchema : LockedMappingSchema + { + // provider hardcodes precision/scale to those values for parameters, so we will use them as safe defaults for now + internal const int DEFAULT_DECIMAL_PRECISION = 22; + internal const int DEFAULT_DECIMAL_SCALE = 9; + internal const string DEFAULT_TIMEZONE = "GMT"; + +#if SUPPORTS_COMPOSITE_FORMAT + private static readonly CompositeFormat DATE_FORMAT = CompositeFormat.Parse("Date('{0:yyyy-MM-dd}')"); + private static readonly CompositeFormat DATETIME_FORMAT = CompositeFormat.Parse("Datetime('{0:yyyy-MM-ddTHH:mm:ssZ}')"); + private static readonly CompositeFormat TIMESTAMP_FORMAT = CompositeFormat.Parse("Timestamp('{0:yyyy-MM-ddTHH:mm:ss.ffffffZ}')"); + + private static readonly CompositeFormat TZ_DATE_FORMAT = CompositeFormat.Parse("TzDate('{0:yyyy-MM-dd},{1}')"); + private static readonly CompositeFormat TZ_DATETIME_FORMAT = CompositeFormat.Parse("TzDatetime('{0:yyyy-MM-ddTHH:mm:ss.fff},{1}')"); + private static readonly CompositeFormat TZ_TIMESTAMP_FORMAT = CompositeFormat.Parse("TzTimestamp('{0:yyyy-MM-ddTHH:mm:ss.fff},{1}')"); + + private static readonly CompositeFormat UUID_FORMAT = CompositeFormat.Parse("Uuid('{0:D}')"); + private static readonly CompositeFormat DECIMAL_FORMAT = CompositeFormat.Parse("Decimal('{0}', {1}, {2})"); + + private static readonly CompositeFormat INT8_FORMAT = CompositeFormat.Parse("{0}t"); + private static readonly CompositeFormat UINT8_FORMAT = CompositeFormat.Parse("{0}ut"); + private static readonly CompositeFormat INT16_FORMAT = CompositeFormat.Parse("{0}s"); + private static readonly CompositeFormat UINT16_FORMAT = CompositeFormat.Parse("{0}us"); + private static readonly CompositeFormat INT32_FORMAT = CompositeFormat.Parse("{0}"); + private static readonly CompositeFormat UINT32_FORMAT = CompositeFormat.Parse("{0}u"); + private static readonly CompositeFormat INT64_FORMAT = CompositeFormat.Parse("{0}l"); + private static readonly CompositeFormat UINT64_FORMAT = CompositeFormat.Parse("{0}ul"); + private static readonly CompositeFormat FLOAT_FORMAT = CompositeFormat.Parse("Float('{0:G9}')"); + private static readonly CompositeFormat DOUBLE_FORMAT = CompositeFormat.Parse("Double('{0:G17}')"); + private static readonly CompositeFormat DY_NUMBER_FORMAT = CompositeFormat.Parse("DyNumber('{0}')"); + +#else + private const string DATE_FORMAT = "Date('{0:yyyy-MM-dd}')"; + private const string DATETIME_FORMAT = "Datetime('{0:yyyy-MM-ddTHH:mm:ssZ}')"; + private const string TIMESTAMP_FORMAT = "Timestamp('{0:yyyy-MM-ddTHH:mm:ss.ffffffZ}')"; + + private const string TZ_DATE_FORMAT = "TzDate('{0:yyyy-MM-dd},{1}')"; + private const string TZ_DATETIME_FORMAT = "TzDatetime('{0:yyyy-MM-ddTHH:mm:ss.fff},{1}')"; + private const string TZ_TIMESTAMP_FORMAT = "TzTimestamp('{0:yyyy-MM-ddTHH:mm:ss.fff},{1}')"; + + private const string UUID_FORMAT = "Uuid('{0:D}')"; + private const string DECIMAL_FORMAT = "Decimal('{0}', {1}, {2})"; + + private const string INT8_FORMAT = "{0}t"; + private const string UINT8_FORMAT = "{0}ut"; + private const string INT16_FORMAT = "{0}s"; + private const string UINT16_FORMAT = "{0}us"; + private const string INT32_FORMAT = "{0}"; + private const string UINT32_FORMAT = "{0}u"; + private const string INT64_FORMAT = "{0}l"; + private const string UINT64_FORMAT = "{0}ul"; + private const string FLOAT_FORMAT = "Float('{0:G9}')"; + private const string DOUBLE_FORMAT = "Double('{0:G17}')"; + private const string DY_NUMBER_FORMAT = "DyNumber('{0}')"; +#endif + + public YdbMappingSchema() : base("YDB") + { + AddScalarType(typeof(DateTimeOffset), DataType.DateTime2); + AddScalarType(typeof(TimeSpan), DataType.Interval); + AddScalarType(typeof(MemoryStream), DataType.VarBinary); + + SetValueToSqlConverter(typeof(string) , (sb,dt,_,v) => ConvertString (sb, dt, (string)v)); + SetValueToSqlConverter(typeof(char) , (sb,dt,_,v) => ConvertString (sb, dt, ((char)v).ToString())); + SetValueToSqlConverter(typeof(byte[]) , (sb,dt,_,v) => ConvertByteArray (sb, dt, (byte[])v)); + SetValueToSqlConverter(typeof(MemoryStream) , (sb,dt,_,v) => ConvertByteArray (sb, dt, ((MemoryStream)v).ToArray())); + + SetValueToSqlConverter(typeof(byte) , (sb,dt,_,v) => ConvertByte (sb, dt, (byte)v)); + SetValueToSqlConverter(typeof(sbyte) , (sb,dt,_,v) => ConvertSByte (sb, dt, (sbyte)v)); + SetValueToSqlConverter(typeof(short) , (sb,dt,_,v) => ConvertInt16 (sb, dt, (short)v)); + SetValueToSqlConverter(typeof(ushort) , (sb,dt,_,v) => ConvertUInt16 (sb, dt, (ushort)v)); + SetValueToSqlConverter(typeof(int) , (sb,dt,_,v) => ConvertInt32 (sb, dt, (int)v)); + SetValueToSqlConverter(typeof(uint) , (sb,dt,_,v) => ConvertUInt32 (sb, dt, (uint)v)); + SetValueToSqlConverter(typeof(long) , (sb,dt,_,v) => ConvertInt64 (sb, dt, (long)v)); + SetValueToSqlConverter(typeof(ulong) , (sb,dt,_,v) => ConvertUInt64 (sb, dt, (ulong)v)); + SetValueToSqlConverter(typeof(float) , (sb,dt,_,v) => ConvertFloat (sb, dt, (float)v)); + SetValueToSqlConverter(typeof(double) , (sb,dt,_,v) => ConvertDouble (sb, dt, (double)v)); + SetValueToSqlConverter(typeof(decimal) , (sb,dt,_,v) => ConvertDecimal (sb, dt, (decimal)v)); + + SetValueToSqlConverter(typeof(Guid) , (sb,dt,_,v) => ConvertGuid (sb, dt, (Guid)v)); + SetValueToSqlConverter(typeof(bool) , (sb,dt,_,v) => ConvertBool (sb, dt, (bool)v)); + + SetValueToSqlConverter(typeof(TimeSpan) , (sb,dt,_,v) => ConvertTimeSpan (sb, dt, (TimeSpan)v)); + SetValueToSqlConverter(typeof(DateTime) , (sb,dt,_,v) => ConvertDateTime (sb, dt, (DateTime)v)); + SetValueToSqlConverter(typeof(DateTimeOffset), (sb,dt,_,v) => ConvertDateTimeOffset(sb, dt, (DateTimeOffset)v)); +#if SUPPORTS_DATEONLY + SetValueToSqlConverter(typeof(DateOnly) , (sb,dt,_,v) => ConvertDateOnly (sb, dt, (DateOnly)v)); +#endif + } + + #region Type to SQL converters (for multi-bindings) + + private static void ConvertString(StringBuilder stringBuilder, SqlDataType dt, string value) + { + if (dt.Type.DataType == DataType.DecFloat) + { + BuildDyNumberLiteral(stringBuilder, value); + } + else if (dt.Type.DataType == DataType.Decimal) + { + BuildDecimalLiteral(stringBuilder, value, dt); + } + else if (dt.Type.DataType == DataType.BinaryJson) + { + stringBuilder.Append("JsonDocument("); + BuildStringLiteral(stringBuilder, value); + stringBuilder.Append(')'); + } + else + { + BuildStringLiteral(stringBuilder, value); + + // apply type for non-String types + var suffix = dt.Type.DataType switch + { + DataType.DecFloat or DataType.Decimal or DataType.VarBinary or DataType.Binary or DataType.Blob => 's', + DataType.Json => 'j', + _ => 'u' + }; + + stringBuilder.Append(suffix); + } + } + + private static void ConvertByteArray(StringBuilder stringBuilder, SqlDataType dt, byte[] value) + { + if (dt.Type.DataType == DataType.BinaryJson) + { + stringBuilder.Append("JsonDocument("); + BuildBinaryLiteral(stringBuilder, value); + stringBuilder.Append(')'); + } + else + { + BuildBinaryLiteral(stringBuilder, value); + + // apply type for non-String types + var suffix = dt.Type.DataType switch + { + DataType.Json => 'j', + _ => 's' + }; + + stringBuilder.Append(suffix); + } + } + + private static void ConvertGuid(StringBuilder sb, SqlDataType dt, Guid value) + { + switch (dt.Type.DataType) + { + case DataType.VarChar: + case DataType.NVarChar: + ConvertString(sb, dt, value.ToString("D")); + break; + case DataType.Binary : + ConvertByteArray(sb, dt, value.ToByteArray()); + break; + case DataType.Guid : + default : + BuildUUIDLiteral(sb, value); + break; + } + } + + private static void ConvertByte(StringBuilder sb, SqlDataType dt, byte value) + { + switch (dt.Type.DataType) + { + case DataType.SByte : BuildSByteLiteral(sb, checked((sbyte)value)); return; + default : + case DataType.Byte : BuildByteLiteral(sb, value); return; + case DataType.UInt16 : BuildUInt16Literal(sb, value); return; + case DataType.Int16 : BuildInt16Literal(sb, value); return; + case DataType.UInt32 : BuildUInt32Literal(sb, value); return; + case DataType.Int32 : BuildInt32Literal(sb, value); return; + case DataType.UInt64 : BuildUInt64Literal(sb, value); return; + case DataType.Int64 : BuildInt64Literal(sb, value); return; + case DataType.Single : BuildFloatLiteral(sb, value); return; + case DataType.Double : BuildDoubleLiteral(sb, value); return; + case DataType.Decimal : BuildDecimalLiteral(sb, value, dt); return; + case DataType.DecFloat : BuildDyNumberLiteral(sb, (decimal)value); return; + } + } + + private static void ConvertSByte(StringBuilder sb, SqlDataType dt, sbyte value) + { + switch (dt.Type.DataType) + { + case DataType.Byte : BuildByteLiteral(sb, checked((byte)value)); return; + default : + case DataType.SByte : BuildSByteLiteral(sb, value); return; + case DataType.UInt16 : BuildUInt16Literal(sb, checked((ushort)value)); return; + case DataType.Int16 : BuildInt16Literal(sb, value); return; + case DataType.UInt32 : BuildUInt32Literal(sb, checked((uint)value)); return; + case DataType.Int32 : BuildInt32Literal(sb, value); return; + case DataType.UInt64 : BuildUInt64Literal(sb, checked((ulong)value)); return; + case DataType.Int64 : BuildInt64Literal(sb, value); return; + case DataType.Single : BuildFloatLiteral(sb, value); return; + case DataType.Double : BuildDoubleLiteral(sb, value); return; + case DataType.Decimal : BuildDecimalLiteral(sb, value, dt); return; + case DataType.DecFloat : BuildDyNumberLiteral(sb, (decimal)value); return; + } + } + + private static void ConvertInt16(StringBuilder sb, SqlDataType dt, short value) + { + switch (dt.Type.DataType) + { + case DataType.Byte : BuildByteLiteral(sb, checked((byte)value)); return; + case DataType.SByte : BuildSByteLiteral(sb, checked((sbyte)value)); return; + case DataType.UInt16 : BuildUInt16Literal(sb, checked((ushort)value)); return; + default : + case DataType.Int16 : BuildInt16Literal(sb, value); return; + case DataType.UInt32 : BuildUInt32Literal(sb, checked((uint)value)); return; + case DataType.Int32 : BuildInt32Literal(sb, value); return; + case DataType.UInt64 : BuildUInt64Literal(sb, checked((ulong)value)); return; + case DataType.Int64 : BuildInt64Literal(sb, value); return; + case DataType.Single : BuildFloatLiteral(sb, value); return; + case DataType.Double : BuildDoubleLiteral(sb, value); return; + case DataType.Decimal : BuildDecimalLiteral(sb, value, dt); return; + case DataType.DecFloat : BuildDyNumberLiteral(sb, (decimal)value); return; + } + } + + private static void ConvertUInt16(StringBuilder sb, SqlDataType dt, ushort value) + { + switch (dt.Type.DataType) + { + case DataType.Byte : BuildByteLiteral(sb, checked((byte)value)); return; + case DataType.SByte : BuildSByteLiteral(sb, checked((sbyte)value)); return; + default : + case DataType.UInt16 : BuildUInt16Literal(sb, value); return; + case DataType.Int16 : BuildInt16Literal(sb, checked((short)value)); return; + case DataType.UInt32 : BuildUInt32Literal(sb, checked((uint)value)); return; + case DataType.Int32 : BuildInt32Literal(sb, value); return; + case DataType.UInt64 : BuildUInt64Literal(sb, checked((ulong)value)); return; + case DataType.Int64 : BuildInt64Literal(sb, value); return; + case DataType.Single : BuildFloatLiteral(sb, value); return; + case DataType.Double : BuildDoubleLiteral(sb, value); return; + case DataType.Decimal : BuildDecimalLiteral(sb, value, dt); return; + case DataType.DecFloat : BuildDyNumberLiteral(sb, (decimal)value); return; + } + } + + private static void ConvertInt32(StringBuilder sb, SqlDataType dt, int value) + { + switch (dt.Type.DataType) + { + case DataType.Byte : BuildByteLiteral(sb, checked((byte)value)); return; + case DataType.SByte : BuildSByteLiteral(sb, checked((sbyte)value)); return; + case DataType.Int16 : BuildInt16Literal(sb, checked((short)value)); return; + case DataType.UInt16 : BuildUInt16Literal(sb, checked((ushort)value)); return; + case DataType.UInt32 : BuildUInt32Literal(sb, checked((uint)value)); return; + default : + case DataType.Int32 : BuildInt32Literal(sb, value); return; + case DataType.UInt64 : BuildUInt64Literal(sb, checked((ulong)value)); return; + case DataType.Int64 : BuildInt64Literal(sb, value); return; + case DataType.Single : BuildFloatLiteral(sb, value); return; + case DataType.Double : BuildDoubleLiteral(sb, value); return; + case DataType.Decimal : BuildDecimalLiteral(sb, value, dt); return; + case DataType.DecFloat : BuildDyNumberLiteral(sb, (decimal)value); return; + } + } + + private static void ConvertUInt32(StringBuilder sb, SqlDataType dt, uint value) + { + switch (dt.Type.DataType) + { + case DataType.Byte : BuildByteLiteral(sb, checked((byte)value)); return; + case DataType.SByte : BuildSByteLiteral(sb, checked((sbyte)value)); return; + case DataType.Int16 : BuildInt16Literal(sb, checked((short)value)); return; + case DataType.UInt16 : BuildUInt16Literal(sb, checked((ushort)value)); return; + default : + case DataType.UInt32 : BuildUInt32Literal(sb, value); return; + case DataType.Int32 : BuildInt32Literal(sb, checked((int)value)); return; + case DataType.UInt64 : BuildUInt64Literal(sb, checked((ulong)value)); return; + case DataType.Int64 : BuildInt64Literal(sb, value); return; + case DataType.Single : BuildFloatLiteral(sb, value); return; + case DataType.Double : BuildDoubleLiteral(sb, value); return; + case DataType.Decimal : BuildDecimalLiteral(sb, value, dt); return; + case DataType.DecFloat : BuildDyNumberLiteral(sb, (decimal)value); return; + } + } + + private static void ConvertInt64(StringBuilder sb, SqlDataType dt, long value) + { + switch (dt.Type.DataType) + { + case DataType.Byte : BuildByteLiteral(sb, checked((byte)value)); return; + case DataType.SByte : BuildSByteLiteral(sb, checked((sbyte)value)); return; + case DataType.Int16 : BuildInt16Literal(sb, checked((short)value)); return; + case DataType.UInt16 : BuildUInt16Literal(sb, checked((ushort)value)); return; + case DataType.Int32 : BuildInt32Literal(sb, checked((int)value)); return; + case DataType.UInt32 : BuildUInt32Literal(sb, checked((uint)value)); return; + default : + case DataType.Int64 : BuildInt64Literal(sb, value); return; + case DataType.UInt64 : BuildUInt64Literal(sb, checked((ulong)value)); return; + case DataType.Single : BuildFloatLiteral(sb, value); return; + case DataType.Double : BuildDoubleLiteral(sb, value); return; + case DataType.Decimal : BuildDecimalLiteral(sb, value, dt); return; + case DataType.DecFloat : BuildDyNumberLiteral(sb, (decimal)value); return; + } + } + + private static void ConvertUInt64(StringBuilder sb, SqlDataType dt, ulong value) + { + switch (dt.Type.DataType) + { + case DataType.Byte : BuildByteLiteral(sb, checked((byte)value)); return; + case DataType.SByte : BuildSByteLiteral(sb, checked((sbyte)value)); return; + case DataType.Int16 : BuildInt16Literal(sb, checked((short)value)); return; + case DataType.UInt16 : BuildUInt16Literal(sb, checked((ushort)value)); return; + case DataType.Int32 : BuildInt32Literal(sb, checked((int)value)); return; + case DataType.UInt32 : BuildUInt32Literal(sb, checked((uint)value)); return; + default : + case DataType.UInt64 : BuildUInt64Literal(sb, value); return; + case DataType.Int64 : BuildInt64Literal(sb, checked((long)value)); return; + case DataType.Single : BuildFloatLiteral(sb, value); return; + case DataType.Double : BuildDoubleLiteral(sb, value); return; + case DataType.Decimal : BuildDecimalLiteral(sb, value, dt); return; + case DataType.DecFloat : BuildDyNumberLiteral(sb, (decimal)value); return; + } + } + + private static void ConvertFloat(StringBuilder sb, SqlDataType dt, float value) + { + switch (dt.Type.DataType) + { + case DataType.Byte : BuildByteLiteral(sb, checked((byte)value)); return; + case DataType.SByte : BuildSByteLiteral(sb, checked((sbyte)value)); return; + case DataType.Int16 : BuildInt16Literal(sb, checked((short)value)); return; + case DataType.UInt16 : BuildUInt16Literal(sb, checked((ushort)value)); return; + case DataType.Int32 : BuildInt32Literal(sb, checked((int)value)); return; + case DataType.UInt32 : BuildUInt32Literal(sb, checked((uint)value)); return; + case DataType.UInt64 : BuildUInt64Literal(sb, checked((ulong)value)); return; + case DataType.Int64 : BuildInt64Literal(sb, checked((long)value)); return; + default : + case DataType.Single : BuildFloatLiteral(sb, value); return; + case DataType.Double : BuildDoubleLiteral(sb, value); return; + case DataType.Decimal : BuildDecimalLiteral(sb, checked((decimal)value), dt); return; + case DataType.DecFloat : BuildDyNumberLiteral(sb, (double)value); return; + } + } + + private static void ConvertDouble(StringBuilder sb, SqlDataType dt, double value) + { + switch (dt.Type.DataType) + { + case DataType.Byte : BuildByteLiteral(sb, checked((byte)value)); return; + case DataType.SByte : BuildSByteLiteral(sb, checked((sbyte)value)); return; + case DataType.Int16 : BuildInt16Literal(sb, checked((short)value)); return; + case DataType.UInt16 : BuildUInt16Literal(sb, checked((ushort)value)); return; + case DataType.Int32 : BuildInt32Literal(sb, checked((int)value)); return; + case DataType.UInt32 : BuildUInt32Literal(sb, checked((uint)value)); return; + case DataType.UInt64 : BuildUInt64Literal(sb, checked((ulong)value)); return; + case DataType.Int64 : BuildInt64Literal(sb, checked((long)value)); return; + case DataType.Single : BuildFloatLiteral(sb, checked((float)value)); return; + default : + case DataType.Double : BuildDoubleLiteral(sb, value); return; + case DataType.Decimal : BuildDecimalLiteral(sb, checked((decimal)value), dt); return; + case DataType.DecFloat : BuildDyNumberLiteral(sb, value); return; + } + } + + private static void ConvertDecimal(StringBuilder sb, SqlDataType dt, decimal value) + { + switch (dt.Type.DataType) + { + case DataType.Byte : BuildByteLiteral(sb, checked((byte)value)); return; + case DataType.SByte : BuildSByteLiteral(sb, checked((sbyte)value)); return; + case DataType.Int16 : BuildInt16Literal(sb, checked((short)value)); return; + case DataType.UInt16 : BuildUInt16Literal(sb, checked((ushort)value)); return; + case DataType.Int32 : BuildInt32Literal(sb, checked((int)value)); return; + case DataType.UInt32 : BuildUInt32Literal(sb, checked((uint)value)); return; + case DataType.UInt64 : BuildUInt64Literal(sb, checked((ulong)value)); return; + case DataType.Int64 : BuildInt64Literal(sb, checked((long)value)); return; + case DataType.Single : BuildFloatLiteral(sb, checked((float)value)); return; + case DataType.Double : BuildDoubleLiteral(sb, checked((double)value)); return; + default : + case DataType.Decimal : BuildDecimalLiteral(sb, checked((decimal)value), dt); return; + case DataType.DecFloat : BuildDyNumberLiteral(sb, value); return; + } + } + + private static void ConvertTimeSpan(StringBuilder sb, SqlDataType dt, TimeSpan value) + { + switch (dt.Type.DataType) + { + case DataType.Int64 : ConvertInt64(sb, dt, value.Ticks); break; + default : + case DataType.Interval : BuildIntervalLiteral(sb, value); break; + } + } + + private static void ConvertDateTime(StringBuilder sb, SqlDataType dt, DateTime value) + { + switch (dt.Type.DataType) + { + case DataType.Date : BuildDateLiteral(sb, value.Date); break; + case DataType.DateTime : BuildDateTimeLiteral(sb, value); break; + default : + case DataType.DateTime2 : BuildTimestampLiteral(sb, value); break; + } + } + + private static void ConvertDateTimeOffset(StringBuilder sb, SqlDataType dt, DateTimeOffset value) + { + switch (dt.Type.DataType) + { + case DataType.Date : BuildDateLiteral(sb, value.Date); break; + case DataType.DateTime : BuildDateTimeLiteral(sb, value.LocalDateTime); break; + default : + case DataType.DateTime2 : BuildTimestampLiteral(sb, value.UtcDateTime); break; + } + } + +#if SUPPORTS_DATEONLY + private static void ConvertDateOnly(StringBuilder sb, SqlDataType dt, DateOnly value) + { + switch (dt.Type.DataType) + { + default : + case DataType.Date : BuildDateLiteral(sb, value.ToDateTime(default)); break; + case DataType.DateTz: BuildDateTzLiteral(sb, value.ToDateTime(default)); break; + } + } +#endif + + private static void ConvertBool(StringBuilder sb, SqlDataType dt, bool value) + { + switch (dt.Type.DataType) + { + case DataType.Byte : BuildByteLiteral(sb, value ? (byte)1 : (byte)0); return; + default : + case DataType.Boolean : BuildBooleanLiteral(sb, value); return; + } + } +#endregion + +#region Literal generators + + private static void BuildStringLiteral(StringBuilder stringBuilder, string value) + { + stringBuilder.Append('\''); + + foreach (var chr in value) + { + switch (chr) + { + case '\0': + stringBuilder.Append("\\x00"); + continue; + case '\'': + stringBuilder.Append("\\'"); + continue; + case '\\': + stringBuilder.Append("\\\\"); + continue; + } + + stringBuilder.Append(chr); + } + + stringBuilder.Append('\''); + } + + private static void BuildBinaryLiteral(StringBuilder stringBuilder, byte[] value) + { + stringBuilder + .Append('\''); + + BuildHexString(stringBuilder, value); + + stringBuilder + .Append('\''); + } + + private static void BuildHexString(StringBuilder stringBuilder, byte[] value) + { + foreach (var @byte in value) + stringBuilder + .Append("\\x") + .AppendByteAsHexViaLookup32(@byte); + } + + private static void BuildUUIDLiteral(StringBuilder sb, Guid value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, UUID_FORMAT, value); + } + + private static void BuildByteLiteral(StringBuilder sb, byte value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, UINT8_FORMAT, value); + } + + private static void BuildSByteLiteral(StringBuilder sb, sbyte value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, INT8_FORMAT, value); + } + + private static void BuildInt16Literal(StringBuilder sb, short value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, INT16_FORMAT, value); + } + + private static void BuildUInt16Literal(StringBuilder sb, ushort value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, UINT16_FORMAT, value); + } + + private static void BuildInt32Literal(StringBuilder sb, int value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, INT32_FORMAT, value); + } + + private static void BuildUInt32Literal(StringBuilder sb, uint value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, UINT32_FORMAT, value); + } + + private static void BuildInt64Literal(StringBuilder sb, long value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, INT64_FORMAT, value); + } + + private static void BuildUInt64Literal(StringBuilder sb, ulong value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, UINT64_FORMAT, value); + } + + private static void BuildFloatLiteral(StringBuilder sb, float value) + { + if (float.IsNegativeInfinity(value)) + sb.Append("Float('-inf')"); + else if (float.IsPositiveInfinity(value)) + sb.Append("Float('inf')"); + else + sb.AppendFormat(CultureInfo.InvariantCulture, FLOAT_FORMAT, value); + } + + private static void BuildDoubleLiteral(StringBuilder sb, double value) + { + if (double.IsNegativeInfinity(value)) + sb.Append("Double('-inf')"); + else if (double.IsPositiveInfinity(value)) + sb.Append("Double('inf')"); + else + sb.AppendFormat(CultureInfo.InvariantCulture, DOUBLE_FORMAT, value); + } + + private static void BuildDecimalLiteral(StringBuilder sb, decimal value, SqlDataType dt) + { + sb.AppendFormat( + CultureInfo.InvariantCulture, + DECIMAL_FORMAT, + value, + dt.Type.Precision ?? DEFAULT_DECIMAL_PRECISION, + dt.Type.Scale ?? DEFAULT_DECIMAL_SCALE); + } + + private static void BuildDecimalLiteral(StringBuilder sb, string value, SqlDataType dt) + { + sb.AppendFormat( + CultureInfo.InvariantCulture, + DECIMAL_FORMAT, + value, + dt.Type.Precision ?? DEFAULT_DECIMAL_PRECISION, + dt.Type.Scale ?? DEFAULT_DECIMAL_SCALE); + } + + private static void BuildDyNumberLiteral(StringBuilder sb, decimal value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, DY_NUMBER_FORMAT, value); + } + + private static void BuildDyNumberLiteral(StringBuilder sb, string value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, DY_NUMBER_FORMAT, value); + } + + private static void BuildDyNumberLiteral(StringBuilder sb, double value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, DY_NUMBER_FORMAT, value); + } + + private static void BuildBooleanLiteral(StringBuilder sb, bool value) + { + sb.Append(value ? "true" : "false"); + } + + private static void BuildIntervalLiteral(StringBuilder sb, TimeSpan value) + { + sb.Append("Interval('"); + // looks like YDB doesn't support non-constant quantifiers Y and M + // which rises question what it means to have min/max values limited by 136 years... + value = value.Ticks % 10 != 0 ? TimeSpan.FromTicks((value.Ticks / 10) * 10) : value; + ConvertToIso8601Interval(sb, value); + sb.Append("')"); + } + + private static void ConvertToIso8601Interval(StringBuilder stringBuilder, TimeSpan interval) + { + var addTicks = 0; + + if (interval < TimeSpan.Zero) + { + _ = stringBuilder.Append('-'); + if (interval == TimeSpan.MinValue) + { + interval = TimeSpan.MaxValue; + addTicks = 1; + } + else + { + interval = interval.Negate(); + } + } + + _ = stringBuilder.Append('P'); + + var iv = CultureInfo.InvariantCulture; + + var ticks = interval.Ticks - new TimeSpan(interval.Days, interval.Hours, interval.Minutes, interval.Seconds, 0).Ticks + addTicks; + + if (interval.Days != 0 || interval == TimeSpan.Zero) + { + _ = stringBuilder.AppendFormat(iv, "{0}D", interval.Days); + } + + if (interval.Hours != 0 || interval.Minutes != 0 || interval.Seconds != 0 || ticks != 0) + { + _ = stringBuilder.Append('T'); + } + + if (interval.Hours != 0) + { + _ = stringBuilder.AppendFormat(iv, "{0}H", interval.Hours); + } + + if (interval.Minutes != 0 + || (interval.Hours != 0 && (interval.Seconds != 0 || ticks != 0))) + { + _ = stringBuilder.AppendFormat(iv, "{0}M", interval.Minutes); + } + + if (interval.Seconds != 0 || ticks != 0) + { + _ = stringBuilder.AppendFormat(iv, "{0}", interval.Seconds); + if (ticks > 0) + { + _ = stringBuilder.Append('.'); + var ticksStr = ticks.ToString(iv); + if (ticksStr.Length < 7) + { + ticksStr = new string('0', 7 - ticksStr.Length) + ticksStr; + } + + _ = stringBuilder.Append(ticksStr.TrimEnd('0')); + } + + _ = stringBuilder.Append('S'); + } + } + + private static void BuildDateLiteral(StringBuilder sb, DateTime value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, DATE_FORMAT, value); + } + + private static void BuildDateTimeLiteral(StringBuilder sb, DateTime value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, DATETIME_FORMAT, value); + } + + + private static void BuildTimestampLiteral(StringBuilder sb, DateTime value) + { + sb.AppendFormat(CultureInfo.InvariantCulture, TIMESTAMP_FORMAT, value); + } + + #endregion + + public static MappingSchema Instance { get; } = new YdbMappingSchema(); + } +} diff --git a/src/Linq2db.Ydb/src/Internal/YdbProviderAdapter.cs b/src/Linq2db.Ydb/src/Internal/YdbProviderAdapter.cs new file mode 100644 index 00000000..fd6ddac8 --- /dev/null +++ b/src/Linq2db.Ydb/src/Internal/YdbProviderAdapter.cs @@ -0,0 +1,337 @@ +using System.Data.Common; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Linq.Expressions; +using System.Numerics; +using System.Reflection; +using LinqToDB.Common; +using LinqToDB.DataProvider; +using LinqToDB.Expressions; +using LinqToDB.Internal.Expressions.Types; + +namespace LinqToDB.Internal.DataProvider.Ydb.Internal +{ + /* + * Misc notes: + * - supported default isolation levels: Unspecified/Serializable (same behavior) === TxMode.SerializableRw + * + * Optional/future features: + * - TODO: add provider-specific retry policy to support YdbException.IsTransientWhenIdempotent + * - TODO: add support for BeginTransaction(TxMode mode) + */ + public sealed class YdbProviderAdapter : IDynamicProviderAdapter + { + public const string AssemblyName = "Ydb.Sdk"; + public const string ClientNamespace = "Ydb.Sdk.Ado"; + + private const string ProtosAssemblyName = "Ydb.Protos"; + + // custom reader methods + readonly Func _getBytes; + readonly Func _getSByte; + readonly Func _getUInt16; + readonly Func _getUInt32; + readonly Func _getUInt64; + readonly Func _getInterval; + readonly Func _getJson; + readonly Func _getJsonDocument; + +// --- 2) Публичные (internal) статические врапперы для использования в Set* вызовах --- + internal static byte[] GetBytes(DbDataReader r, int i) => Instance._getBytes(r, i); + internal static sbyte GetSByte(DbDataReader r, int i) => Instance._getSByte(r, i); + internal static ushort GetUInt16(DbDataReader r, int i) => Instance._getUInt16(r, i); + internal static uint GetUInt32(DbDataReader r, int i) => Instance._getUInt32(r, i); + internal static ulong GetUInt64(DbDataReader r, int i) => Instance._getUInt64(r, i); + internal static TimeSpan GetInterval(DbDataReader r, int i) => Instance._getInterval(r, i); + internal static string GetJson(DbDataReader r, int i) => Instance._getJson(r, i); + internal static string GetJsonDocument(DbDataReader r, int i) => Instance._getJsonDocument(r, i); + + private Func BuildReaderGetter(string methodName) + { + var mi = DataReaderType.GetMethod(methodName, new[] { typeof(int) }); + if (mi == null) + { + return (r, i) => r.GetFieldValue(i); + } + + var pR = Expression.Parameter(typeof(DbDataReader), "r"); + var pI = Expression.Parameter(typeof(int), "i"); + var call = Expression.Call(Expression.Convert(pR, DataReaderType), mi, pI); + + Expression body = call; + if (mi.ReturnType != typeof(T)) + body = Expression.Convert(call, typeof(T)); + + return Expression.Lambda>(body, pR, pI).Compile(); + } + + YdbProviderAdapter() + { + + var assembly = Common.Tools.TryLoadAssembly(AssemblyName, null) + ?? throw new InvalidOperationException($"Cannot load assembly {AssemblyName}."); + var protosAsembly = Common.Tools.TryLoadAssembly(ProtosAssemblyName, null) + ?? throw new InvalidOperationException($"Cannot load assembly {ProtosAssemblyName}."); + + ConnectionType = assembly.GetType($"{ClientNamespace}.YdbConnection", true)!; + CommandType = assembly.GetType($"{ClientNamespace}.YdbCommand", true)!; + ParameterType = assembly.GetType($"{ClientNamespace}.YdbParameter", true)!; + DataReaderType = assembly.GetType($"{ClientNamespace}.YdbDataReader", true)!; + TransactionType = assembly.GetType($"{ClientNamespace}.YdbTransaction", true)!; + + _getBytes = BuildReaderGetter("GetBytes"); + _getSByte = BuildReaderGetter("GetSByte"); + _getUInt16 = BuildReaderGetter("GetUInt16"); + _getUInt32 = BuildReaderGetter("GetUInt32"); + _getUInt64 = BuildReaderGetter("GetUInt64"); + _getInterval = BuildReaderGetter("GetInterval"); + _getJson = BuildReaderGetter("GetJson"); + _getJsonDocument = BuildReaderGetter("GetJsonDocument"); + + var parameterType = assembly.GetType($"{ClientNamespace}.YdbParameter" , true)!; + var dbType = assembly.GetType($"{ClientNamespace}.YdbType.YdbDbType" , true)!; + var bulkCopy = assembly.GetType("Ydb.Sdk.Ado.BulkUpsert.IBulkUpsertImporter", true)!; + + var typeMapper = new TypeMapper(); + + typeMapper.RegisterTypeWrapper(ConnectionType); + typeMapper.RegisterTypeWrapper(bulkCopy); + typeMapper.RegisterTypeWrapper(parameterType); + typeMapper.RegisterTypeWrapper(dbType); + + typeMapper.FinalizeMappings(); + + _connectionFactory = typeMapper.BuildTypedFactory(connectionString => new YdbConnection(connectionString)); + var mapped = typeMapper.MapExpression(() => YdbConnection.ClearAllPools()); + ClearAllPools = typeMapper.BuildFunc(Expression.Lambda>(mapped)); + ClearPool = typeMapper.BuildFunc( + typeMapper.MapLambda((YdbConnection connection) => YdbConnection.ClearPool(connection))); + + var paramMapper = typeMapper.Type(); + var dbTypeBuilder = paramMapper.Member(p => p.YdbDbType); + SetDbType = dbTypeBuilder.BuildSetter(); + GetDbType = dbTypeBuilder.BuildGetter(); + + MakeDecimalFromString = (value, p, s) => + { + return decimal.Parse(value, NumberStyles.Number, CultureInfo.InvariantCulture); + }; + + var pConnection = Expression.Parameter(typeof(DbConnection)); + var pName = Expression.Parameter(typeof(string)); + var pColumns = Expression.Parameter(typeof(IReadOnlyList)); + var pToken = Expression.Parameter(typeof(CancellationToken)); + + BeginBulkCopy = Expression.Lambda, CancellationToken, IBulkUpsertImporter>>( + typeMapper.MapExpression((DbConnection conn, string name, IReadOnlyList columns, CancellationToken cancellationToken) => typeMapper.Wrap(((YdbConnection)(object)conn).BeginBulkUpsertImport(name, columns, cancellationToken)), pConnection, pName, pColumns, pToken), + pConnection, pName, pColumns, pToken) + .CompileExpression(); + } + + record struct DecimalValue(ulong Low, ulong High, uint Precision, uint Scale); + + private static DecimalValue MakeDecimalValue(string value, int precision, int scale) + { + var valuePrecision = value.Count(char.IsDigit); + var dot = value.IndexOf('.'); + var valueScale = dot == -1 ? 0 : value.Length - dot - 1; + + if (valueScale < scale) + { + if (dot == -1) + value += "."; + + value += new string('0', scale - valueScale); + valuePrecision += scale - valueScale; + } + +#if SUPPORTS_INT128 + var raw128 = Int128.Parse(value.Replace(".", ""), CultureInfo.InvariantCulture); + + var low64 = (ulong)(raw128 & 0xFFFFFFFFFFFFFFFF); + var high64 = (ulong)(raw128 >> 64); +#else + var raw128 = BigInteger.Parse(value.Replace(".", ""), CultureInfo.InvariantCulture); + var bytes = raw128.ToByteArray(); + var raw = new byte[16]; + + if (raw128 < BigInteger.Zero && bytes.Length < raw.Length) + { + for (var i = bytes.Length; i < raw.Length; i++) + raw[i] = 0xFF; + } + + Array.Copy(bytes, raw, bytes.Length > 16 ? 16 : bytes.Length); + var low64 = BitConverter.ToUInt64(raw, 0); + var high64 = BitConverter.ToUInt64(raw, 8); +#endif + + return new DecimalValue(low64, high64, (uint)precision, (uint)scale); + } + + static readonly Lazy _lazy = new (() => new ()); + internal static YdbProviderAdapter Instance => _lazy.Value; + + #region IDynamicProviderAdapter + + public Type ConnectionType { get; } + public Type DataReaderType { get; } + public Type ParameterType { get; } + public Type CommandType { get; } + public Type TransactionType { get; } + + readonly Func _connectionFactory; + public DbConnection CreateConnection(string connectionString) => _connectionFactory(connectionString); + + #endregion + + public Func ClearAllPools { get; } + public Func ClearPool { get; } + + public Action SetDbType { get; } + public Func GetDbType { get; } + + // missing parameter value factories + public Func MakeYson { get; } + public object YsonNull { get; } + + internal Func MakeDecimalFromString { get; } + + internal Func, CancellationToken, IBulkUpsertImporter> BeginBulkCopy { get; } + + #region wrappers + [Wrapper] + internal sealed class YdbConnection + { + public YdbConnection(string connectionString) => throw new NotImplementedException(); + + public IBulkUpsertImporter BeginBulkUpsertImport(string name, IReadOnlyList columns, CancellationToken cancellationToken) => throw new NotImplementedException(); + + public static Task ClearAllPools() => throw new NotImplementedException(); + + public static Task ClearPool(YdbConnection connection) => throw new NotImplementedException(); + } + + [Wrapper] + internal sealed class IBulkUpsertImporter : TypeWrapper + { + [SuppressMessage("Style", "IDE0051:Remove unused private members", Justification = "Used from reflection")] + private static LambdaExpression[] Wrappers { get; } = +{ + // [0]: AddRowAsync + (Expression>)((this_, row) => this_.AddRowAsync(row)), + // [1]: FlushAsync + (Expression>)(this_ => this_.FlushAsync()), + }; + + public IBulkUpsertImporter(object instance, Delegate[] wrappers) : base(instance, wrappers) + { + } + + public ValueTask AddRowAsync(object?[] row) => ((Func)CompiledWrappers[0])(this, row); + + public ValueTask FlushAsync() => ((Func)CompiledWrappers[1])(this); + } + + [Wrapper] + internal sealed class YdbValue + { + public YdbValue(ProtoType type, ProtoValue value) => throw new NotImplementedException(); + + public static YdbValue MakeYson (byte[] value) => throw new NotImplementedException(); + public static YdbValue MakeOptionalYson(byte[]? value) => throw new NotImplementedException(); + } + + [Wrapper("Value")] + internal sealed class ProtoValue + { + public ProtoValue() => throw new NotImplementedException(); + + public ulong High128 + { + get => throw new NotImplementedException(); + set => throw new NotImplementedException(); + } + + public ulong Low128 + { + get => throw new NotImplementedException(); + set => throw new NotImplementedException(); + } + } + + [Wrapper("Type")] + internal sealed class ProtoType + { + public ProtoType() => throw new NotImplementedException(); + + public DecimalType DecimalType + { + get => throw new NotImplementedException(); + set => throw new NotImplementedException(); + } + } + + [Wrapper] + internal sealed class DecimalType + { + public DecimalType() => throw new NotImplementedException(); + + public uint Scale + { + get => throw new NotImplementedException(); + set => throw new NotImplementedException(); + } + + public uint Precision + { + get => throw new NotImplementedException(); + set => throw new NotImplementedException(); + } + } + + [Wrapper] + private sealed class YdbParameter + { + public YdbDbType YdbDbType { get; set; } + } + + [Wrapper] + public enum YdbDbType + { + Bool = 1, + Bytes = 13, + Date = 18, + DateTime = 19, + Decimal = 12, + Double = 11, + Float = 10, + Int16 = 3, + Int32 = 4, + Int64 = 5, + Int8 = 2, + Interval = 21, + Json = 15, + JsonDocument = 16, + Text = 14, + Timestamp = 20, + UInt16 = 7, + UInt32 = 8, + UInt64 = 9, + UInt8 = 6, + Unspecified = 0, + Uuid = 17, + + // missing simple types: + // DyNumber + // Yson + + // missing simple non-column types: + // TzDate + // TzDateTime + // TzTimestamp + } + + #endregion + } +} diff --git a/src/Linq2db.Ydb/src/Internal/YdbRetryPolicy.cs b/src/Linq2db.Ydb/src/Internal/YdbRetryPolicy.cs new file mode 100644 index 00000000..2d4b02d8 --- /dev/null +++ b/src/Linq2db.Ydb/src/Internal/YdbRetryPolicy.cs @@ -0,0 +1,144 @@ +using LinqToDB.Data.RetryPolicy; + +namespace LinqToDB.Internal.DataProvider.Ydb.Internal +{ + /// + /// YDB-specific retry policy implementation. + /// Implements official YDB retry strategies such as Full Jitter and Equal Jitter, + /// with different backoff timings depending on status codes. + /// Based on the retry logic from the YDB SDK (YdbRetryPolicy + YdbRetryPolicyConfig). + /// + public sealed class YdbRetryPolicy : RetryPolicyBase + { + // Default configuration matching YDB SDK: + // MaxAttempts=10; FastBase=5ms; SlowBase=50ms; + // FastCap=500ms; SlowCap=5000ms; EnableRetryIdempotence=false + private readonly int _maxAttempts; + private readonly int _fastBackoffBaseMs; + private readonly int _slowBackoffBaseMs; + private readonly int _fastCeiling; + private readonly int _slowCeiling; + private readonly int _fastCapBackoffMs; + private readonly int _slowCapBackoffMs; + private readonly bool _enableRetryIdempotence; + + /// + /// Default constructor — fully matches the default settings from YDB SDK. + /// + public YdbRetryPolicy() + : this( + maxAttempts: 10, + fastBackoffBaseMs: 5, + slowBackoffBaseMs: 50, + fastCapBackoffMs: 500, + slowCapBackoffMs: 5000, + enableRetryIdempotence: false) + { } + + /// + /// Extended constructor (can be used together with RetryPolicyOptions if desired). + /// Base class parameters are not critical since we completely override the delay calculation logic. + /// + public YdbRetryPolicy( + int maxAttempts, + int fastBackoffBaseMs, + int slowBackoffBaseMs, + int fastCapBackoffMs, + int slowCapBackoffMs, + bool enableRetryIdempotence) + : base( + maxRetryCount: Math.Max(0, maxAttempts - 1), // prevent base mechanics from interfering + maxRetryDelay: TimeSpan.FromMilliseconds(Math.Max(fastCapBackoffMs, slowCapBackoffMs)), + randomFactor: 1.1, + exponentialBase: 2.0, + coefficient: TimeSpan.FromMilliseconds(fastBackoffBaseMs)) + { + _maxAttempts = maxAttempts; + _fastBackoffBaseMs = fastBackoffBaseMs; + _slowBackoffBaseMs = slowBackoffBaseMs; + _fastCapBackoffMs = fastCapBackoffMs; + _slowCapBackoffMs = slowCapBackoffMs; + _fastCeiling = (int)Math.Ceiling(Math.Log(fastCapBackoffMs + 1, 2)); + _slowCeiling = (int)Math.Ceiling(Math.Log(slowCapBackoffMs + 1, 2)); + _enableRetryIdempotence = enableRetryIdempotence; + } + + /// + /// Determines if the given exception is retryable according to YDB rules. + /// + protected override bool ShouldRetryOn(Exception exception) + => YdbTransientExceptionDetector.ShouldRetryOn(exception, _enableRetryIdempotence); + + /// + /// Calculates the next retry delay based on the last exception and retry attempt count. + /// + protected override TimeSpan? GetNextDelay(Exception lastException) + { + // If it's not a YDB-specific exception — fallback to the base exponential retry logic + if (!YdbTransientExceptionDetector.TryGetYdbException(lastException, out var ydbEx)) + return base.GetNextDelay(lastException); + + var attempt = ExceptionsEncountered.Count - 1; + + // Lifetime of retry strategy: stop retrying after reaching the maximum number of attempts + if (attempt >= _maxAttempts - 1) + return null; + + // Respect the IsTransient flag if idempotence is disabled + _ = YdbTransientExceptionDetector.TryGetCodeAndTransient(ydbEx, out var code, out var isTransient); + if (!_enableRetryIdempotence && !isTransient) + return null; + + // Mapping of status codes to jitter type — same as in the YDB SDK: + // - BadSession/SessionBusy -> 0ms + // - Aborted/Undetermined -> FullJitter (fast) + // - Unavailable/ClientTransport* -> EqualJitter (fast) + // - Overloaded/ClientTransportRes* -> EqualJitter (slow) + return code switch + { + "BadSession" or "SessionBusy" + => TimeSpan.Zero, + + "Aborted" or "Undetermined" + => FullJitter(_fastBackoffBaseMs, _fastCapBackoffMs, _fastCeiling, attempt), + + "Unavailable" or "ClientTransportUnknown" or "ClientTransportUnavailable" + => EqualJitter(_fastBackoffBaseMs, _fastCapBackoffMs, _fastCeiling, attempt), + + "Overloaded" or "ClientTransportResourceExhausted" + => EqualJitter(_slowBackoffBaseMs, _slowCapBackoffMs, _slowCeiling, attempt), + + _ => null + }; + } + + // ===== Algorithms based on the official YDB SDK ===== + + /// + /// Full Jitter backoff calculation — completely random delay in the range [0..maxBackoff]. + /// + private TimeSpan FullJitter(int backoffBaseMs, int capMs, int ceiling, int attempt) + { + var maxMs = CalculateBackoff(backoffBaseMs, capMs, ceiling, attempt); + // Random.Next(max) — [0..max-1], in SDK they add +1 to avoid a strictly zero delay + return TimeSpan.FromMilliseconds(Random.Next(maxMs + 1)); + } + + /// + /// Equal Jitter backoff calculation — base delay + random jitter in [0..halfBackoff]. + /// + private TimeSpan EqualJitter(int backoffBaseMs, int capMs, int ceiling, int attempt) + { + var calc = CalculateBackoff(backoffBaseMs, capMs, ceiling, attempt); + var half = calc / 2; + // SDK: temp + calculatedBackoff % 2 + random.Next(temp + 1) + return TimeSpan.FromMilliseconds(half + calc % 2 + Random.Next(half + 1)); + } + + /// + /// Exponential backoff calculation with upper cap. + /// + private static int CalculateBackoff(int backoffBaseMs, int capMs, int ceiling, int attempt) + => Math.Min(backoffBaseMs * (1 << Math.Min(ceiling, attempt)), capMs); + } +} diff --git a/src/Linq2db.Ydb/src/Internal/YdbSchemaProvider.cs b/src/Linq2db.Ydb/src/Internal/YdbSchemaProvider.cs new file mode 100644 index 00000000..9b524f44 --- /dev/null +++ b/src/Linq2db.Ydb/src/Internal/YdbSchemaProvider.cs @@ -0,0 +1,513 @@ +using System.Data; +using System.Data.Common; +using System.Globalization; +using System.Text.RegularExpressions; +using LinqToDB.Data; +using LinqToDB.Internal.SchemaProvider; +using LinqToDB.SchemaProvider; + +namespace LinqToDB.Internal.DataProvider.Ydb.Internal +{ + /// Schema-provider YDB + public class YdbSchemaProvider : SchemaProviderBase + { + readonly HashSet _collections = new(StringComparer.OrdinalIgnoreCase); + Dictionary>? _pkMap; + + static DbConnection GetOpenConnection(DataConnection dc, out bool created) + { + var c = dc.TryGetDbConnection(); + if (c != null) + { + created = false; + if (c.State == ConnectionState.Closed) c.Open(); + return c; + } + + created = true; + return dc.OpenDbConnection(); + } + + static string Invariant(object? v) => + Convert.ToString(v, CultureInfo.InvariantCulture) ?? string.Empty; + + void LoadCollections(DbConnection c) + { + if (_collections.Count != 0) return; + using var t = c.GetSchema("MetaDataCollections"); + foreach (DataRow r in t.Rows) + _collections.Add(Invariant(r["CollectionName"])); + } + + bool Has(string name) => _collections.Contains(name); + + static string MakeTableId(string? schema, string name) => + string.IsNullOrEmpty(schema) + ? name + : FormattableString.Invariant($"{schema}.{name}"); + + static string ComposeColumnType(string baseType, int? len, int? prec, int? scale) + { + if (string.IsNullOrEmpty(baseType)) return string.Empty; + if (prec is not null) + return scale is not null + ? FormattableString.Invariant($"{baseType}({prec.Value},{scale.Value})") + : FormattableString.Invariant($"{baseType}({prec.Value})"); + return len is not null + ? FormattableString.Invariant($"{baseType}({len.Value})") + : baseType; + } + + static bool IsSchemaAllowed(GetSchemaOptions o, string? schema) + { + if (schema is null) return true; + + var exc = o.ExcludedSchemas; + if (exc != null && exc.Any(s => o.StringComparer.Equals(s, schema))) + return false; + + var inc = o.IncludedSchemas; + if (inc != null && inc.Length > 0 && !inc.Any(s => o.StringComparer.Equals(s, schema))) + return false; + + return true; + } + + static bool IsCatalogAllowed(GetSchemaOptions o, string? catalog) + { + if (catalog is null) return true; + + var exc = o.ExcludedCatalogs; + if (exc != null && exc.Any(c => o.StringComparer.Equals(c, catalog))) + return false; + + var inc = o.IncludedCatalogs; + if (inc != null && inc.Length > 0 && !inc.Any(c => o.StringComparer.Equals(c, catalog))) + return false; + + return true; + } + + static DataTable QueryToDataTable(DataConnection dc, string sql) + { + var conn = GetOpenConnection(dc, out var created); + try + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + using var reader = cmd.ExecuteReader(); + var table = new DataTable(); + table.Load(reader); + return table; + } + finally + { + if (created) conn.Close(); + } + } + + protected override string GetDataSourceName(DataConnection dbConnection) => dbConnection.DataProvider.Name; + protected override string GetDatabaseName(DataConnection dbConnection) => dbConnection.DataProvider.Name; + protected override string? GetProviderSpecificTypeNamespace() => null; + + protected override List GetTables(DataConnection dataConnection, GetSchemaOptions options) + { + + const string sql = @" + SELECT + NULL AS TABLE_CATALOG, + NULL AS TABLE_SCHEMA, + t.TABLE_NAME AS TABLE_NAME, + t.TABLE_TYPE AS TABLE_TYPE + FROM INFORMATION_SCHEMA.TABLES AS t + WHERE t.TABLE_TYPE IN ('BASE TABLE','TABLE') + "; + + try + { + using var dt = QueryToDataTable(dataConnection, sql); + var result = new List(); + + foreach (DataRow row in dt.Rows) + { + var name = Invariant(row["TABLE_NAME"]); + var schemaName = dt.Columns.Contains("TABLE_SCHEMA") ? Invariant(row["TABLE_SCHEMA"]) : null; + var catalog = dt.Columns.Contains("TABLE_CATALOG") ? Invariant(row["TABLE_CATALOG"]) : null; + + // delete .sys* + if (schemaName?.StartsWith(".sys", StringComparison.OrdinalIgnoreCase) == true) continue; + if (name.StartsWith(".sys", StringComparison.OrdinalIgnoreCase)) continue; + + if (!IsSchemaAllowed(options, schemaName) || !IsCatalogAllowed(options, catalog)) + continue; + + result.Add(new TableInfo + { + TableID = MakeTableId(schemaName, name), + CatalogName = catalog, + SchemaName = schemaName, + TableName = name, + IsView = false, + IsDefaultSchema = string.IsNullOrEmpty(schemaName) + || (!string.IsNullOrEmpty(options.DefaultSchema) && options.StringComparer.Equals(schemaName, options.DefaultSchema)), + IsProviderSpecific = false + }); + } + + return result; + } + catch + { + // If provider INFORMATION_SCHEMA is empty + var conn = GetOpenConnection(dataConnection, out var created); + try + { + LoadCollections(conn); + if (!Has("Tables")) + return new List(); + + using var schema = conn.GetSchema("Tables", new[] { null, "TABLE" }); + + var result = new List(); + foreach (DataRow row in schema.Rows) + { + string name = Invariant(row["TABLE_NAME"]); + string? schemaName = schema.Columns.Contains("TABLE_SCHEMA") ? Invariant(row["TABLE_SCHEMA"]) : null; + string? catalog = schema.Columns.Contains("TABLE_CATALOG") ? Invariant(row["TABLE_CATALOG"]) : null; + + if (schemaName?.StartsWith(".sys", StringComparison.OrdinalIgnoreCase) == true) continue; + if (name.StartsWith(".sys", StringComparison.OrdinalIgnoreCase)) continue; + + if (!IsSchemaAllowed(options, schemaName) || !IsCatalogAllowed(options, catalog)) + continue; + + result.Add(new TableInfo + { + TableID = MakeTableId(schemaName, name), + CatalogName = catalog, + SchemaName = schemaName, + TableName = name, + IsView = false, + IsDefaultSchema = string.IsNullOrEmpty(schemaName) + || (!string.IsNullOrEmpty(options.DefaultSchema) && options.StringComparer.Equals(schemaName, options.DefaultSchema)), + IsProviderSpecific = false + }); + } + + return result; + } + finally { if (created) conn.Close(); } + } + } + + protected override List GetColumns(DataConnection dataConnection, GetSchemaOptions options) + { + _pkMap = new Dictionary>(options.StringComparer); + + const string sql = @" + SELECT + NULL AS TABLE_CATALOG, + c.TABLE_SCHEMA AS TABLE_SCHEMA, + c.TABLE_NAME AS TABLE_NAME, + c.COLUMN_NAME AS COLUMN_NAME, + c.ORDINAL_POSITION AS ORDINAL_POSITION, + c.IS_NULLABLE AS IS_NULLABLE, + c.DATA_TYPE AS DATA_TYPE, + c.CHARACTER_MAXIMUM_LENGTH AS CHARACTER_MAXIMUM_LENGTH, + c.NUMERIC_PRECISION AS NUMERIC_PRECISION, + c.NUMERIC_SCALE AS NUMERIC_SCALE + FROM INFORMATION_SCHEMA.COLUMNS AS c + "; + + try + { + using var dt = QueryToDataTable(dataConnection, sql); + var result = new List(); + + foreach (DataRow row in dt.Rows) + { + var tableName = Invariant(row["TABLE_NAME"]); + var columnName = Invariant(row["COLUMN_NAME"]); + var schemaName = dt.Columns.Contains("TABLE_SCHEMA") ? Invariant(row["TABLE_SCHEMA"]) : null; + + if (!IsSchemaAllowed(options, schemaName)) + continue; + + var tableId = MakeTableId(schemaName, tableName); + var ordinal = dt.Columns.Contains("ORDINAL_POSITION") + ? Convert.ToInt32(row["ORDINAL_POSITION"], CultureInfo.InvariantCulture) + : 0; + + var isNullable = !dt.Columns.Contains("IS_NULLABLE") + || !Invariant(row["IS_NULLABLE"]).Equals("NO", StringComparison.OrdinalIgnoreCase); + + var dataTypeName = + dt.Columns.Contains("DATA_TYPE") ? Invariant(row["DATA_TYPE"]) : + dt.Columns.Contains("TYPE_NAME") ? Invariant(row["TYPE_NAME"]) : + dt.Columns.Contains("DATA_TYPE_NAME")? Invariant(row["DATA_TYPE_NAME"]) : + string.Empty; + + int? length = null; + if (dt.Columns.Contains("CHARACTER_MAXIMUM_LENGTH") + && int.TryParse(Invariant(row["CHARACTER_MAXIMUM_LENGTH"]), NumberStyles.Integer, CultureInfo.InvariantCulture, out var len)) + length = len; + + int? precision = null; + if (dt.Columns.Contains("NUMERIC_PRECISION") + && int.TryParse(Invariant(row["NUMERIC_PRECISION"]), NumberStyles.Integer, CultureInfo.InvariantCulture, out var prec)) + precision = prec; + else if (dt.Columns.Contains("COLUMN_SIZE") + && int.TryParse(Invariant(row["COLUMN_SIZE"]), NumberStyles.Integer, CultureInfo.InvariantCulture, out prec)) + precision = prec; + + int? scale = null; + if (dt.Columns.Contains("NUMERIC_SCALE") + && int.TryParse(Invariant(row["NUMERIC_SCALE"]), NumberStyles.Integer, CultureInfo.InvariantCulture, out var sc)) + scale = sc; + else if (dt.Columns.Contains("DECIMAL_DIGITS") + && int.TryParse(Invariant(row["DECIMAL_DIGITS"]), NumberStyles.Integer, CultureInfo.InvariantCulture, out sc)) + scale = sc; + + var columnType = ComposeColumnType(dataTypeName, length, precision, scale); + + var l2dbType = GetDataType(dataTypeName, columnType, length, precision, scale); + + result.Add(new ColumnInfo + { + TableID = tableId, + Name = columnName, + Ordinal = ordinal, + IsNullable = isNullable, + DataType = dataTypeName, + ColumnType = columnType, + Type = l2dbType, + Length = length, + Precision = precision, + Scale = scale, + IsIdentity = false + }); + + } + + return result; + } + catch + { + // Fallback to old GetSchema("Columns") + var conn = GetOpenConnection(dataConnection, out var created); + try + { + LoadCollections(conn); + if (!Has("Columns")) + return new(); + + using var schemaTable = conn.GetSchema("Columns"); + + var result = new List(); + foreach (DataRow row in schemaTable.Rows) + { + string tableName = Invariant(row["TABLE_NAME"]); + string columnName = Invariant(row["COLUMN_NAME"]); + string? schemaName = schemaTable.Columns.Contains("TABLE_SCHEMA") ? Invariant(row["TABLE_SCHEMA"]) : null; + + if (!IsSchemaAllowed(options, schemaName)) + continue; + + string tableId = MakeTableId(schemaName, tableName); + + int ordinal = schemaTable.Columns.Contains("ORDINAL_POSITION") + ? Convert.ToInt32(row["ORDINAL_POSITION"], CultureInfo.InvariantCulture) + : 0; + + bool isNullable = !schemaTable.Columns.Contains("IS_NULLABLE") + || !Invariant(row["IS_NULLABLE"]).Equals("NO", StringComparison.OrdinalIgnoreCase); + + string dataTypeName = string.Empty; + if (schemaTable.Columns.Contains("TYPE_NAME")) + dataTypeName = Invariant(row["TYPE_NAME"]); + else if (schemaTable.Columns.Contains("DATA_TYPE_NAME")) + dataTypeName = Invariant(row["DATA_TYPE_NAME"]); + if (string.IsNullOrWhiteSpace(dataTypeName)) + dataTypeName = schemaTable.Columns.Contains("DATA_TYPE") + ? Invariant(row["DATA_TYPE"]) + : string.Empty; + + int? length = null; + if (schemaTable.Columns.Contains("CHARACTER_MAXIMUM_LENGTH") + && int.TryParse(Invariant(row["CHARACTER_MAXIMUM_LENGTH"]), NumberStyles.Integer, CultureInfo.InvariantCulture, out var len)) + length = len; + + int? precision = null; + if (schemaTable.Columns.Contains("NUMERIC_PRECISION") + && int.TryParse(Invariant(row["NUMERIC_PRECISION"]), NumberStyles.Integer, CultureInfo.InvariantCulture, out var prec)) + precision = prec; + else if (schemaTable.Columns.Contains("COLUMN_SIZE") + && int.TryParse(Invariant(row["COLUMN_SIZE"]), NumberStyles.Integer, CultureInfo.InvariantCulture, out prec)) + precision = prec; + + int? scale = null; + if (schemaTable.Columns.Contains("NUMERIC_SCALE") + && int.TryParse(Invariant(row["NUMERIC_SCALE"]), NumberStyles.Integer, CultureInfo.InvariantCulture, out var sc)) + scale = sc; + else if (schemaTable.Columns.Contains("DECIMAL_DIGITS") + && int.TryParse(Invariant(row["DECIMAL_DIGITS"]), NumberStyles.Integer, CultureInfo.InvariantCulture, out sc)) + scale = sc; + + string columnType = ComposeColumnType(dataTypeName, length, precision, scale); + DataType linq2dbType = GetDataType(dataTypeName, columnType, length, precision, scale); + + result.Add(new ColumnInfo + { + TableID = tableId, + Name = columnName, + Ordinal = ordinal, + IsNullable = isNullable, + DataType = dataTypeName, + ColumnType = columnType, + Type = linq2dbType, + Length = length, + Precision = precision, + Scale = scale, + IsIdentity = false + }); + } + + return result; + } + finally { if (created) conn.Close(); } + } + } + + protected override IReadOnlyCollection GetPrimaryKeys( + DataConnection dataConnection, IEnumerable tables, GetSchemaOptions options) + { + if (_pkMap is null || _pkMap.Count == 0) return Array.Empty(); + + var res = new List(); + foreach (var t in tables) + { + if (t.ID == null) continue; + if (!_pkMap.TryGetValue(t.ID, out var cols)) continue; + + for (int i = 0; i < cols.Count; i++) + { + res.Add(new PrimaryKeyInfo + { + TableID = t.ID, + ColumnName = cols[i], + Ordinal = i, + PrimaryKeyName = FormattableString.Invariant($"{t.TableName}_pk") + }); + } + } + + return res; + } + + protected override IReadOnlyCollection GetForeignKeys( + DataConnection dataConnection, IEnumerable tables, GetSchemaOptions options) => + Array.Empty(); + + private static readonly Regex _decimalRegex = + new(@"^Decimal\(\d+,\s*\d+\)$", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + protected override DataType GetDataType( + string? dataType, // INFORMATION_SCHEMA.COLUMNS.DATA_TYPE or TYPE_NAME + string? columnType, + int? length, + int? precision, + int? scale) + { + var baseType = dataType?.Trim() ?? string.Empty; + int paren = baseType.IndexOf('('); + if (paren > 0) + baseType = baseType.Substring(0, paren).Trim(); + + columnType = columnType?.Trim() ?? baseType; + + switch (baseType) + { + case "Bool": return DataType.Boolean; + case "Int8": return DataType.SByte; + case "Uint8": return DataType.Byte; + case "Int16": return DataType.Int16; + case "Uint16": return DataType.UInt16; + case "Int32": return DataType.Int32; + case "Uint32": return DataType.UInt32; + case "Int64": return DataType.Int64; + case "Uint64": return DataType.UInt64; + case "Float": return DataType.Single; + case "Double": return DataType.Double; + + case "String": + case "StringData": return DataType.VarBinary; + + case "Utf8": + case "Text": return DataType.NVarChar; + + case "Date" : return DataType.Date; + case "Date32" : return DataType.Date; + case "Datetime" : return DataType.DateTime; + case "Datetime64" : return DataType.DateTime2; + case "Timestamp" : return DataType.DateTime2; + case "Timestamp64": return DataType.DateTime2; + case "Interval" : return DataType.Interval; + case "Interval64" : return DataType.Interval; + + case "Json": return DataType.Json; + case "Uuid": return DataType.Guid; + case "DyNumber": return DataType.VarChar; + + case "Decimal": + case "Numeric": return DataType.Decimal; + + case "Unspecified": + if (_decimalRegex.IsMatch(columnType)) + return DataType.Decimal; + if (columnType.Equals("Json", StringComparison.OrdinalIgnoreCase)) + return DataType.Json; + if (columnType.Equals("Uuid", StringComparison.OrdinalIgnoreCase)) + return DataType.Guid; + if (columnType.Equals("DyNumber", StringComparison.OrdinalIgnoreCase)) + return DataType.VarChar; + return DataType.Undefined; + + default: + return DataType.Undefined; + } + } + + protected override List GetDataTypes(DataConnection dataConnection) => _dataTypes; + + static readonly List _dataTypes = + [ + new() { TypeName = "Bool", DataType = typeof(bool). AssemblyQualifiedName! }, + new() { TypeName = "Int8", DataType = typeof(sbyte). AssemblyQualifiedName! }, + new() { TypeName = "Uint8", DataType = typeof(byte). AssemblyQualifiedName! }, + new() { TypeName = "Int16", DataType = typeof(short). AssemblyQualifiedName! }, + new() { TypeName = "Uint16", DataType = typeof(ushort). AssemblyQualifiedName! }, + new() { TypeName = "Int32", DataType = typeof(int). AssemblyQualifiedName! }, + new() { TypeName = "Uint32", DataType = typeof(uint). AssemblyQualifiedName! }, + new() { TypeName = "Int64", DataType = typeof(long). AssemblyQualifiedName! }, + new() { TypeName = "Uint64", DataType = typeof(ulong). AssemblyQualifiedName! }, + new() { TypeName = "Float", DataType = typeof(float). AssemblyQualifiedName! }, + new() { TypeName = "Double", DataType = typeof(double). AssemblyQualifiedName! }, + new() { TypeName = "Decimal", DataType = typeof(decimal).AssemblyQualifiedName! }, + new() { TypeName = "DyNumber", DataType = typeof(string). AssemblyQualifiedName! }, + new() { TypeName = "String", DataType = typeof(byte[]). AssemblyQualifiedName! }, + new() { TypeName = "Utf8", DataType = typeof(string). AssemblyQualifiedName! }, + new() { TypeName = "Json", DataType = typeof(string). AssemblyQualifiedName! }, + new() { TypeName = "JsonDocument", DataType = typeof(byte[]). AssemblyQualifiedName! }, + new() { TypeName = "Yson", DataType = typeof(byte[]). AssemblyQualifiedName! }, + new() { TypeName = "Uuid", DataType = typeof(Guid). AssemblyQualifiedName! }, + new() { TypeName = "Date", DataType = typeof(DateTime).AssemblyQualifiedName! }, + new() { TypeName = "Date32", DataType = typeof(DateTime).AssemblyQualifiedName! }, + new() { TypeName = "Datetime", DataType = typeof(DateTime).AssemblyQualifiedName! }, + new() { TypeName = "Datetime64", DataType = typeof(DateTime).AssemblyQualifiedName! }, + new() { TypeName = "Timestamp", DataType = typeof(DateTime).AssemblyQualifiedName! }, + new() { TypeName = "Timestamp64", DataType = typeof(DateTime).AssemblyQualifiedName! }, + new() { TypeName = "Interval", DataType = typeof(TimeSpan).AssemblyQualifiedName! }, + new() { TypeName = "Interval64", DataType = typeof(TimeSpan).AssemblyQualifiedName! }, + ]; + } +} diff --git a/src/Linq2db.Ydb/src/Internal/YdbSpecificQueryable.cs b/src/Linq2db.Ydb/src/Internal/YdbSpecificQueryable.cs new file mode 100644 index 00000000..f78f74f7 --- /dev/null +++ b/src/Linq2db.Ydb/src/Internal/YdbSpecificQueryable.cs @@ -0,0 +1,12 @@ +using LinqToDB.DataProvider; +using LinqToDB.DataProvider.Ydb; + +namespace LinqToDB.Internal.DataProvider.Ydb.Internal +{ + sealed class YdbSpecificQueryable + : DatabaseSpecificQueryable, + IYdbSpecificQueryable + { + public YdbSpecificQueryable(IQueryable queryable) : base(queryable) { } + } +} diff --git a/src/Linq2db.Ydb/src/Internal/YdbSpecificTable.cs b/src/Linq2db.Ydb/src/Internal/YdbSpecificTable.cs new file mode 100644 index 00000000..67f7de2d --- /dev/null +++ b/src/Linq2db.Ydb/src/Internal/YdbSpecificTable.cs @@ -0,0 +1,13 @@ +using LinqToDB.DataProvider; +using LinqToDB.DataProvider.Ydb; + +namespace LinqToDB.Internal.DataProvider.Ydb.Internal +{ + sealed class YdbSpecificTable + : DatabaseSpecificTable, + IYdbSpecificTable + where TSource : notnull + { + public YdbSpecificTable(ITable table) : base(table) { } + } +} diff --git a/src/Linq2db.Ydb/src/Internal/YdbSqlBuilder.cs b/src/Linq2db.Ydb/src/Internal/YdbSqlBuilder.cs new file mode 100644 index 00000000..338aef5d --- /dev/null +++ b/src/Linq2db.Ydb/src/Internal/YdbSqlBuilder.cs @@ -0,0 +1,396 @@ +using System.Collections; +using System.Data.Common; +using System.Globalization; +using System.Text; +using LinqToDB.Common; +using LinqToDB.DataProvider; +using LinqToDB.DataProvider.Ydb; +using LinqToDB.Internal.SqlProvider; +using LinqToDB.Internal.SqlQuery; +using LinqToDB.Mapping; +using LinqToDB.SqlQuery; + +namespace LinqToDB.Internal.DataProvider.Ydb.Internal +{ + public class YdbSqlBuilder : BasicSqlBuilder + { + readonly YdbOptions _providerOptions; + + public YdbSqlBuilder(IDataProvider? provider, MappingSchema mappingSchema, DataOptions dataOptions, ISqlOptimizer sqlOptimizer, SqlProviderFlags sqlProviderFlags) + : base(provider, mappingSchema, dataOptions, sqlOptimizer, sqlProviderFlags) + { + _providerOptions = dataOptions.FindOrDefault(YdbOptions.Default); + } + + YdbSqlBuilder(BasicSqlBuilder parentBuilder, YdbOptions ydbOptions) : base(parentBuilder) + { + _providerOptions = ydbOptions; + } + + protected override ISqlBuilder CreateSqlBuilder() => new YdbSqlBuilder(this, _providerOptions); + + protected override string LimitFormat(SelectQuery selectQuery) => "LIMIT {0}"; + protected override string OffsetFormat(SelectQuery selectQuery) => "OFFSET {0} "; + + protected override void PrintParameterName(StringBuilder sb, DbParameter parameter) + { + sb.Append(parameter.ParameterName); + } + + protected override void BuildCreateTablePrimaryKey(SqlCreateTableStatement createTable, string pkName, IEnumerable fieldNames) + { + AppendIndent() + .Append("PRIMARY KEY (") + .Append(string.Join(InlineComma, fieldNames)) + .Append(')'); + } + + protected override void BuildDropTableStatement(SqlDropTableStatement dropTable) + => BuildDropTableStatementIfExists(dropTable); + + protected override void BuildCreateTableNullAttribute(SqlField field, DefaultNullable defaultNullable) + { + // columns are nullable by default + // primary key columns always non-nullable even with NULL specified + if (!field.CanBeNull && !field.IsPrimaryKey) + StringBuilder.Append("NOT NULL"); + } + + protected override void BuildGetIdentity(SqlInsertClause insertClause) + { + var id = insertClause.Into?.GetIdentityField() + ?? throw new LinqToDBException($"Identity field must be defined for '{insertClause.Into?.NameForLogging}'."); + + AppendIndent().AppendLine("RETURNING"); + AppendIndent().Append('\t'); + BuildExpression(id, false, true); + StringBuilder.AppendLine(); + } + + protected override void BuildCreateTableCommand(SqlTable table) + { + string command; + + if (table.TableOptions.IsTemporaryOptionSet()) + { + switch (table.TableOptions & TableOptions.IsTemporaryOptionSet) + { + case TableOptions.IsTemporary: + case TableOptions.IsTemporary | TableOptions.IsLocalTemporaryData: + case TableOptions.IsTemporary | TableOptions.IsLocalTemporaryStructure: + case TableOptions.IsTemporary | TableOptions.IsLocalTemporaryStructure | TableOptions.IsLocalTemporaryData: + case TableOptions.IsLocalTemporaryData: + case TableOptions.IsLocalTemporaryStructure: + case TableOptions.IsLocalTemporaryStructure | TableOptions.IsLocalTemporaryData: + command = "CREATE TEMPORARY TABLE "; + break; + case var value: + throw new LinqToDBException($"Incompatible table options '{value}'"); + } + } + else + { + command = "CREATE TABLE "; + } + + StringBuilder.Append(command); + } + + // duplicate aliases in final select are not supported + protected override bool CanSkipRootAliases(SqlStatement statement) => false; + + protected override void BuildCreateTableFieldType(SqlField field) + { + if (field.IsIdentity) + { + StringBuilder.Append(field.Type.DataType switch + { + DataType.Int16 => "SMALLSERIAL", + DataType.Int32 => "SERIAL", + DataType.Int64 => "BIGSERIAL", + _ => throw new InvalidOperationException($"Unsupported identity field type {field.Type.DataType}") + }); + + return; + } + + base.BuildCreateTableFieldType(field); + } + + protected override void BuildDataTypeFromDataType(DbDataType type, bool forCreateTable, bool canBeNull) + { + switch (type.DataType) + { + case DataType.Boolean : StringBuilder.Append("Bool"); break; + case DataType.SByte : StringBuilder.Append("Int8"); break; + case DataType.Byte : StringBuilder.Append("Uint8"); break; + case DataType.Int16 : StringBuilder.Append("Int16"); break; + case DataType.UInt16 : StringBuilder.Append("Uint16"); break; + case DataType.Int32 : StringBuilder.Append("Int32"); break; + case DataType.UInt32 : StringBuilder.Append("Uint32"); break; + case DataType.Int64 : StringBuilder.Append("Int64"); break; + case DataType.UInt64 : StringBuilder.Append("Uint64"); break; + case DataType.Single : StringBuilder.Append("Float"); break; + case DataType.Double : StringBuilder.Append("Double"); break; + case DataType.DecFloat : StringBuilder.Append("DyNumber"); break; + case DataType.Binary + or DataType.Blob + or DataType.VarBinary: StringBuilder.Append("Bytes"); break; + case DataType.NChar + or DataType.Char + or DataType.NVarChar + or DataType.VarChar + : StringBuilder.Append("Text"); break; + case DataType.Json : StringBuilder.Append("Json"); break; + case DataType.BinaryJson : StringBuilder.Append("JsonDocument"); break; + case DataType.Guid : StringBuilder.Append("Uuid"); break; + case DataType.Date : StringBuilder.Append("Date"); break; + case DataType.DateTime : StringBuilder.Append("Datetime"); break; + case DataType.DateTime2 : StringBuilder.Append("Timestamp"); break; + case DataType.Interval : StringBuilder.Append("Interval"); break; + + case DataType.Decimal: + { + if (_providerOptions.UseParametrizedDecimal) + { + StringBuilder.AppendFormat( + CultureInfo.InvariantCulture, + "Decimal({0},{1})", + type.Precision ?? YdbMappingSchema.DEFAULT_DECIMAL_PRECISION, + type.Scale ?? YdbMappingSchema.DEFAULT_DECIMAL_SCALE); + } + else + { + StringBuilder.Append("Decimal"); + } + + break; + } + + default: + base.BuildDataTypeFromDataType(type, false, false); + break; + } + } + + protected sealed override bool IsReserved(string word) + { + return ReservedWords.IsReserved(word, "YDB"); + } + + public override StringBuilder Convert(StringBuilder sb, string value, ConvertType convertType) + { + // https://ydb.tech/docs/en/yql/reference/syntax/create_table/#object-naming-rules + // https://ydb.tech/docs/en/yql/reference/syntax/lexer#keywords-and-ids + // Documentation doesn't match to database behavior in some places: + // - keywords could be used as identifiers + // - . and - are not allowed + // - reserved words work strange - in some places you can use them as-is, in some - need quotation + switch (convertType) + { + case ConvertType.NameToQueryParameter: + { + return sb.Append('@').Append(value); + } + + + case ConvertType.NameToQueryField + or ConvertType.NameToQueryFieldAlias + or ConvertType.NameToQueryTable + or ConvertType.NameToQueryTableAlias: + { + // don't check for __ydb_ prefix as it is not allowed even in quoted mode + var quote = (value.Length > 0 && char.IsDigit(value[0])) + || value.Any(c => !c.IsAsciiLetterOrDigit() && c is not '_') + || IsReserved(value); + + if (quote) + return sb.Append('`').Append(value.Replace("`", "\\`")).Append('`'); + + return sb.Append(value); + } + + default: + return sb.Append(value); + } + } + + protected override void BuildOutputColumnExpressions(IReadOnlyList expressions) + { + Indent++; + + var first = true; + + // RETURNING supports only column names without table reference + // expressions also not supported, but it is user's fault + var oldValue = _buildTableName; + _buildTableName = false; + + foreach (var expr in expressions) + { + if (!first) + StringBuilder.AppendLine(Comma); + + first = false; + + var addAlias = true; + + AppendIndent(); + BuildColumnExpression(null, expr, null, ref addAlias); + } + + _buildTableName = oldValue; + + Indent--; + + StringBuilder.AppendLine(); + } + + private bool _buildTableName= true; + + protected override void BuildColumnExpression(SelectQuery? selectQuery, ISqlExpression expr, string? alias, ref bool addAlias) + { + BuildExpression(expr, _buildTableName, true, alias, ref addAlias, true); + } + + protected override bool IsCteColumnListSupported => false; + + protected override void BuildFromClause(SqlStatement statement, SelectQuery selectQuery) + { + if (!statement.IsUpdate()) + base.BuildFromClause(statement, selectQuery); + } + + protected override string? GetProviderTypeName(IDataContext dataContext, DbParameter parameter) + { + if (DataProvider is YdbDataProvider provider) + { + var param = provider.TryGetProviderParameter(dataContext, parameter); + if (param != null) + { + return provider.Adapter.GetDbType(param).ToString(); + } + } + + return base.GetProviderTypeName(dataContext, parameter); + } + + protected override void BuildSubQueryExtensions(SqlStatement statement) + { + var sqlExts = statement.SelectQuery?.SqlQueryExtensions; + if (sqlExts is null || sqlExts.Count == 0) + return; + + BuildQueryExtensions( + StringBuilder, + sqlExts, + prefix: null, // PragmaQueryHintBuilder will insert line breaks/PRAGMA + delimiter: "\n", + suffix: null, + Sql.QueryExtensionScope.SubQueryHint); + } + + protected override void BuildInListPredicate(SqlPredicate.InList predicate) + { + static List? TryMaterializeItems( + OptimizationContext opt, + IReadOnlyList values) + { + if (values is [SqlParameter pr]) + { + var pv = pr.GetParameterValue(opt.EvaluationContext.ParameterValues).ProviderValue; + switch (pv) + { + case string: + return null; + case IEnumerable en: + { + return en.Cast().ToList(); + } + } + } + + var tmp = new List(values.Count); + foreach (var v in values) + { + switch (v) + { + case SqlValue sv: + tmp.Add(sv.Value); + break; + case SqlParameter sp: + tmp.Add(sp.GetParameterValue(opt.EvaluationContext.ParameterValues).ProviderValue); + break; + default: + return null; + } + } + + return tmp; + } + + var items = TryMaterializeItems(OptimizationContext, predicate.Values); + if (items == null) + { + base.BuildInListPredicate(predicate); + return; + } + + var dbDataType = QueryHelper.GetDbDataType(predicate.Expr1, MappingSchema); + + var hasNull = false; + for (var i = items.Count - 1; i >= 0; i--) + { + if (items[i] != null) + { + continue; + } + + hasNull = true; + items.RemoveAt(i); + } + + if (items.Count == 0) + { + BuildPredicate(new SqlPredicate.IsNull(predicate.Expr1, predicate.IsNot)); + return; + } + + var max = SqlProviderFlags.MaxInListValuesCount; + var startLen = StringBuilder.Length; + var bucketIndex = 0; + for (var i = 0; i < items.Count; i += max, bucketIndex++) + { + if (i > 0) + StringBuilder.Append(predicate.IsNot ? " AND " : " OR "); + + BuildExpression(GetPrecedence(predicate), predicate.Expr1); + StringBuilder.Append(predicate.IsNot ? " NOT IN (" : " IN ("); + + var within = 1; + for (var j = i; j < Math.Min(i + max, items.Count); j++, within++) + { + var p = new SqlParameter(dbDataType, FormattableString.Invariant($"Ids{bucketIndex}_{within}"), items[j]); + BuildParameter(p); + StringBuilder.Append(InlineComma); + } + + RemoveInlineComma().Append(')'); + } + + // 'x IN (...) OR x IS NULL' + if (hasNull) + { + StringBuilder.Append(predicate.IsNot ? " AND " : " OR "); + BuildPredicate(new SqlPredicate.IsNull(predicate.Expr1, predicate.IsNot)); + } + + if (bucketIndex > 1 || hasNull) + { + StringBuilder.Insert(startLen, "(").Append(')'); + } + } + + protected override void BuildMergeStatement(SqlMergeStatement merge) => throw new LinqToDBException($"{Name} provider doesn't support SQL MERGE statement"); + } +} diff --git a/src/Linq2db.Ydb/src/Internal/YdbSqlExpressionConvertVisitor.cs b/src/Linq2db.Ydb/src/Internal/YdbSqlExpressionConvertVisitor.cs new file mode 100644 index 00000000..e9f44093 --- /dev/null +++ b/src/Linq2db.Ydb/src/Internal/YdbSqlExpressionConvertVisitor.cs @@ -0,0 +1,166 @@ +using LinqToDB.Common; +using LinqToDB.Internal.SqlProvider; +using LinqToDB.Internal.SqlQuery; +using LinqToDB.SqlQuery; + +namespace LinqToDB.Internal.DataProvider.Ydb.Internal +{ + public class YdbSqlExpressionConvertVisitor : SqlExpressionConvertVisitor + { + public YdbSqlExpressionConvertVisitor(bool allowModify) : base(allowModify) { } + + ///// + //protected override bool SupportsNullInColumn => false; + + #region (I)LIKE https://ydb.tech/docs/en/yql/reference/syntax/expressions#check-match + + protected static string[] YdbLikeCharactersToEscape = {"%", "_"}; + + public override string[] LikeCharactersToEscape => YdbLikeCharactersToEscape; + + // escape value literal should have String type + public ISqlExpression CreateLikeEscapeCharacter() => new SqlValue(new DbDataType(typeof(string), DataType.VarBinary), LikeEscapeCharacter); + + public override ISqlPredicate ConvertSearchStringPredicate(SqlPredicate.SearchString predicate) + { + var searchPredicate = ConvertSearchStringPredicateViaLike(predicate); + + // use ILIKE for case-insensitive search + if (false == predicate.CaseSensitive.EvaluateBoolExpression(EvaluationContext) && searchPredicate is SqlPredicate.Like likePredicate) + { + searchPredicate = new SqlPredicate.Like(likePredicate.Expr1, likePredicate.IsNot, likePredicate.Expr2, likePredicate.Escape, "ILIKE"); + } + + return searchPredicate; + } + + #endregion + + public override IQueryElement ConvertSqlBinaryExpression(SqlBinaryExpression element) + { + switch (element.Operation) + { + case "+" when element.Type.DataType == DataType.Undefined && element.SystemType == typeof(string): + case "+" when element.Type.DataType + is DataType.NVarChar + or DataType.NChar + or DataType.Char + or DataType.VarChar + or DataType.Binary + or DataType.VarBinary + or DataType.Blob: + return new SqlBinaryExpression(element.SystemType, element.Expr1, "||", element.Expr2, element.Precedence); + + //case "%": + //{ + // var dbType = QueryHelper.GetDbDataType(element.Expr1, MappingSchema); + + // if (dbType.SystemType.ToNullableUnderlying() != typeof(decimal)) + // { + // var toType = MappingSchema.GetDbDataType(typeof(decimal)); + // var newLeft = PseudoFunctions.MakeCast(element.Expr1, toType); + + // var sysType = dbType.SystemType?.IsNullable() == true + // ? typeof(decimal?) + // : typeof(decimal); + + // var newExpr = PseudoFunctions.MakeMandatoryCast( + // new SqlBinaryExpression(sysType, newLeft, "%", element.Expr2), + // toType); + + // return Visit(Optimize(newExpr)); + // } + + // break; + //} + } + + return base.ConvertSqlBinaryExpression(element); + } + + public override ISqlExpression ConvertSqlFunction(SqlFunction func) + { + switch (func.Name) + { + case PseudoFunctions.TO_LOWER: + return func.WithName("Unicode::ToLower"); + + case PseudoFunctions.TO_UPPER: + return func.WithName("Unicode::ToUpper"); + + ////---------------------------------------------------------------- + //// save cast + //case PseudoFunctions.TRY_CONVERT: + // // CAST(x AS ?) → null + // return new SqlExpression( + // func.Type, + // "CAST({0} AS {1}?)", + // Precedence.Primary, + // func.Parameters[2], // value + // func.Parameters[0]); // type + + //case PseudoFunctions.TRY_CONVERT_OR_DEFAULT: + // // COALESCE(CAST(x AS ?), default) + // return new SqlExpression( + // func.Type, + // "COALESCE(CAST({0} AS {1}?), {2})", + // Precedence.Primary, + // func.Parameters[2], // value + // func.Parameters[0], // type + // func.Parameters[3]) // default + // { + // CanBeNull = + // func.Parameters[2].CanBeNullable(NullabilityContext) || + // func.Parameters[3].CanBeNullable(NullabilityContext) + // }; + + ////---------------------------------------------------------------- + //// CharIndex (there is no POSITION analog in YDB; using FIND) + //case "CharIndex": + // switch (func.Parameters.Length) + // { + // // CharIndex(substr, str) + // case 2: + // return new SqlExpression( + // func.Type, + // "COALESCE(FIND({1}, {0}) + 1, 0)", + // Precedence.Primary, + // func.Parameters[0], // substring + // func.Parameters[1]); // source + + // // CharIndex(substr, str, start) + // case 3: + // return new SqlExpression( + // func.Type, + // "COALESCE(FIND(SUBSTRING({1}, {2} - 1), {0}) + {2}, 0)", + // Precedence.Primary, + // func.Parameters[0], // substring + // func.Parameters[1], // source + // func.Parameters[2]); // start + // } + + // break; + } + + return base.ConvertSqlFunction(func); + } + + //// ------------------------------------------------------------------ + //// CAST bool → CASE + //// ------------------------------------------------------------------ + //protected override ISqlExpression ConvertConversion(SqlCastExpression cast) + //{ + // if (cast.SystemType.ToUnderlying() == typeof(bool) && + // !(cast.IsMandatory && cast.Expression.SystemType?.ToNullableUnderlying() == typeof(bool)) && + // cast.Expression is not SqlSearchCondition and not SqlCaseExpression) + // { + // // YDB not supporting CAST(condition AS bool), + // // cast to CASE WHEN ... THEN TRUE ELSE FALSE END + // return ConvertBooleanToCase(cast.Expression, cast.ToType); + // } + + // cast = FloorBeforeConvert(cast); + // return base.ConvertConversion(cast); + //} + } +} diff --git a/src/Linq2db.Ydb/src/Internal/YdbSqlOptimizer.cs b/src/Linq2db.Ydb/src/Internal/YdbSqlOptimizer.cs new file mode 100644 index 00000000..49f03b0c --- /dev/null +++ b/src/Linq2db.Ydb/src/Internal/YdbSqlOptimizer.cs @@ -0,0 +1,234 @@ +using LinqToDB.Internal.SqlProvider; +using LinqToDB.Internal.SqlQuery; +using LinqToDB.Mapping; + +namespace LinqToDB.Internal.DataProvider.Ydb.Internal +{ + public class YdbSqlOptimizer : BasicSqlOptimizer + { + public YdbSqlOptimizer(SqlProviderFlags sqlProviderFlags) + : base(sqlProviderFlags) { } + + public override SqlExpressionConvertVisitor CreateConvertVisitor(bool allowModify) + => new YdbSqlExpressionConvertVisitor(allowModify); + + public override SqlStatement TransformStatement(SqlStatement statement, DataOptions dataOptions, MappingSchema mappingSchema) + { + statement = base.TransformStatement(statement, dataOptions, mappingSchema); + + switch (statement.QueryType) + { + case QueryType.Delete: + // disable table alias + statement = GetAlternativeDelete((SqlDeleteStatement)statement); + statement.SelectQuery!.From.Tables[0].Alias = "$"; + break; + case QueryType.Update: + // disable table alias + statement.SelectQuery!.From.Tables[0].Alias = "$"; + break; + case QueryType.Insert: + statement = CorrectUpdateStatement((SqlInsertStatement)statement); + break; + } + + return statement; + } + + private SqlStatement CorrectUpdateStatement(SqlInsertStatement statement) + { + if (statement.SelectQuery != null + && statement.SelectQuery.Select.Columns.Count == statement.Insert.Items.Count) + { + for (var i = 0; i < statement.Insert.Items.Count; i++) + { + statement.SelectQuery.Select.Columns[i].Alias = ((SqlField)statement.Insert.Items[i].Column).Name; + } + + statement.SelectQuery.DoNotSetAliases = true; + } + + return statement; + } + + public override SqlStatement Finalize(MappingSchema mappingSchema, SqlStatement statement, DataOptions dataOptions) + { + statement = base.Finalize(mappingSchema, statement, dataOptions); + + if (MoveScalarSubQueriesToCte(statement)) + FinalizeCteCompat(statement); + + return statement; + } + + // Todo remove and replace for FinalizeCte + private void FinalizeCteCompat(SqlStatement statement) + { + if (statement is not SqlStatementWithQueryBase withStmt) + return; + + // 1) Собираем зависимости CTE: для каждого встреченного SqlCteTable регистрируем его Cte и все его зависимости. + var deps = new Dictionary>(); + + void TouchCteRefs(IQueryElement root) + { + root.Visit>>(deps, (map, e) => + { + if (e is SqlCteTable cteRef) + RegisterDependencyCompat(cteRef.Cte, map); + }); + } + + if (withStmt is SqlMergeStatement merge) + { + TouchCteRefs(merge.Target); + TouchCteRefs(merge.Source); + } + else + { + TouchCteRefs(withStmt.SelectQuery); + } + + // Если CTE не используются — очищаем WITH. + if (deps.Count == 0) + { + withStmt.With = null; + return; + } + + // Если провайдер не поддерживает CTE — кидаем исключение. + if (!this.SqlProviderFlags.IsCommonTableExpressionsSupported) + throw new LinqToDBException("DataProvider do not supports Common Table Expressions."); + + // 2) Уточняем флаги/самозависимости. + foreach (var kv in deps) + { + // если у CTE нет зависимостей — точно не рекурсивная + if (kv.Value.Count == 0) + kv.Key.IsRecursive = false; + + // удаляем самоссылки + kv.Value.Remove(kv.Key); + } + + // 3) Топологическая сортировка CTE по зависимостям. + var sorted = TopoSortCompat(deps); + + // 4) Делаем имена уникальными и задаём пустые по необходимости. + var used = new HashSet(StringComparer.OrdinalIgnoreCase); + var seq = 1; + foreach (var cte in sorted) + { + if (string.IsNullOrEmpty(cte.Name) || used.Contains(cte.Name)) + { + string newName; + do newName = $"CTE_{seq++}"; + while (used.Contains(newName)); + cte.Name = newName; + } + + used.Add(cte.Name); + } + + // 5) Записываем WITH + withStmt.With = new SqlWithClause(); + withStmt.With.Clauses.AddRange(sorted); + } + +// Рекурсивно регистрирует зависимости для одного CTE + private static void RegisterDependencyCompat(CteClause cte, Dictionary> map) + { + if (map.ContainsKey(cte)) + return; + + var dependsOn = new HashSet(); + + cte.Body.Visit>(dependsOn, (set, e) => + { + if (e is SqlCteTable refTable) + set.Add(refTable.Cte); + }); + + map.Add(cte, dependsOn); + + foreach (var d in dependsOn) + RegisterDependencyCompat(d, map); + } + +// Простой DFS-топосорт без внешних утилит + private static List TopoSortCompat(Dictionary> graph) + { + var result = new List(graph.Count); + var state = new Dictionary(graph.Count); // 0=unvisited,1=visiting,2=visited + + void Dfs(CteClause node) + { + if (state.TryGetValue(node, out var s)) + { + if (s == 2) return; + if (s == 1) return; // цикл: оставим как есть, чтобы не зациклиться + } + + state[node] = 1; + if (graph.TryGetValue(node, out var children)) + { + foreach (var child in children) + Dfs(child); + } + + state[node] = 2; + result.Add(node); + } + + foreach (var n in graph.Keys) + Dfs(n); + + // порядок обратный завершению — уже подходит для WITH (зависимости идут раньше использующих) + return result; + } + + private bool MoveScalarSubQueriesToCte(SqlStatement statement) + { + if (statement is not SqlStatementWithQueryBase withStatement) + return false; + + var cteCount = withStatement.With?.Clauses.Count ?? 0; + + if (statement.SelectQuery != null && statement.QueryType != QueryType.Merge) + statement.SelectQuery = ConvertToCte(statement.SelectQuery, withStatement); + + if (statement is SqlInsertStatement insert) + insert.Insert = ConvertToCte(insert.Insert, withStatement); + + return withStatement.With?.Clauses.Count > cteCount; + + static T ConvertToCte(T statement, SqlStatementWithQueryBase withStatement) + where T: class, IQueryElement + { + return statement.Convert(withStatement, static (visitor, elem) => + { + if (elem is SelectQuery { Select.Columns: [var column] } subQuery + && !QueryHelper.IsDependsOnOuterSources(subQuery)) + { + if (column.SystemType == null) + throw new InvalidOperationException(); + + if (visitor.Stack?.Count > 1 + // in column or predicate + && visitor.Stack[^2] is SqlSelectClause + or ISqlPredicate + or SqlExpressionBase + or SqlSetExpression) + { + var cte = new CteClause(subQuery, column.SystemType, false, null); + (visitor.Context.With ??= new()).Clauses.Add(cte); + return new SqlCteTable(cte, column.SystemType); + } + } + + return elem; + }, true); + } + } + } +} diff --git a/src/Linq2db.Ydb/src/Internal/YdbTransientExceptionDetector.cs b/src/Linq2db.Ydb/src/Internal/YdbTransientExceptionDetector.cs new file mode 100644 index 00000000..5e67fbf0 --- /dev/null +++ b/src/Linq2db.Ydb/src/Internal/YdbTransientExceptionDetector.cs @@ -0,0 +1,116 @@ +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Reflection; + +namespace LinqToDB.Internal.DataProvider.Ydb.Internal +{ + /// + /// Detector for identifying whether an exception represents a transient YDB error, + /// implemented without a hard dependency on Ydb.Sdk. + /// + /// Works by inspecting two properties of YdbException: + /// + /// bool IsTransient — indicates if the error is transient (temporary). + /// enum Code — contains the YDB status code for the error. + /// + /// + /// + public static class YdbTransientExceptionDetector + { + private const string YdbExceptionFullName = "Ydb.Sdk.Ado.YdbException"; + + /// + /// Checks whether there is a YDB exception either at the top level + /// or nested inside the provided exception hierarchy. + /// + /// The exception to search. + /// + /// When this method returns true, contains the first discovered YDB exception. + /// Otherwise, null. + /// + /// + /// true if a YdbException was found, otherwise false. + /// + public static bool TryGetYdbException(Exception ex, [NotNullWhen(true)] out Exception? ydbEx) + { + // YdbException is typically at the top level in the ADO client, + // but we traverse the inner exceptions just in case. + for (var e = ex; e != null; e = e.InnerException!) + { + if (e.GetType().FullName == YdbExceptionFullName) + { + ydbEx = e; + return true; + } + } + + ydbEx = null; + return false; + } + + /// + /// Reads the YDB Code property (status enum) as a string and also retrieves the IsTransient flag. + /// + /// The YDB exception instance to inspect. + /// Outputs the name of the status code as a string. + /// Outputs whether the error is transient. + /// + /// true if the Code property was successfully read, otherwise false. + /// + public static bool TryGetCodeAndTransient(Exception ydbEx, out string? codeName, out bool isTransient) + { + var t = ydbEx.GetType(); + + // bool IsTransient { get; } + var isTransientProp = t.GetProperty("IsTransient", BindingFlags.Public | BindingFlags.Instance); + isTransient = isTransientProp is not null && isTransientProp.GetValue(ydbEx) is bool b && b; + + // StatusCode Code { get; } + var codeProp = t.GetProperty("Code", BindingFlags.Public | BindingFlags.Instance); + var codeVal = codeProp?.GetValue(ydbEx); + codeName = Convert.ToString(codeVal, CultureInfo.InvariantCulture); + + return codeProp != null; + } + + /// + /// Determines whether the given exception should trigger a retry attempt + /// according to minimal YDB retry strategy rules. + /// + /// The logic closely follows the official YDB SDK: + /// it considers transient statuses and certain service-level timeouts. + /// + /// + /// The exception to evaluate. + /// + /// If true, adds additional YDB codes that should be retried based on SDK retry schemes. + /// If false, only the IsTransient flag is considered. + /// + /// + /// true if the operation should be retried; otherwise, false. + /// + public static bool ShouldRetryOn(Exception ex, bool enableRetryIdempotence) + { + if (TryGetYdbException(ex, out var ydbEx)) + { + _ = TryGetCodeAndTransient(ydbEx, out var code, out var isTransient); + + // When idempotence is disabled, rely only on IsTransient flag + if (!enableRetryIdempotence) + return isTransient; + + // When idempotence=true, include specific codes that the SDK retries + // using its own backoff strategy. + // (We use string names of enum members to avoid direct dependency on YDB SDK enums.) + return isTransient || code is + "BadSession" or "SessionBusy" or + "Aborted" or "Undetermined" or + "Unavailable" or "ClientTransportUnknown" or "ClientTransportUnavailable" or + "Overloaded" or "ClientTransportResourceExhausted"; + } + + // Also retry for common network or timeout-related exceptions. + return ex is TimeoutException; + } + } +} diff --git a/src/Linq2db.Ydb/src/Linq2db.csproj b/src/Linq2db.Ydb/src/Linq2db.csproj new file mode 100644 index 00000000..99c1884f --- /dev/null +++ b/src/Linq2db.Ydb/src/Linq2db.csproj @@ -0,0 +1,31 @@ + + + + net8.0 + enable + enable + LinqToDB.DataProvider.Ydb + LinqToDB.Internal.DataProvider.Ydb + latest + true + + + + + + + + + + + + + + + + + Never + + + + diff --git a/src/Linq2db.Ydb/src/YdbHints.cs b/src/Linq2db.Ydb/src/YdbHints.cs new file mode 100644 index 00000000..b50a3b0d --- /dev/null +++ b/src/Linq2db.Ydb/src/YdbHints.cs @@ -0,0 +1,112 @@ +using System; +using System.Linq; +using System.Linq.Expressions; +using System.Text; + +using JetBrains.Annotations; + +using LinqToDB.Internal.DataProvider.Ydb; +using LinqToDB.Internal.DataProvider.Ydb.Internal; +using LinqToDB.Internal.Linq; +using LinqToDB.Internal.SqlProvider; +using LinqToDB.Internal.SqlQuery; +using LinqToDB.Mapping; +using LinqToDB.SqlQuery; + +namespace LinqToDB.DataProvider.Ydb +{ + /// + /// SQL hints for YDB (YQL). See: + /// - SQL hints syntax: comments starting with "--+". + /// - SELECT-level hints: 'unique' / 'distinct' right after SELECT. + /// + /// + /// Emits YQL hint comment lines like: + /// --+ unique(col1 col2) + /// --+ distinct() + /// + public static partial class YdbHints + { + public const string Unique = "unique"; + public const string Distinct = "distinct"; + + [LinqTunnel, Pure, IsQueryable] + [Sql.QueryExtension("YDB", Sql.QueryExtensionScope.SubQueryHint, typeof(YdbQueryHintExtensionBuilder))] + [Sql.QueryExtension(null, Sql.QueryExtensionScope.None, typeof(NoneExtensionBuilder))] + public static IYdbSpecificQueryable QueryHint( + this IQueryable source, + [SqlQueryDependent] string hint, + [SqlQueryDependent] params string[] values) + where TSource : notnull + { + var current = source.ProcessIQueryable(); + + return new YdbSpecificQueryable(current.Provider.CreateQuery( + Expression.Call( + null, + MethodHelper.GetMethodInfo(QueryHint, source, hint, values), + current.Expression, + Expression.Constant(hint), + Expression.NewArrayInit(typeof(string), values.Select(Expression.Constant))))); + } + + /// + /// Generic query-hint injector for YDB/YQL. + /// + [LinqTunnel, Pure, IsQueryable] + [Sql.QueryExtension("YDB", Sql.QueryExtensionScope.SubQueryHint, typeof(YdbQueryHintExtensionBuilder))] + [Sql.QueryExtension(null, Sql.QueryExtensionScope.None, typeof(NoneExtensionBuilder))] + public static IYdbSpecificQueryable QueryHint( + this IYdbSpecificQueryable source, + [SqlQueryDependent] string hint, + [SqlQueryDependent] params string[] values) + where TSource : notnull + { + var current = source.ProcessIQueryable(); + + return new YdbSpecificQueryable(current.Provider.CreateQuery( + Expression.Call( + null, + MethodHelper.GetMethodInfo(QueryHint, source, hint, values), + current.Expression, + Expression.Constant(hint), + Expression.NewArrayInit(typeof(string), values.Select(Expression.Constant))))); + } + + /// + /// Builds a YQL SQL-hint comment line: + /// --+ hint(v1 v2 ...) + /// + sealed class YdbQueryHintExtensionBuilder : ISqlQueryExtensionBuilder + { + void ISqlQueryExtensionBuilder.Build(NullabilityContext nullability, ISqlBuilder sqlBuilder, StringBuilder stringBuilder, SqlQueryExtension sqlQueryExtension) + { + var hint = (string)((SqlValue)sqlQueryExtension.Arguments["hint"]).Value!; + + stringBuilder.Append("--+ ").Append(hint).Append('('); + + var count = (int)((SqlValue)sqlQueryExtension.Arguments["values.Count"]).Value!; + for (var i = 0; i < count; i++) + { + if (i > 0) stringBuilder.Append(' '); + + var raw = (string)((SqlValue)sqlQueryExtension.Arguments[FormattableString.Invariant($"values.{i}")]).Value!; + // quote value if it contains whitespace or parentheses or quote + var needQuote = raw.Any(ch => char.IsWhiteSpace(ch) || ch is '(' or ')' or '\''); + if (needQuote) + { + stringBuilder.Append('\'') + .Append(raw.Replace("'", "''")) + .Append('\''); + } + else + { + stringBuilder.Append(raw); + } + } + + stringBuilder.Append(')').AppendLine(); + } + } + } +} diff --git a/src/Linq2db.Ydb/src/YdbHints.generated.cs b/src/Linq2db.Ydb/src/YdbHints.generated.cs new file mode 100644 index 00000000..ee4eaa81 --- /dev/null +++ b/src/Linq2db.Ydb/src/YdbHints.generated.cs @@ -0,0 +1,77 @@ +#nullable enable +// Generated. +// +using System; +using System.Linq; +using System.Linq.Expressions; +using LinqToDB.Internal.DataProvider.Ydb; +using LinqToDB.Mapping; + +namespace LinqToDB.DataProvider.Ydb +{ + public static partial class YdbHints + { + // 1) IYdbSpecificQueryable + [ExpressionMethod(nameof(UniqueHintImpl))] + public static IYdbSpecificQueryable UniqueHint( + this IYdbSpecificQueryable query, + params string[] columns) + where TSource : notnull + { + return QueryHint(query, Unique, columns); + } + static Expression,string[],IYdbSpecificQueryable>> UniqueHintImpl() + where TSource : notnull + { + return (query, columns) => QueryHint(query, Unique, columns); + } + + // 2) IQueryable + [ExpressionMethod(nameof(UniqueHintQImpl))] + public static IYdbSpecificQueryable UniqueHint( + this IQueryable query, + params string[] columns) + where TSource : notnull + { + // QueryHint(IQueryable) + return QueryHint(query, Unique, columns); + } + static Expression,string[],IYdbSpecificQueryable>> UniqueHintQImpl() + where TSource : notnull + { + return (query, columns) => QueryHint(query, Unique, columns); + } + + // 1) IYdbSpecificQueryable + [ExpressionMethod(nameof(DistinctHintImpl))] + public static IYdbSpecificQueryable DistinctHint( + this IYdbSpecificQueryable query, + params string[] columns) + where TSource : notnull + { + return QueryHint(query, Distinct, columns); + } + static Expression,string[],IYdbSpecificQueryable>> DistinctHintImpl() + where TSource : notnull + { + return (query, columns) => QueryHint(query, Distinct, columns); + } + + // 2) IQueryable + [ExpressionMethod(nameof(DistinctHintQImpl))] + public static IYdbSpecificQueryable DistinctHint( + this IQueryable query, + params string[] columns) + where TSource : notnull + { + // QueryHint(IQueryable) + return QueryHint(query, Distinct, columns); + } + static Expression,string[],IYdbSpecificQueryable>> DistinctHintQImpl() + where TSource : notnull + { + return (query, columns) => QueryHint(query, Distinct, columns); + } + + } +} diff --git a/src/Linq2db.Ydb/src/YdbHints.tt b/src/Linq2db.Ydb/src/YdbHints.tt new file mode 100644 index 00000000..0b179e7b --- /dev/null +++ b/src/Linq2db.Ydb/src/YdbHints.tt @@ -0,0 +1,61 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ assembly name="System.Core" #> +<#@ import namespace="System.Linq" #> +<#@ import namespace="System.Text" #> +<#@ import namespace="System.Collections.Generic" #> +<#@ output extension=".generated.cs" #> +#nullable enable +// Generated. +// +using System; +using System.Linq; +using System.Linq.Expressions; + +using LinqToDB.Mapping; + +namespace LinqToDB.DataProvider.Ydb +{ + public static partial class YdbHints + { +<# + void Gen(string name) + { +#> + // 1) IYdbSpecificQueryable + [ExpressionMethod(nameof(<#= name #>HintImpl))] + public static IYdbSpecificQueryable <#= name #>Hint( + this IYdbSpecificQueryable query, + params string[] columns) + where TSource : notnull + { + return QueryHint(query, <#= name #>, columns); + } + static Expression,string[],IYdbSpecificQueryable>> <#= name #>HintImpl() + where TSource : notnull + { + return (query, columns) => QueryHint(query, <#= name #>, columns); + } + + // 2) IQueryable + [ExpressionMethod(nameof(<#= name #>HintQImpl))] + public static IYdbSpecificQueryable <#= name #>Hint( + this IQueryable query, + params string[] columns) + where TSource : notnull + { + // QueryHint(IQueryable) + return QueryHint(query, <#= name #>, columns); + } + static Expression,string[],IYdbSpecificQueryable>> <#= name #>HintQImpl() + where TSource : notnull + { + return (query, columns) => QueryHint(query, <#= name #>, columns); + } + +<# + } + Gen("Unique"); + Gen("Distinct"); +#> + } +} diff --git a/src/Linq2db.Ydb/src/YdbOptions.cs b/src/Linq2db.Ydb/src/YdbOptions.cs new file mode 100644 index 00000000..18435fd0 --- /dev/null +++ b/src/Linq2db.Ydb/src/YdbOptions.cs @@ -0,0 +1,55 @@ +using LinqToDB.Common; +using LinqToDB.Data; +using LinqToDB.DataProvider; +using LinqToDB.Internal.Common; +using LinqToDB.Internal.Options; + +namespace LinqToDB.Internal.DataProvider.Ydb +{ + /// + /// YDB data provider configuration options. + /// + /// + /// Default bulk copy mode for YDB. + /// Default value: . + /// + /// + /// Use Decimal(p, s) type name instead of Decimal. + /// Default value: true. + /// + public sealed record YdbOptions( + BulkCopyType BulkCopyType = BulkCopyType.ProviderSpecific, + bool UseParametrizedDecimal = true + ) : DataProviderOptions(BulkCopyType) + { + public YdbOptions() : this(BulkCopyType.ProviderSpecific) + { + } + + private YdbOptions(YdbOptions original) : base(original) + { + UseParametrizedDecimal = original.UseParametrizedDecimal; + } + + protected override IdentifierBuilder CreateID(IdentifierBuilder builder) => builder + .Add(UseParametrizedDecimal) + ; + + #region IEquatable implementation + + public bool Equals(YdbOptions? other) + { + if (ReferenceEquals(null, other)) return false; + if (ReferenceEquals(this, other)) return true; + + return ((IOptionSet)this).ConfigurationID == ((IOptionSet)other).ConfigurationID; + } + + public override int GetHashCode() + { + return ((IOptionSet)this).ConfigurationID; + } + + #endregion + } +} diff --git a/src/Linq2db.Ydb/src/YdbSpecificExtensions.cs b/src/Linq2db.Ydb/src/YdbSpecificExtensions.cs new file mode 100644 index 00000000..0d039ac7 --- /dev/null +++ b/src/Linq2db.Ydb/src/YdbSpecificExtensions.cs @@ -0,0 +1,44 @@ +using System.Linq.Expressions; +using JetBrains.Annotations; +using LinqToDB.Expressions; +using LinqToDB.Internal.DataProvider.Ydb.Internal; +using LinqToDB.Internal.Linq; +using LinqToDB.Linq; +using LinqToDB.Mapping; +using LinqToDB.SqlQuery; + +namespace LinqToDB.Internal.DataProvider.Ydb +{ + public static class YdbSpecificExtensions + { + [LinqTunnel, Pure, IsQueryable] + [Sql.QueryExtension(null, Sql.QueryExtensionScope.None, typeof(NoneExtensionBuilder))] + public static IYdbSpecificTable AsYdb(this ITable table) + where TSource : notnull + { + var wrapped = new Table( + table.DataContext, + Expression.Call( + null, + MethodHelper.GetMethodInfo(AsYdb, table), + table.Expression)); + + return new YdbSpecificTable(wrapped); + } + + [LinqTunnel, Pure, IsQueryable] + [Sql.QueryExtension(null, Sql.QueryExtensionScope.None, typeof(NoneExtensionBuilder))] + public static IYdbSpecificQueryable AsYdb(this IQueryable source) + where TSource : notnull + { + var normal = source.ProcessIQueryable(); + + return new YdbSpecificQueryable( + normal.Provider.CreateQuery( + Expression.Call( + null, + MethodHelper.GetMethodInfo(AsYdb, source), + normal.Expression))); + } + } +} diff --git a/src/Linq2db.Ydb/src/YdbTools.Registration.cs b/src/Linq2db.Ydb/src/YdbTools.Registration.cs new file mode 100644 index 00000000..6d3054de --- /dev/null +++ b/src/Linq2db.Ydb/src/YdbTools.Registration.cs @@ -0,0 +1,15 @@ +using System.Runtime.CompilerServices; +using LinqToDB.Data; + +namespace LinqToDB.Internal.DataProvider.Ydb +{ + public static partial class YdbTools // сделайте YdbTools partial, либо поместите код в сам файл YdbTools.cs + { + [ModuleInitializer] + public static void Register() + { + // ЭТО главный хук: связываем имя "YDB" и ваш провайдер + DataConnection.AddProviderDetector(ProviderDetector); + } + } +} diff --git a/src/Linq2db.Ydb/src/YdbTools.cs b/src/Linq2db.Ydb/src/YdbTools.cs new file mode 100644 index 00000000..11ef1d6a --- /dev/null +++ b/src/Linq2db.Ydb/src/YdbTools.cs @@ -0,0 +1,76 @@ +using System.Data.Common; +using System.Reflection; +using JetBrains.Annotations; +using LinqToDB.Data; +using LinqToDB.DataProvider; +using LinqToDB.Internal.DataProvider.Ydb.Internal; + +namespace LinqToDB.Internal.DataProvider.Ydb +{ + /// + /// Utility methods for working with Linq To DB and YDB, + /// similar to PostgreSQLTools for PostgreSQL. + /// + [PublicAPI] + public static partial class YdbTools + { + enum Fake { }; + + static readonly Lazy _ydbDataProvider = ProviderDetectorBase.CreateDataProvider(); + + public static IDataProvider? ProviderDetector(ConnectionOptions options) + { + static bool HasYdb(string? s) => + s?.IndexOf("Ydb", StringComparison.OrdinalIgnoreCase) >= 0; + + if (HasYdb(options.ProviderName) || HasYdb(options.ConfigurationString)) + return _ydbDataProvider.Value; + + return null; + } + + public static IDataProvider GetDataProvider() => _ydbDataProvider.Value; + + #region CreateDataConnection + + public static DataConnection CreateDataConnection(string connectionString) + { + return new DataConnection(new DataOptions() + .UseConnectionString(_ydbDataProvider.Value, connectionString)); + } + + public static DataConnection CreateDataConnection(DbConnection connection) + { + return new DataConnection(new DataOptions() + .UseConnection(_ydbDataProvider.Value, connection)); + } + + public static DataConnection CreateDataConnection(DbTransaction transaction) + { + return new DataConnection(new DataOptions() + .UseTransaction(_ydbDataProvider.Value, transaction)); + } + + #endregion + + public static void ResolveYdb(string path) + { + _ = new AssemblyResolver(path, YdbProviderAdapter.AssemblyName); + } + + public static void ResolveYdb(Assembly assembly) + { + _ = new AssemblyResolver(assembly, assembly.FullName!); + } + + /// + /// Clear all YDB client connection pools. + /// + public static Task ClearAllPools() => YdbProviderAdapter.Instance.ClearAllPools(); + + /// + /// Clear connection pool for connection's connection string. + /// + public static Task ClearPool(DbConnection connection) => YdbProviderAdapter.Instance.ClearPool(connection); + } +} diff --git a/src/Linq2db.Ydb/test/Linq2db.Tests/Linq2db.Tests.csproj b/src/Linq2db.Ydb/test/Linq2db.Tests/Linq2db.Tests.csproj new file mode 100644 index 00000000..598df6e3 --- /dev/null +++ b/src/Linq2db.Ydb/test/Linq2db.Tests/Linq2db.Tests.csproj @@ -0,0 +1,21 @@ + + + + net8.0 + false + enable + enable + + + + + + + + + + + + + + diff --git a/src/Linq2db.Ydb/test/Linq2db.Tests/YdbTests.cs b/src/Linq2db.Ydb/test/Linq2db.Tests/YdbTests.cs new file mode 100644 index 00000000..fe85361d --- /dev/null +++ b/src/Linq2db.Ydb/test/Linq2db.Tests/YdbTests.cs @@ -0,0 +1,286 @@ +#nullable enable + +using System.Diagnostics; +using LinqToDB; +using LinqToDB.Data; +using LinqToDB.Internal.DataProvider.Ydb; +using LinqToDB.Mapping; +using NUnit.Framework; + +namespace Linq2db.Tests +{ + [TestFixture] + public class YdbTests + + { + // --------- Маппинг под твою схему --------- + [Table("series")] + sealed class Series + { + [PrimaryKey, Column("series_id")] + public ulong SeriesId { get; set; } + + [Column("title"), NotNull] + public string Title { get; set; } = null!; + + [Column("series_info")] + public string? SeriesInfo { get; set; } + + [Column("release_date"), DataType(DataType.Date)] + public DateTime ReleaseDate { get; set; } + } + + [Table("seasons")] + sealed class Season + { + [PrimaryKey, Column("series_id")] + public ulong SeriesId { get; set; } + + [PrimaryKey, Column("season_id")] + public ulong SeasonId { get; set; } + + [Column("title"), NotNull] + public string Title { get; set; } = null!; + + [Column("first_aired"), DataType(DataType.Date)] + public DateTime FirstAired { get; set; } + + [Column("last_aired"), DataType(DataType.Date)] + public DateTime LastAired { get; set; } + } + + [Table("episodes")] + sealed class Episode + { + [PrimaryKey, Column("series_id")] + public ulong SeriesId { get; set; } + + [PrimaryKey, Column("season_id")] + public ulong SeasonId { get; set; } + + [PrimaryKey, Column("episode_id")] + public ulong EpisodeId { get; set; } + + [Column("title"), NotNull] + public string Title { get; set; } = null!; + + [Column("air_date"), DataType(DataType.Date)] + public DateTime AirDate { get; set; } + } + + // --------- константы / окружение --------- + const ulong SeriesIdFixed = 1; + const ulong SeasonIdFixed = 1; + + DataConnection _db = null!; + + static string BuildConnectionString() => "Host=localhost;Port=2136;Database=/local;UseTls=false;DisableDiscovery=true"; + + static int TryInt (string? s, int d) => int.TryParse(s, out var v) ? v : d; + static bool TryBool(string? s, bool d) => bool.TryParse(s, out var v) ? v : d; + + // --------- жизненный цикл тестов --------- + [SetUp] + public void SetUp() + { + + DataConnection.AddProviderDetector(YdbTools.ProviderDetector); + _db = new DataConnection("YDB", BuildConnectionString()); + DropSchema(_db); + CreateSchema(_db); + } + + + [TearDown] + public void TearDown() + { + try { DropSchema(_db); } catch { /* ignore */ } + _db.Dispose(); + } + + static void DropSchema(DataConnection db) + { + try { db.DropTable(); } catch { } + try { db.DropTable(); } catch { } + try { db.DropTable(); } catch { } + } + + static void CreateSchema(DataConnection db) + { + db.CreateTable(); + db.CreateTable(); + db.CreateTable(); + + db.Insert(new Series + { + SeriesId = SeriesIdFixed, + Title = "Demo Series", + SeriesInfo = "Synthetic dataset", + ReleaseDate = new DateTime(2010, 1, 1) + }); + + db.Insert(new Season + { + SeriesId = SeriesIdFixed, + SeasonId = SeasonIdFixed, + Title = "Season 1", + FirstAired = new DateTime(2010, 1, 1), + LastAired = new DateTime(2010, 12, 31) + }); + } + + // ===================== CRUD ===================== + + [Test] + public void CreateSchema_CreatesAllThreeTables() + { + // Просто проверим, что можно сделать COUNT по каждой таблице + Assert.DoesNotThrow(() => _db.GetTable().Count()); + Assert.DoesNotThrow(() => _db.GetTable().Count()); + Assert.DoesNotThrow(() => _db.GetTable().Count()); + } + + [Test] + public void Insert_Read_Update_Delete_SingleEpisode() + { + var episodes = _db.GetTable(); + + // CREATE + READ + var e1 = new Episode + { + SeriesId = SeriesIdFixed, + SeasonId = SeasonIdFixed, + EpisodeId = 1, + Title = "Pilot", + AirDate = new DateTime(2010, 1, 2) + }; + + Assert.DoesNotThrow(() => _db.Insert(e1)); + + var got = episodes.SingleOrDefault(e => + e.SeriesId == SeriesIdFixed && e.SeasonId == SeasonIdFixed && e.EpisodeId == 1); + + Assert.That(got, Is.Not.Null); + Assert.That(got!.Title, Is.EqualTo("Pilot")); + Assert.That(got.AirDate, Is.EqualTo(new DateTime(2010, 1, 2))); + + // UPDATE + var affected = episodes + .Where(e => e.SeriesId == SeriesIdFixed && e.SeasonId == SeasonIdFixed && e.EpisodeId == 1) + .Set(e => e.Title, _ => "Updated") + .Set(e => e.AirDate, _ => new DateTime(2010, 1, 3)) + .Update(); + + + var after = episodes.Single(e => + e.SeriesId == SeriesIdFixed && e.SeasonId == SeasonIdFixed && e.EpisodeId == 1); + + Assert.That(after.Title, Is.EqualTo("Updated")); + Assert.That(after.AirDate, Is.EqualTo(new DateTime(2010, 1, 3))); + + // DELETE + episodes.Delete(e => + e.SeriesId == SeriesIdFixed && e.SeasonId == SeasonIdFixed && e.EpisodeId == 1); + + Assert.That(episodes.Any(e => + e.SeriesId == SeriesIdFixed && e.SeasonId == SeasonIdFixed && e.EpisodeId == 1), Is.False); + } + + [Test] + public void InsertMany_UpdateAll_DeleteAll_SmallBatch() + { + // Чтобы быстро бегало в CI: маленькая партия + const int batch = 2000; + + var startDate = new DateTime(2010, 1, 1); + var data = Enumerable.Range(1, batch).Select(i => new Episode + { + SeriesId = SeriesIdFixed, + SeasonId = SeasonIdFixed, + EpisodeId = (ulong)i, + Title = $"Episode {i}", + AirDate = startDate.AddDays(i % 365), + }); + + var copied = _db.BulkCopy(data); + Assert.That(copied.RowsCopied, Is.EqualTo(batch)); + + var table = _db.GetTable(); + Assert.That(table.Count(e => e.SeriesId == SeriesIdFixed && e.SeasonId == SeasonIdFixed), Is.EqualTo(batch)); + + var newTitle = "updated"; + var newDate = DateTime.UtcNow.Date; + + table + .Where(e => e.SeriesId == SeriesIdFixed && e.SeasonId == SeasonIdFixed) + .Set(e => e.Title, _ => newTitle) + .Set(e => e.AirDate, _ => newDate) + .Update(); + + var mismatches = table.Count(e => + e.SeriesId == SeriesIdFixed && e.SeasonId == SeasonIdFixed && + (e.Title != newTitle || e.AirDate != newDate)); + + Assert.That(mismatches, Is.EqualTo(0)); + + table.Delete(e => e.SeriesId == SeriesIdFixed && e.SeasonId == SeasonIdFixed); + Assert.That(table.Count(e => e.SeriesId == SeriesIdFixed && e.SeasonId == SeasonIdFixed), Is.EqualTo(0)); + } + + // Тяжёлый e2e-тест на 15к — запускается только вручную + [Test] + public void Insert_Update_Delete_15000() + { + const int BatchSize = 15_000; + var episodes = new List(BatchSize); + var startDate = new DateTime(2010, 1, 1); + + for (int i = 1; i <= BatchSize; i++) + { + episodes.Add(new Episode + { + SeriesId = SeriesIdFixed, + SeasonId = SeasonIdFixed, + EpisodeId = (ulong)i, + Title = $"Episode {i}", + AirDate = startDate.AddDays(i % 365) + }); + } + + static T LogTime(string op, Func action) + { + var sw = Stopwatch.StartNew(); + try { return action(); } + finally { sw.Stop(); TestContext.Progress.WriteLine($"{op} | {sw.Elapsed}"); } + } + + _db.BulkCopy(episodes); + + var table = _db.GetTable(); + Assert.That( + LogTime("COUNT after insert", () => table.Count(e => e.SeriesId == SeriesIdFixed && e.SeasonId == SeasonIdFixed)), + Is.EqualTo(BatchSize)); + + var newTitle = "updated"; + var newDate = DateTime.UtcNow.Date; + + LogTime("UPDATE 15k", () => + table.Where(e => e.SeriesId == SeriesIdFixed && e.SeasonId == SeasonIdFixed) + .Set(e => e.Title, _ => newTitle) + .Set(e => e.AirDate, _ => newDate) + .Update()); + + + var mismatches = LogTime("Validate updates", () => + table.Count(e => e.SeriesId == SeriesIdFixed && e.SeasonId == SeasonIdFixed && + (e.Title != newTitle || e.AirDate != newDate))); + Assert.That(mismatches, Is.EqualTo(0)); + + var deleted = LogTime("DELETE 15k", () => + table.Delete(e => e.SeriesId == SeriesIdFixed && e.SeasonId == SeasonIdFixed)); + + TestContext.Progress.WriteLine($"Deleted reported: {deleted}"); + Assert.That(LogTime("Final COUNT(*)", () => table.Count()), Is.EqualTo(0)); + } + } +} diff --git a/src/YdbSdk.sln b/src/YdbSdk.sln index 834b85f2..738938b7 100644 --- a/src/YdbSdk.sln +++ b/src/YdbSdk.sln @@ -40,6 +40,16 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Ydb.Sdk.Ado.Benchmarks", "Y EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Ydb.Sdk.Ado.Stress.Loader", "Ydb.Sdk\test\Ydb.Sdk.Ado.Stress.Loader\Ydb.Sdk.Ado.Stress.Loader.csproj", "{22436AE0-178B-40B4-BD20-50F801C80B98}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Linq2db.Ydb", "Linq2db.Ydb", "{B1FBD80E-BD55-43C0-B8D0-8C7006A13408}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{8456AE49-967A-49D8-BC0A-03E7255D838C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Linq2db", "Linq2db.Ydb\src\Linq2db.csproj", "{CA3B48E6-3672-4A8B-A1EB-F58875753F57}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{B6E033D3-4852-48B9-8CB1-149B6AF5E603}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Linq2db.Tests", "Linq2db.Ydb\test\Linq2db.Tests\Linq2db.Tests.csproj", "{DBEB8B43-6298-43BC-9F51-FA4FDDA3FE98}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -82,6 +92,14 @@ Global {22436AE0-178B-40B4-BD20-50F801C80B98}.Debug|Any CPU.Build.0 = Debug|Any CPU {22436AE0-178B-40B4-BD20-50F801C80B98}.Release|Any CPU.ActiveCfg = Release|Any CPU {22436AE0-178B-40B4-BD20-50F801C80B98}.Release|Any CPU.Build.0 = Release|Any CPU + {CA3B48E6-3672-4A8B-A1EB-F58875753F57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CA3B48E6-3672-4A8B-A1EB-F58875753F57}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CA3B48E6-3672-4A8B-A1EB-F58875753F57}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CA3B48E6-3672-4A8B-A1EB-F58875753F57}.Release|Any CPU.Build.0 = Release|Any CPU + {DBEB8B43-6298-43BC-9F51-FA4FDDA3FE98}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DBEB8B43-6298-43BC-9F51-FA4FDDA3FE98}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DBEB8B43-6298-43BC-9F51-FA4FDDA3FE98}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DBEB8B43-6298-43BC-9F51-FA4FDDA3FE98}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -100,6 +118,10 @@ Global {AABFEDAE-1AEE-4ACF-804F-7F3C4D610CD8} = {316B82EF-019D-4267-95A9-5E243086B240} {3C20B44A-9649-4927-9D4A-8605CC1D1271} = {316B82EF-019D-4267-95A9-5E243086B240} {22436AE0-178B-40B4-BD20-50F801C80B98} = {316B82EF-019D-4267-95A9-5E243086B240} + {8456AE49-967A-49D8-BC0A-03E7255D838C} = {B1FBD80E-BD55-43C0-B8D0-8C7006A13408} + {CA3B48E6-3672-4A8B-A1EB-F58875753F57} = {8456AE49-967A-49D8-BC0A-03E7255D838C} + {B6E033D3-4852-48B9-8CB1-149B6AF5E603} = {B1FBD80E-BD55-43C0-B8D0-8C7006A13408} + {DBEB8B43-6298-43BC-9F51-FA4FDDA3FE98} = {B6E033D3-4852-48B9-8CB1-149B6AF5E603} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {0AB27123-0C66-4E43-A75F-D9EAB9ED0849} From 4ee4209f009ec483c2fdaf0ecb517c87575fa787 Mon Sep 17 00:00:00 2001 From: poma12390 Date: Thu, 30 Oct 2025 15:56:04 +0300 Subject: [PATCH 03/11] optimize import --- slo/src/Linq2db.Slo/SloLinq2DbContext.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/slo/src/Linq2db.Slo/SloLinq2DbContext.cs b/slo/src/Linq2db.Slo/SloLinq2DbContext.cs index 47d699d3..7b68df36 100644 --- a/slo/src/Linq2db.Slo/SloLinq2DbContext.cs +++ b/slo/src/Linq2db.Slo/SloLinq2DbContext.cs @@ -1,8 +1,5 @@ -using System; -using System.Threading.Tasks; -using Internal; -using LinqToDB; -using LinqToDB.Data; // <= вот это нужно для DataConnection +using Internal; +using LinqToDB.Data; namespace Linq2db; From f5edd5889d94299d26055ad3091a6b3053dafbfb Mon Sep 17 00:00:00 2001 From: poma12390 Date: Thu, 30 Oct 2025 23:10:42 +0300 Subject: [PATCH 04/11] using ydb.sdk retry --- slo/src/Linq2db.Slo/Program.cs | 5 +- slo/src/Linq2db.Slo/SloLinq2DbContext.cs | 89 ++++++---- .../src/Internal/YdbRetryPolicy.cs | 144 --------------- .../src/Internal/YdbSdkRetryPolicyAdapter.cs | 164 ++++++++++++++++++ 4 files changed, 225 insertions(+), 177 deletions(-) delete mode 100644 src/Linq2db.Ydb/src/Internal/YdbRetryPolicy.cs create mode 100644 src/Linq2db.Ydb/src/Internal/YdbSdkRetryPolicyAdapter.cs diff --git a/slo/src/Linq2db.Slo/Program.cs b/slo/src/Linq2db.Slo/Program.cs index 8407a665..01be97f8 100644 --- a/slo/src/Linq2db.Slo/Program.cs +++ b/slo/src/Linq2db.Slo/Program.cs @@ -1,3 +1,4 @@ -using Internal; +using Linq2db; +using Internal; -await Cli.Run(new Linq2db.SloTableContext(), args); \ No newline at end of file +await Cli.Run(new SloTableContext(), args); \ No newline at end of file diff --git a/slo/src/Linq2db.Slo/SloLinq2DbContext.cs b/slo/src/Linq2db.Slo/SloLinq2DbContext.cs index 7b68df36..ecdc1bc0 100644 --- a/slo/src/Linq2db.Slo/SloLinq2DbContext.cs +++ b/slo/src/Linq2db.Slo/SloLinq2DbContext.cs @@ -1,5 +1,9 @@ -using Internal; +using LinqToDB; using LinqToDB.Data; +using LinqToDB.Mapping; +using Internal; +using LinqToDB.Async; +using LinqToDB.Internal.DataProvider.Ydb.Internal; namespace Linq2db; @@ -7,6 +11,18 @@ public sealed class SloTableContext : SloTableContext "Linq2DB"; + // ВКЛЮЧАЕМ ретраи SDK глобально для всех DataConnection + static SloTableContext() + { + YdbSdkRetryPolicyRegistration.UseGloballyWithIdempotence( + maxAttempts: 10, + onRetry: (attempt, ex, delay) => + { + // здесь при желании снимать метрики: attempt, delay, ((YdbException)ex).Code + } + ); + } + public sealed class Linq2dbClient { private readonly string _connectionString; @@ -16,7 +32,8 @@ public Linq2dbClient(string connectionString) _connectionString = connectionString; } - public DataConnection Open() => new DataConnection("YDB", _connectionString); + public DataConnection Open() + => new DataConnection("YDB", _connectionString); } protected override Linq2dbClient CreateClient(Config config) @@ -27,54 +44,64 @@ protected override async Task Create(Linq2dbClient client, int operationTimeout) await using var db = client.Open(); db.CommandTimeout = operationTimeout; - // 1) CREATE TABLE - await db.ExecuteAsync($@" - CREATE TABLE `{SloTable.Name}` ( - Guid Uuid, - Id Int32, - PayloadStr Text, - PayloadDouble Double, - PayloadTimestamp Timestamp, - PRIMARY KEY (Guid, Id) - ); - "); - - await db.ExecuteAsync(SloTable.Options); + try + { + await db.ExecuteAsync($@" +CREATE TABLE `{SloTable.Name}` ( + Guid Uuid, + Id Int32, + PayloadStr Utf8, + PayloadDouble Double, + PayloadTimestamp Timestamp, + PRIMARY KEY (Guid, Id) +)"); + } + catch + { + // YDB не поддерживает IF NOT EXISTS; если таблица есть — окей + } + + if (!string.IsNullOrWhiteSpace(SloTable.Options)) + { + await db.ExecuteAsync(SloTable.Options); + } } - protected override async Task Save(Linq2dbClient client, SloTable row, int writeTimeout) + protected override async Task Save(Linq2dbClient client, SloTable sloTable, int writeTimeout) { await using var db = client.Open(); db.CommandTimeout = writeTimeout; await db.ExecuteAsync($@" - UPSERT INTO `{SloTable.Name}` (Guid, Id, PayloadStr, PayloadDouble, PayloadTimestamp) - VALUES ({row.Guid}, {row.Id}, {row.PayloadStr}, {row.PayloadDouble}, {row.PayloadTimestamp}); - "); +UPSERT INTO `{SloTable.Name}` (Guid, Id, PayloadStr, PayloadDouble, PayloadTimestamp) +VALUES ({sloTable.Guid}, {sloTable.Id}, {sloTable.PayloadStr}, {sloTable.PayloadDouble}, {sloTable.PayloadTimestamp}); +"); - return 1; + return 0; } - protected override async Task Select(Linq2dbClient client, (Guid Guid, int Id) key, int readTimeout) + protected override async Task Select(Linq2dbClient client, (Guid Guid, int Id) select, int readTimeout) { await using var db = client.Open(); db.CommandTimeout = readTimeout; - var exists = await db.ExecuteAsync($@" - SELECT COUNT(*) FROM `{SloTable.Name}` WHERE Guid = {key.Guid} AND Id = {key.Id}; - "); - - return exists > 0 ? 1 : null; + var t = db.GetTable(); + return await t.FirstOrDefaultAsync(r => r.Guid == select.Guid && r.Id == select.Id); } protected override async Task SelectCount(Linq2dbClient client) { await using var db = client.Open(); + return await db.GetTable().CountAsync(); + } - var maxId = await db.ExecuteAsync($@" - SELECT MAX(Id) FROM `{SloTable.Name}`; - "); - - return maxId ?? 0; + [Table(SloTable.Name)] + private sealed class SloRow + { + [Column] public Guid Guid { get; set; } + [Column] public int Id { get; set; } + [Column] public string? PayloadStr { get; set; } + [Column] public double PayloadDouble { get; set; } + [Column] public DateTime PayloadTimestamp { get; set; } } } diff --git a/src/Linq2db.Ydb/src/Internal/YdbRetryPolicy.cs b/src/Linq2db.Ydb/src/Internal/YdbRetryPolicy.cs deleted file mode 100644 index 2d4b02d8..00000000 --- a/src/Linq2db.Ydb/src/Internal/YdbRetryPolicy.cs +++ /dev/null @@ -1,144 +0,0 @@ -using LinqToDB.Data.RetryPolicy; - -namespace LinqToDB.Internal.DataProvider.Ydb.Internal -{ - /// - /// YDB-specific retry policy implementation. - /// Implements official YDB retry strategies such as Full Jitter and Equal Jitter, - /// with different backoff timings depending on status codes. - /// Based on the retry logic from the YDB SDK (YdbRetryPolicy + YdbRetryPolicyConfig). - /// - public sealed class YdbRetryPolicy : RetryPolicyBase - { - // Default configuration matching YDB SDK: - // MaxAttempts=10; FastBase=5ms; SlowBase=50ms; - // FastCap=500ms; SlowCap=5000ms; EnableRetryIdempotence=false - private readonly int _maxAttempts; - private readonly int _fastBackoffBaseMs; - private readonly int _slowBackoffBaseMs; - private readonly int _fastCeiling; - private readonly int _slowCeiling; - private readonly int _fastCapBackoffMs; - private readonly int _slowCapBackoffMs; - private readonly bool _enableRetryIdempotence; - - /// - /// Default constructor — fully matches the default settings from YDB SDK. - /// - public YdbRetryPolicy() - : this( - maxAttempts: 10, - fastBackoffBaseMs: 5, - slowBackoffBaseMs: 50, - fastCapBackoffMs: 500, - slowCapBackoffMs: 5000, - enableRetryIdempotence: false) - { } - - /// - /// Extended constructor (can be used together with RetryPolicyOptions if desired). - /// Base class parameters are not critical since we completely override the delay calculation logic. - /// - public YdbRetryPolicy( - int maxAttempts, - int fastBackoffBaseMs, - int slowBackoffBaseMs, - int fastCapBackoffMs, - int slowCapBackoffMs, - bool enableRetryIdempotence) - : base( - maxRetryCount: Math.Max(0, maxAttempts - 1), // prevent base mechanics from interfering - maxRetryDelay: TimeSpan.FromMilliseconds(Math.Max(fastCapBackoffMs, slowCapBackoffMs)), - randomFactor: 1.1, - exponentialBase: 2.0, - coefficient: TimeSpan.FromMilliseconds(fastBackoffBaseMs)) - { - _maxAttempts = maxAttempts; - _fastBackoffBaseMs = fastBackoffBaseMs; - _slowBackoffBaseMs = slowBackoffBaseMs; - _fastCapBackoffMs = fastCapBackoffMs; - _slowCapBackoffMs = slowCapBackoffMs; - _fastCeiling = (int)Math.Ceiling(Math.Log(fastCapBackoffMs + 1, 2)); - _slowCeiling = (int)Math.Ceiling(Math.Log(slowCapBackoffMs + 1, 2)); - _enableRetryIdempotence = enableRetryIdempotence; - } - - /// - /// Determines if the given exception is retryable according to YDB rules. - /// - protected override bool ShouldRetryOn(Exception exception) - => YdbTransientExceptionDetector.ShouldRetryOn(exception, _enableRetryIdempotence); - - /// - /// Calculates the next retry delay based on the last exception and retry attempt count. - /// - protected override TimeSpan? GetNextDelay(Exception lastException) - { - // If it's not a YDB-specific exception — fallback to the base exponential retry logic - if (!YdbTransientExceptionDetector.TryGetYdbException(lastException, out var ydbEx)) - return base.GetNextDelay(lastException); - - var attempt = ExceptionsEncountered.Count - 1; - - // Lifetime of retry strategy: stop retrying after reaching the maximum number of attempts - if (attempt >= _maxAttempts - 1) - return null; - - // Respect the IsTransient flag if idempotence is disabled - _ = YdbTransientExceptionDetector.TryGetCodeAndTransient(ydbEx, out var code, out var isTransient); - if (!_enableRetryIdempotence && !isTransient) - return null; - - // Mapping of status codes to jitter type — same as in the YDB SDK: - // - BadSession/SessionBusy -> 0ms - // - Aborted/Undetermined -> FullJitter (fast) - // - Unavailable/ClientTransport* -> EqualJitter (fast) - // - Overloaded/ClientTransportRes* -> EqualJitter (slow) - return code switch - { - "BadSession" or "SessionBusy" - => TimeSpan.Zero, - - "Aborted" or "Undetermined" - => FullJitter(_fastBackoffBaseMs, _fastCapBackoffMs, _fastCeiling, attempt), - - "Unavailable" or "ClientTransportUnknown" or "ClientTransportUnavailable" - => EqualJitter(_fastBackoffBaseMs, _fastCapBackoffMs, _fastCeiling, attempt), - - "Overloaded" or "ClientTransportResourceExhausted" - => EqualJitter(_slowBackoffBaseMs, _slowCapBackoffMs, _slowCeiling, attempt), - - _ => null - }; - } - - // ===== Algorithms based on the official YDB SDK ===== - - /// - /// Full Jitter backoff calculation — completely random delay in the range [0..maxBackoff]. - /// - private TimeSpan FullJitter(int backoffBaseMs, int capMs, int ceiling, int attempt) - { - var maxMs = CalculateBackoff(backoffBaseMs, capMs, ceiling, attempt); - // Random.Next(max) — [0..max-1], in SDK they add +1 to avoid a strictly zero delay - return TimeSpan.FromMilliseconds(Random.Next(maxMs + 1)); - } - - /// - /// Equal Jitter backoff calculation — base delay + random jitter in [0..halfBackoff]. - /// - private TimeSpan EqualJitter(int backoffBaseMs, int capMs, int ceiling, int attempt) - { - var calc = CalculateBackoff(backoffBaseMs, capMs, ceiling, attempt); - var half = calc / 2; - // SDK: temp + calculatedBackoff % 2 + random.Next(temp + 1) - return TimeSpan.FromMilliseconds(half + calc % 2 + Random.Next(half + 1)); - } - - /// - /// Exponential backoff calculation with upper cap. - /// - private static int CalculateBackoff(int backoffBaseMs, int capMs, int ceiling, int attempt) - => Math.Min(backoffBaseMs * (1 << Math.Min(ceiling, attempt)), capMs); - } -} diff --git a/src/Linq2db.Ydb/src/Internal/YdbSdkRetryPolicyAdapter.cs b/src/Linq2db.Ydb/src/Internal/YdbSdkRetryPolicyAdapter.cs new file mode 100644 index 00000000..2cdec88a --- /dev/null +++ b/src/Linq2db.Ydb/src/Internal/YdbSdkRetryPolicyAdapter.cs @@ -0,0 +1,164 @@ +// для DataConnection в Factory +using L2RetryOptions = LinqToDB.Data.RetryPolicy.RetryPolicyOptions; +using L2IRetryPolicy = LinqToDB.Data.RetryPolicy.IRetryPolicy; +using Ydb.Sdk.Ado; +using Ydb.Sdk.Ado.RetryPolicy; + +namespace LinqToDB.Internal.DataProvider.Ydb.Internal +{ + /// + /// Адаптер YDB SDK retry policy под интерфейс ретраев linq2db. + /// + public sealed class YdbSdkRetryPolicyAdapter : L2IRetryPolicy + { + private readonly YdbRetryPolicy _inner; + private readonly Action? _onRetry; + + public YdbSdkRetryPolicyAdapter( + YdbRetryPolicyConfig? config = null, + Action? onRetry = null) + { + _inner = new YdbRetryPolicy(config ?? YdbRetryPolicyConfig.Default); + _onRetry = onRetry; + } + + // ---------- Sync ---------- + public TResult Execute(Func operation) + { + if (operation == null) throw new ArgumentNullException(nameof(operation)); + + var attempt = 0; + while (true) + { + try + { + return operation(); + } + catch (Exception ex) when (TryGetDelay(ex, attempt, out var delay)) + { + _onRetry?.Invoke(attempt, ex, delay); + if (delay > TimeSpan.Zero) + Thread.Sleep(delay); + attempt++; + } + } + } + + public void Execute(Action operation) + { + Execute(() => + { + operation(); + return null; + }); + } + + // ---------- Async ---------- + public async Task ExecuteAsync( + Func> operation, + CancellationToken cancellationToken = default) + { + if (operation == null) throw new ArgumentNullException(nameof(operation)); + + var attempt = 0; + while (true) + { + try + { + return await operation(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (TryGetDelay(ex, attempt, out var delay)) + { + _onRetry?.Invoke(attempt, ex, delay); + if (delay > TimeSpan.Zero) + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + attempt++; + } + } + } + + public async Task ExecuteAsync( + Func operation, + CancellationToken cancellationToken = default) + { + await ExecuteAsync(async ct => + { + await operation(ct).ConfigureAwait(false); + return null; + }, cancellationToken).ConfigureAwait(false); + } + + // ---------- Helpers ---------- + private bool TryGetDelay(Exception ex, int attempt, out TimeSpan delay) + { + delay = default; + + if (!TryFindYdbException(ex, out var ydbEx)) + return false; + + var next = _inner.GetNextDelay(ydbEx, attempt); + if (next is null) + return false; + + delay = next.Value; + return true; + } + + private static bool TryFindYdbException(Exception ex, out YdbException ydbEx) + { + var cur = ex; + while (cur != null) + { + if (cur is YdbException ye) + { + ydbEx = ye; + return true; + } + cur = cur.InnerException!; + } + + ydbEx = null!; + return false; + } + } + + /// + /// Утилиты для подключения политики к linq2db. + /// + public static class YdbSdkRetryPolicyRegistration + { + /// + /// Подключить ретраи SDK глобально для всех новых DataConnection. + /// Вызывать один раз при старте/перед SLO-тестом. + /// + public static void UseGlobally(YdbRetryPolicyConfig? config = null, Action? onRetry = null) + { + L2RetryOptions.Default = L2RetryOptions.Default with + { + Factory = (_ /*DataConnection dc*/) => new YdbSdkRetryPolicyAdapter(config ?? YdbRetryPolicyConfig.Default, onRetry) + }; + } + + /// + /// Вариант с идемпотентностью (создаёт новый конфиг, т.к. YdbRetryPolicyConfig — не record). + /// + public static void UseGloballyWithIdempotence( + int? maxAttempts = null, + int? fastBaseMs = null, int? slowBaseMs = null, + int? fastCapMs = null, int? slowCapMs = null, + Action? onRetry = null) + { + var cfg = new YdbRetryPolicyConfig + { + EnableRetryIdempotence = true, + MaxAttempts = maxAttempts ?? YdbRetryPolicyConfig.Default.MaxAttempts, + FastBackoffBaseMs = fastBaseMs ?? YdbRetryPolicyConfig.Default.FastBackoffBaseMs, + SlowBackoffBaseMs = slowBaseMs ?? YdbRetryPolicyConfig.Default.SlowBackoffBaseMs, + FastCapBackoffMs = fastCapMs ?? YdbRetryPolicyConfig.Default.FastCapBackoffMs, + SlowCapBackoffMs = slowCapMs ?? YdbRetryPolicyConfig.Default.SlowCapBackoffMs + }; + + UseGlobally(cfg, onRetry); + } + } +} From 354f4ccc17acb5a135c9a824a1d6fd8f9f08714c Mon Sep 17 00:00:00 2001 From: poma12390 Date: Thu, 30 Oct 2025 23:41:07 +0300 Subject: [PATCH 05/11] create workflow --- .github/workflows/slo.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/slo.yml b/.github/workflows/slo.yml index cf4fe1a5..621d4be7 100644 --- a/.github/workflows/slo.yml +++ b/.github/workflows/slo.yml @@ -20,6 +20,7 @@ jobs: - AdoNet - Dapper - EF + - Linq2db.Slo include: - workload: AdoNet read_rps: 1000 @@ -30,7 +31,10 @@ jobs: - workload: EF read_rps: 1000 write_rps: 100 - + - workload: Linq2db.Slo + read_rps: 1000 + write_rps: 100 + concurrency: group: slo-${{ github.ref }}-${{ matrix.workload }} cancel-in-progress: true From f2703009a36775341d3f43d2e9094109f0a5736d Mon Sep 17 00:00:00 2001 From: poma12390 Date: Fri, 31 Oct 2025 00:40:01 +0300 Subject: [PATCH 06/11] fix write graphs --- slo/src/Linq2db.Slo/SloLinq2DbContext.cs | 23 ++++++++----------- .../src/Internal/YdbSdkRetryPolicyAdapter.cs | 19 ++++++--------- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/slo/src/Linq2db.Slo/SloLinq2DbContext.cs b/slo/src/Linq2db.Slo/SloLinq2DbContext.cs index ecdc1bc0..af1b2bfe 100644 --- a/slo/src/Linq2db.Slo/SloLinq2DbContext.cs +++ b/slo/src/Linq2db.Slo/SloLinq2DbContext.cs @@ -1,4 +1,6 @@ -using LinqToDB; +using System; +using System.Threading.Tasks; +using LinqToDB; using LinqToDB.Data; using LinqToDB.Mapping; using Internal; @@ -18,7 +20,7 @@ static SloTableContext() maxAttempts: 10, onRetry: (attempt, ex, delay) => { - // здесь при желании снимать метрики: attempt, delay, ((YdbException)ex).Code + // метрики/логи при желании } ); } @@ -26,14 +28,8 @@ static SloTableContext() public sealed class Linq2dbClient { private readonly string _connectionString; - - public Linq2dbClient(string connectionString) - { - _connectionString = connectionString; - } - - public DataConnection Open() - => new DataConnection("YDB", _connectionString); + public Linq2dbClient(string connectionString) => _connectionString = connectionString; + public DataConnection Open() => new DataConnection("YDB", _connectionString); } protected override Linq2dbClient CreateClient(Config config) @@ -58,15 +54,14 @@ PRIMARY KEY (Guid, Id) } catch { - // YDB не поддерживает IF NOT EXISTS; если таблица есть — окей + // YDB не поддерживает IF NOT EXISTS; если таблица есть — это норм } if (!string.IsNullOrWhiteSpace(SloTable.Options)) - { await db.ExecuteAsync(SloTable.Options); - } } + // ВАЖНО: вернуть >0 при успехе, иначе write-графики будут пустые. protected override async Task Save(Linq2dbClient client, SloTable sloTable, int writeTimeout) { await using var db = client.Open(); @@ -77,7 +72,7 @@ await db.ExecuteAsync($@" VALUES ({sloTable.Guid}, {sloTable.Id}, {sloTable.PayloadStr}, {sloTable.PayloadDouble}, {sloTable.PayloadTimestamp}); "); - return 0; + return 1; } protected override async Task Select(Linq2dbClient client, (Guid Guid, int Id) select, int readTimeout) diff --git a/src/Linq2db.Ydb/src/Internal/YdbSdkRetryPolicyAdapter.cs b/src/Linq2db.Ydb/src/Internal/YdbSdkRetryPolicyAdapter.cs index 2cdec88a..a8e3631a 100644 --- a/src/Linq2db.Ydb/src/Internal/YdbSdkRetryPolicyAdapter.cs +++ b/src/Linq2db.Ydb/src/Internal/YdbSdkRetryPolicyAdapter.cs @@ -1,7 +1,6 @@ -// для DataConnection в Factory -using L2RetryOptions = LinqToDB.Data.RetryPolicy.RetryPolicyOptions; +using L2RetryOptions = LinqToDB.Data.RetryPolicy.RetryPolicyOptions; using L2IRetryPolicy = LinqToDB.Data.RetryPolicy.IRetryPolicy; -using Ydb.Sdk.Ado; +using Ydb.Sdk.Ado; // <== важно: тут YdbException using Ydb.Sdk.Ado.RetryPolicy; namespace LinqToDB.Internal.DataProvider.Ydb.Internal @@ -127,10 +126,6 @@ private static bool TryFindYdbException(Exception ex, out YdbException ydbEx) /// public static class YdbSdkRetryPolicyRegistration { - /// - /// Подключить ретраи SDK глобально для всех новых DataConnection. - /// Вызывать один раз при старте/перед SLO-тестом. - /// public static void UseGlobally(YdbRetryPolicyConfig? config = null, Action? onRetry = null) { L2RetryOptions.Default = L2RetryOptions.Default with @@ -151,11 +146,11 @@ public static void UseGloballyWithIdempotence( var cfg = new YdbRetryPolicyConfig { EnableRetryIdempotence = true, - MaxAttempts = maxAttempts ?? YdbRetryPolicyConfig.Default.MaxAttempts, - FastBackoffBaseMs = fastBaseMs ?? YdbRetryPolicyConfig.Default.FastBackoffBaseMs, - SlowBackoffBaseMs = slowBaseMs ?? YdbRetryPolicyConfig.Default.SlowBackoffBaseMs, - FastCapBackoffMs = fastCapMs ?? YdbRetryPolicyConfig.Default.FastCapBackoffMs, - SlowCapBackoffMs = slowCapMs ?? YdbRetryPolicyConfig.Default.SlowCapBackoffMs + MaxAttempts = maxAttempts ?? YdbRetryPolicyConfig.Default.MaxAttempts, + FastBackoffBaseMs = fastBaseMs ?? YdbRetryPolicyConfig.Default.FastBackoffBaseMs, + SlowBackoffBaseMs = slowBaseMs ?? YdbRetryPolicyConfig.Default.SlowBackoffBaseMs, + FastCapBackoffMs = fastCapMs ?? YdbRetryPolicyConfig.Default.FastCapBackoffMs, + SlowCapBackoffMs = slowCapMs ?? YdbRetryPolicyConfig.Default.SlowCapBackoffMs }; UseGlobally(cfg, onRetry); From 243e8fda322f4cda12e5ad35afb8d8449ca6f797 Mon Sep 17 00:00:00 2001 From: poma12390 Date: Fri, 31 Oct 2025 01:30:18 +0300 Subject: [PATCH 07/11] fix write row affected --- slo/src/Linq2db.Slo/SloLinq2DbContext.cs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/slo/src/Linq2db.Slo/SloLinq2DbContext.cs b/slo/src/Linq2db.Slo/SloLinq2DbContext.cs index af1b2bfe..f3fe8414 100644 --- a/slo/src/Linq2db.Slo/SloLinq2DbContext.cs +++ b/slo/src/Linq2db.Slo/SloLinq2DbContext.cs @@ -13,15 +13,12 @@ public sealed class SloTableContext : SloTableContext "Linq2DB"; - // ВКЛЮЧАЕМ ретраи SDK глобально для всех DataConnection static SloTableContext() { + // Включаем ретраи SDK глобально (как и раньше) YdbSdkRetryPolicyRegistration.UseGloballyWithIdempotence( maxAttempts: 10, - onRetry: (attempt, ex, delay) => - { - // метрики/логи при желании - } + onRetry: (attempt, ex, delay) => { /* лог/метрики при желании */ } ); } @@ -32,8 +29,7 @@ public sealed class Linq2dbClient public DataConnection Open() => new DataConnection("YDB", _connectionString); } - protected override Linq2dbClient CreateClient(Config config) - => new Linq2dbClient(config.ConnectionString); + protected override Linq2dbClient CreateClient(Config config) => new(config.ConnectionString); protected override async Task Create(Linq2dbClient client, int operationTimeout) { @@ -46,7 +42,7 @@ await db.ExecuteAsync($@" CREATE TABLE `{SloTable.Name}` ( Guid Uuid, Id Int32, - PayloadStr Utf8, + PayloadStr Text, PayloadDouble Double, PayloadTimestamp Timestamp, PRIMARY KEY (Guid, Id) @@ -54,7 +50,7 @@ PRIMARY KEY (Guid, Id) } catch { - // YDB не поддерживает IF NOT EXISTS; если таблица есть — это норм + // Таблица уже есть — ок } if (!string.IsNullOrWhiteSpace(SloTable.Options)) @@ -67,12 +63,13 @@ protected override async Task Save(Linq2dbClient client, SloTable sloTable, await using var db = client.Open(); db.CommandTimeout = writeTimeout; - await db.ExecuteAsync($@" + // rowsAffected >= 0 (для UPSERT в YDB может быть 0), поэтому страхуемся и возвращаем минимум 1 + var rowsAffected = await db.ExecuteAsync($@" UPSERT INTO `{SloTable.Name}` (Guid, Id, PayloadStr, PayloadDouble, PayloadTimestamp) VALUES ({sloTable.Guid}, {sloTable.Id}, {sloTable.PayloadStr}, {sloTable.PayloadDouble}, {sloTable.PayloadTimestamp}); "); - return 1; + return rowsAffected > 0 ? rowsAffected : 1; } protected override async Task Select(Linq2dbClient client, (Guid Guid, int Id) select, int readTimeout) From e6a6cd614cae3dc354352c5e3ea92fb1f4656924 Mon Sep 17 00:00:00 2001 From: poma12390 Date: Fri, 31 Oct 2025 12:23:35 +0300 Subject: [PATCH 08/11] fix save method --- slo/src/Linq2db.Slo/SloLinq2DbContext.cs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/slo/src/Linq2db.Slo/SloLinq2DbContext.cs b/slo/src/Linq2db.Slo/SloLinq2DbContext.cs index f3fe8414..08749fc1 100644 --- a/slo/src/Linq2db.Slo/SloLinq2DbContext.cs +++ b/slo/src/Linq2db.Slo/SloLinq2DbContext.cs @@ -63,13 +63,20 @@ protected override async Task Save(Linq2dbClient client, SloTable sloTable, await using var db = client.Open(); db.CommandTimeout = writeTimeout; - // rowsAffected >= 0 (для UPSERT в YDB может быть 0), поэтому страхуемся и возвращаем минимум 1 - var rowsAffected = await db.ExecuteAsync($@" + var sql = $@" UPSERT INTO `{SloTable.Name}` (Guid, Id, PayloadStr, PayloadDouble, PayloadTimestamp) -VALUES ({sloTable.Guid}, {sloTable.Id}, {sloTable.PayloadStr}, {sloTable.PayloadDouble}, {sloTable.PayloadTimestamp}); -"); +VALUES (@Guid, @Id, @PayloadStr, @PayloadDouble, @PayloadTimestamp);"; - return rowsAffected > 0 ? rowsAffected : 1; + var affected = await db.ExecuteAsync( + sql, + new DataParameter("Guid", sloTable.Guid, DataType.Guid), + new DataParameter("Id", sloTable.Id, DataType.Int32), + new DataParameter("PayloadStr", sloTable.PayloadStr, DataType.NVarChar), + new DataParameter("PayloadDouble", sloTable.PayloadDouble, DataType.Double), + new DataParameter("PayloadTimestamp",sloTable.PayloadTimestamp,DataType.DateTime2) + ); + + return affected > 0 ? affected : 1; } protected override async Task Select(Linq2dbClient client, (Guid Guid, int Id) select, int readTimeout) From e1d2bb4afeb79b41f40fa1e383f5d9bebf393e29 Mon Sep 17 00:00:00 2001 From: poma12390 Date: Fri, 31 Oct 2025 13:45:17 +0300 Subject: [PATCH 09/11] create quickStart --- .../Linq2db.QuickStart.csproj | 21 + examples/Linq2db.QuickStart/Program.cs | 367 ++++++++++++++++++ examples/YdbExamples.sln | 6 + 3 files changed, 394 insertions(+) create mode 100644 examples/Linq2db.QuickStart/Linq2db.QuickStart.csproj create mode 100644 examples/Linq2db.QuickStart/Program.cs diff --git a/examples/Linq2db.QuickStart/Linq2db.QuickStart.csproj b/examples/Linq2db.QuickStart/Linq2db.QuickStart.csproj new file mode 100644 index 00000000..993ccebc --- /dev/null +++ b/examples/Linq2db.QuickStart/Linq2db.QuickStart.csproj @@ -0,0 +1,21 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + + + + + diff --git a/examples/Linq2db.QuickStart/Program.cs b/examples/Linq2db.QuickStart/Program.cs new file mode 100644 index 00000000..2b3bf27c --- /dev/null +++ b/examples/Linq2db.QuickStart/Program.cs @@ -0,0 +1,367 @@ +namespace Linq2db.QuickStart; + +using Microsoft.Extensions.Logging; +using Polly; +using LinqToDB; +using LinqToDB.Async; +using LinqToDB.Data; +using LinqToDB.Mapping; + +internal static class Program +{ + public static async Task Main(string[] args) + { + using var factory = LoggerFactory.Create(b => b.AddConsole()); + var app = new AppContext(factory.CreateLogger()); + await app.Run(); + } +} + +#region LINQ2DB MODELS + +[Table("series")] +public sealed class Series +{ + [PrimaryKey, Column("series_id")] + public ulong SeriesId { get; set; } + + [Column("title"), NotNull] + public string Title { get; set; } = null!; + + [Column("series_info")] + public string? SeriesInfo { get; set; } + + [Column("release_date"), DataType(DataType.Date)] + public DateTime ReleaseDate { get; set; } +} + +[Table("seasons")] +public sealed class Season +{ + [PrimaryKey, Column("series_id")] + public ulong SeriesId { get; set; } + + [PrimaryKey, Column("season_id")] + public ulong SeasonId { get; set; } + + [Column("title"), NotNull] + public string Title { get; set; } = null!; + + [Column("first_aired"), DataType(DataType.Date)] + public DateTime FirstAired { get; set; } + + [Column("last_aired"), DataType(DataType.Date)] + public DateTime LastAired { get; set; } +} + +[Table("episodes")] +public sealed class Episode +{ + [PrimaryKey, Column("series_id")] + public ulong SeriesId { get; set; } + + [PrimaryKey, Column("season_id")] + public ulong SeasonId { get; set; } + + [PrimaryKey, Column("episode_id")] + public ulong EpisodeId { get; set; } + + [Column("title"), NotNull] + public string Title { get; set; } = null!; + + [Column("air_date"), DataType(DataType.Date)] + public DateTime AirDate { get; set; } +} + +#endregion + +#region LINQ2DB DATACONTEXT + +internal sealed class MyYdb : DataConnection +{ + public MyYdb(string connectionString) : base("YDB", connectionString) { } + public MyYdb(DataOptions options) : base(options) { } + + public ITable Series => this.GetTable(); + public ITable Seasons => this.GetTable(); + public ITable Episodes => this.GetTable(); +} + +#endregion + +#region SETTINGS (без CmdOptions) + +internal sealed record Settings( + string Host, + int Port, + string Database, + bool UseTls, + int TlsPort) +{ + public string SimpleConnectionString => + $"Host={Host};Port={(UseTls ? TlsPort : Port)};Database={Database};UseTls={(UseTls ? "true" : "false")}"; +} + +internal static class SettingsLoader +{ + public static Settings Load() + { + string host = Environment.GetEnvironmentVariable("YDB_HOST") ?? "localhost"; + int port = TryInt(Environment.GetEnvironmentVariable("YDB_PORT"), 2136); + string db = Environment.GetEnvironmentVariable("YDB_DB") ?? "/local"; + bool useTls = TryBool(Environment.GetEnvironmentVariable("YDB_USE_TLS"), false); + int tls = TryInt(Environment.GetEnvironmentVariable("YDB_TLS_PORT"), 2135); + + return new Settings(host, port, db, useTls, tls); + + static int TryInt(string? s, int d) => int.TryParse(s, out var v) ? v : d; + static bool TryBool(string? s, bool d) => bool.TryParse(s, out var v) ? v : d; + } +} + +#endregion + +internal class AppContext +{ + private readonly ILogger _logger; + private readonly Settings _settings; + + public AppContext(ILogger logger) + { + _logger = logger; + _settings = SettingsLoader.Load(); + } + + DataOptions BuildOptions(string? overrideConnectionString = null) + { + var cs = overrideConnectionString ?? _settings.SimpleConnectionString; + return new DataOptions().UseConnectionString("YDB", cs); + } + + public async Task Run() + { + _logger.LogInformation("Start app example"); + + await InitTables(); + await LoadData(); + await SelectWithParameters(); + await RetryPolicy(); + + + await InteractiveTransaction(); + await TlsConnectionExample(); + await ConnectionWithLoggerFactory(); + + _logger.LogInformation("Finish app example"); + } + + private async Task InitTables() + { + await using var db = new MyYdb(BuildOptions()); + + try { await db.CreateTableAsync(); } catch { _logger.LogDebug("series exists"); } + try { await db.CreateTableAsync(); } catch { _logger.LogDebug("seasons exists"); } + try { await db.CreateTableAsync(); } catch { _logger.LogDebug("episodes exists"); } + + _logger.LogInformation("Created tables"); + } + + private async Task LoadData() + { + await using var db = new MyYdb(BuildOptions()); + + var series = new[] + { + new Series { SeriesId = 1, Title = "IT Crowd", ReleaseDate = new DateTime(2006,02,03), SeriesInfo="British sitcom..." }, + new Series { SeriesId = 2, Title = "Silicon Valley", ReleaseDate = new DateTime(2014,04,06), SeriesInfo="American comedy..." } + }; + foreach (var s in series) await db.InsertAsync(s); + + var seasons = new List + { + new() { SeriesId=1, SeasonId=1, Title="Season 1", FirstAired=new DateTime(2006,02,03), LastAired=new DateTime(2006,03,03)}, + new() { SeriesId=1, SeasonId=2, Title="Season 2", FirstAired=new DateTime(2007,08,24), LastAired=new DateTime(2007,09,28)}, + new() { SeriesId=1, SeasonId=3, Title="Season 3", FirstAired=new DateTime(2008,11,21), LastAired=new DateTime(2008,12,26)}, + new() { SeriesId=1, SeasonId=4, Title="Season 4", FirstAired=new DateTime(2010,06,25), LastAired=new DateTime(2010,07,30)}, + new() { SeriesId=2, SeasonId=1, Title="Season 1", FirstAired=new DateTime(2014,04,06), LastAired=new DateTime(2014,06,01)}, + new() { SeriesId=2, SeasonId=2, Title="Season 2", FirstAired=new DateTime(2015,04,12), LastAired=new DateTime(2015,06,14)}, + new() { SeriesId=2, SeasonId=3, Title="Season 3", FirstAired=new DateTime(2016,04,24), LastAired=new DateTime(2016,06,26)}, + new() { SeriesId=2, SeasonId=4, Title="Season 4", FirstAired=new DateTime(2017,04,23), LastAired=new DateTime(2017,06,25)}, + new() { SeriesId=2, SeasonId=5, Title="Season 5", FirstAired=new DateTime(2018,03,25), LastAired=new DateTime(2018,05,13)}, + }; + await db.BulkCopyAsync(seasons); + + var eps = new List + { + new() { SeriesId=1, SeasonId=1, EpisodeId=1, Title="Yesterday's Jam", AirDate=new DateTime(2006,02,03)}, + new() { SeriesId=1, SeasonId=1, EpisodeId=2, Title="Calamity Jen", AirDate=new DateTime(2006,02,03)}, + new() { SeriesId=1, SeasonId=1, EpisodeId=3, Title="Fifty-Fifty", AirDate=new DateTime(2006,02,10)}, + new() { SeriesId=1, SeasonId=1, EpisodeId=4, Title="The Red Door", AirDate=new DateTime(2006,02,17)}, + new() { SeriesId=1, SeasonId=2, EpisodeId=1, Title="The Work Outing", AirDate=new DateTime(2007,08,24)}, + new() { SeriesId=1, SeasonId=2, EpisodeId=2, Title="Return of the Golden Child", AirDate=new DateTime(2007,08,31)}, + new() { SeriesId=1, SeasonId=3, EpisodeId=1, Title="From Hell", AirDate=new DateTime(2008,11,21)}, + new() { SeriesId=1, SeasonId=3, EpisodeId=2, Title="Are We Not Men?", AirDate=new DateTime(2008,11,28)}, + new() { SeriesId=1, SeasonId=4, EpisodeId=1, Title="Jen The Fredo", AirDate=new DateTime(2010,06,25)}, + new() { SeriesId=1, SeasonId=4, EpisodeId=2, Title="The Final Countdown", AirDate=new DateTime(2010,07,02)}, + new() { SeriesId=2, SeasonId=2, EpisodeId=1, Title="Minimum Viable Product", AirDate=new DateTime(2014,04,06)}, + new() { SeriesId=2, SeasonId=2, EpisodeId=2, Title="The Cap Table", AirDate=new DateTime(2014,04,13)}, + new() { SeriesId=2, SeasonId=1, EpisodeId=3, Title="Articles of Incorporation", AirDate=new DateTime(2014,04,20)}, + new() { SeriesId=2, SeasonId=1, EpisodeId=4, Title="Fiduciary Duties", AirDate=new DateTime(2014,04,27)}, + }; + + await db.BulkCopyAsync(eps); + + _logger.LogInformation("Loaded data"); + } + + private async Task SelectWithParameters() + { + await using var db = new MyYdb(BuildOptions()); + + ulong seriesId = 1; + ulong seasonId = 1; + ulong limit = 3; + + var rows = await db.Episodes + .Where(e => e.SeriesId == seriesId && e.SeasonId > seasonId) + .OrderBy(e => e.SeriesId) + .ThenBy(e => e.SeasonId) + .ThenBy(e => e.EpisodeId) + .Take((int)limit) + .Select(e => new { e.SeriesId, e.SeasonId, e.EpisodeId, e.AirDate, e.Title }) + .ToListAsync(); + + _logger.LogInformation("Selected rows:"); + foreach (var r in rows) + _logger.LogInformation( + "series_id: {series_id}, season_id: {season_id}, episode_id: {episode_id}, air_date: {air_date}, title: {title}", + r.SeriesId, r.SeasonId, r.EpisodeId, r.AirDate, r.Title); + } + + private async Task RetryPolicy() + { + var policy = Policy + .Handle(_ => true) + .WaitAndRetryAsync(10, _ => TimeSpan.FromSeconds(1)); + + await policy.ExecuteAsync(async () => + { + await using var db = new MyYdb(BuildOptions()); + + var statsRaw = await db.Episodes + .GroupBy(e => new { e.SeriesId, e.SeasonId }) + .Select(g => new + { + SeriesId = g.Key.SeriesId, + SeasonId = g.Key.SeasonId, + Cnt = g.Count() + }) + .ToListAsync(); + + var stats = statsRaw + .OrderBy(x => x.SeriesId) + .ThenBy(x => x.SeasonId); + + foreach (var x in stats) + _logger.LogInformation("series_id: {series_id}, season_id: {season_id}, cnt: {cnt}", + x.SeriesId, x.SeasonId, x.Cnt); + }); + } + + private async Task InteractiveTransaction() + { + await using var db = new MyYdb(BuildOptions()); + using var tr = await db.BeginTransactionAsync(); + + await db.InsertAsync(new Episode + { + SeriesId = 2, SeasonId = 5, EpisodeId = 13, + Title = "Test Episode", AirDate = new DateTime(2018, 08, 27) + }); + await db.InsertAsync(new Episode + { + SeriesId = 2, SeasonId = 5, EpisodeId = 21, + Title = "Test 21", AirDate = new DateTime(2018, 08, 27) + }); + await db.InsertAsync(new Episode + { + SeriesId = 2, SeasonId = 5, EpisodeId = 22, + Title = "Test 22", AirDate = new DateTime(2018, 08, 27) + }); + + await tr.CommitAsync(); + _logger.LogInformation("Commit transaction"); + + string title21 = await db.Episodes + .Where(e => e.SeriesId == 2 && e.SeasonId == 5 && e.EpisodeId == 21) + .Select(e => e.Title) + .SingleAsync(); + _logger.LogInformation("New episode title: {title}", title21); + + string title22 = await db.Episodes + .Where(e => e.SeriesId == 2 && e.SeasonId == 5 && e.EpisodeId == 22) + .Select(e => e.Title) + .SingleAsync(); + _logger.LogInformation("New episode title: {title}", title22); + + string title13 = await db.Episodes + .Where(e => e.SeriesId == 2 && e.SeasonId == 5 && e.EpisodeId == 13) + .Select(e => e.Title) + .SingleAsync(); + _logger.LogInformation("Updated episode title: {title}", title13); + } + + private async Task TlsConnectionExample() + { + if (!_settings.UseTls) + { + _logger.LogInformation("Tls example was ignored"); + return; + } + + var caPath = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "ca.pem"); + var tlsCs = $"Host={_settings.Host};Port={_settings.TlsPort};RootCertificate={caPath}"; + await using var db = new MyYdb(BuildOptions(tlsCs)); + + var rows = await db.Seasons + .Where(sa => sa.SeriesId == 1) + .Join(db.Series, sa => sa.SeriesId, sr => sr.SeriesId, + (sa, sr) => new { SeasonTitle = sa.Title, SeriesTitle = sr.Title, sr.SeriesId, sa.SeasonId }) + .OrderBy(x => x.SeriesId).ThenBy(x => x.SeasonId) + .ToListAsync(); + + foreach (var r in rows) + _logger.LogInformation( + "season_title: {SeasonTitle}, series_title: {SeriesTitle}, series_id: {SeriesId}, season_id: {SeasonId}", + r.SeasonTitle, r.SeriesTitle, r.SeriesId, r.SeasonId); + } + + private async Task ConnectionWithLoggerFactory() + { + await using var db = new MyYdb(BuildOptions( + $"Host={_settings.Host};Port={_settings.Port}")); + + db.OnTraceConnection = ti => + { + switch (ti.TraceInfoStep) + { + case TraceInfoStep.BeforeExecute: + _logger.LogInformation("BeforeExecute: {sql}", ti.SqlText); + break; + case TraceInfoStep.AfterExecute: + _logger.LogInformation("AfterExecute: {time} {records} recs", ti.ExecutionTime, ti.RecordsAffected); + break; + case TraceInfoStep.Error: + _logger.LogError(ti.Exception, "SQL error"); + break; + } + }; + + _logger.LogInformation("Dropping tables of examples"); + try { await db.DropTableAsync(); } catch { /* ignore */ } + try { await db.DropTableAsync(); } catch { /* ignore */ } + try { await db.DropTableAsync(); } catch { /* ignore */ } + _logger.LogInformation("Dropped tables of examples"); + } +} diff --git a/examples/YdbExamples.sln b/examples/YdbExamples.sln index 31c103ee..ecf70e2a 100644 --- a/examples/YdbExamples.sln +++ b/examples/YdbExamples.sln @@ -27,6 +27,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Database.Operations.Tutoria EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Ydb.Sdk.AdoNet.Yandex.Cloud.Serverless.Container", "Ydb.Sdk.AdoNet.Yandex.Cloud.Serverless.Container\Ydb.Sdk.AdoNet.Yandex.Cloud.Serverless.Container.csproj", "{77625697-498B-4879-BABA-046EE93E7AF7}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Linq2db.QuickStart", "Linq2db.QuickStart\Linq2db.QuickStart.csproj", "{2D3A5D18-9005-4E19-92C0-002069CAC7BB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -81,6 +83,10 @@ Global {77625697-498B-4879-BABA-046EE93E7AF7}.Debug|Any CPU.Build.0 = Debug|Any CPU {77625697-498B-4879-BABA-046EE93E7AF7}.Release|Any CPU.ActiveCfg = Release|Any CPU {77625697-498B-4879-BABA-046EE93E7AF7}.Release|Any CPU.Build.0 = Release|Any CPU + {2D3A5D18-9005-4E19-92C0-002069CAC7BB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2D3A5D18-9005-4E19-92C0-002069CAC7BB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2D3A5D18-9005-4E19-92C0-002069CAC7BB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2D3A5D18-9005-4E19-92C0-002069CAC7BB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From d45366e0fcb11ece08523e0a2114213d1dc3fdb8 Mon Sep 17 00:00:00 2001 From: poma12390 Date: Fri, 31 Oct 2025 13:57:30 +0300 Subject: [PATCH 10/11] create Readme Quickstart --- examples/Linq2db.QuickStart/README.md | 39 +++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 examples/Linq2db.QuickStart/README.md diff --git a/examples/Linq2db.QuickStart/README.md b/examples/Linq2db.QuickStart/README.md new file mode 100644 index 00000000..df510fe7 --- /dev/null +++ b/examples/Linq2db.QuickStart/README.md @@ -0,0 +1,39 @@ +# Linq2DB YDB Quick Start + +A tiny sample that shows how to connect to **YDB** with **Linq2DB**, create tables, seed demo data, run parameterized queries and a transaction. + +## Running QuickStart + +1. **Start YDB Local** + Follow the official guide: https://ydb.tech/docs/en/reference/docker/start + Defaults expected by the sample: + - non‑TLS port: **2136** + - TLS port: **2135** + - database: **/local** + +2. **(Optional) Configure environment variables** + The app reads connection settings from env vars (safe defaults are used if missing). + + **Bash** + ```bash + export YDB_HOST=localhost + export YDB_PORT=2136 + export YDB_DB=/local + export YDB_USE_TLS=false + # export YDB_TLS_PORT=2135 + ``` + + **PowerShell** + ```powershell + $env:YDB_HOST="localhost" + $env:YDB_PORT="2136" + $env:YDB_DB="/local" + $env:YDB_USE_TLS="false" + # $env:YDB_TLS_PORT="2135" + ``` + +3. **Restore & run** + ```bash + dotnet restore + dotnet run + ``` \ No newline at end of file From 2f985c67d662474654028de0edd21e8c5b66e555 Mon Sep 17 00:00:00 2001 From: poma12390 Date: Fri, 31 Oct 2025 18:35:06 +0300 Subject: [PATCH 11/11] fix quckStart --- examples/Linq2db.QuickStart/Program.cs | 6 ++++-- src/Linq2db.Ydb/src/Linq2db.csproj | 3 --- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/Linq2db.QuickStart/Program.cs b/examples/Linq2db.QuickStart/Program.cs index 2b3bf27c..846e7eb8 100644 --- a/examples/Linq2db.QuickStart/Program.cs +++ b/examples/Linq2db.QuickStart/Program.cs @@ -1,4 +1,6 @@ -namespace Linq2db.QuickStart; +using LinqToDB.Internal.DataProvider.Ydb; + +namespace Linq2db.QuickStart; using Microsoft.Extensions.Logging; using Polly; @@ -157,8 +159,8 @@ public async Task Run() private async Task InitTables() { + DataConnection.AddProviderDetector(YdbTools.ProviderDetector); await using var db = new MyYdb(BuildOptions()); - try { await db.CreateTableAsync(); } catch { _logger.LogDebug("series exists"); } try { await db.CreateTableAsync(); } catch { _logger.LogDebug("seasons exists"); } try { await db.CreateTableAsync(); } catch { _logger.LogDebug("episodes exists"); } diff --git a/src/Linq2db.Ydb/src/Linq2db.csproj b/src/Linq2db.Ydb/src/Linq2db.csproj index 99c1884f..f76d19cc 100644 --- a/src/Linq2db.Ydb/src/Linq2db.csproj +++ b/src/Linq2db.Ydb/src/Linq2db.csproj @@ -10,7 +10,6 @@ true - @@ -20,8 +19,6 @@ - - Never