diff --git a/src/Core/Resolvers/IQueryBuilder.cs b/src/Core/Resolvers/IQueryBuilder.cs index 3a14e472cd..f6a6222ddc 100644 --- a/src/Core/Resolvers/IQueryBuilder.cs +++ b/src/Core/Resolvers/IQueryBuilder.cs @@ -90,5 +90,7 @@ public interface IQueryBuilder /// DB Connection Param. /// public string QuoteTableNameAsDBConnectionParam(string param); + + public string BuildGetAutoentitiesQuery() => throw new NotSupportedException($"{GetType().Name} does not support Autoentities yet."); } } diff --git a/src/Core/Resolvers/MsSqlQueryBuilder.cs b/src/Core/Resolvers/MsSqlQueryBuilder.cs index 798c20975a..7adedc64d4 100644 --- a/src/Core/Resolvers/MsSqlQueryBuilder.cs +++ b/src/Core/Resolvers/MsSqlQueryBuilder.cs @@ -506,7 +506,7 @@ public string BuildStoredProcedureResultDetailsQuery(string databaseObjectName) /// 2. are computed based on other columns, /// are considered as read only columns. The query combines both the types of read-only columns and returns the list. /// - /// Param name of the schema/database. + /// Param name of the schema. /// Param name of the table. /// public string BuildQueryToGetReadOnlyColumns(string schemaParamName, string tableParamName) @@ -560,5 +560,188 @@ protected override string BuildPredicates(SqlQueryStructure structure) // contains LIKE and add the ESCAPE clause accordingly. return AddEscapeToLikeClauses(predicates); } + + /// + /// Builds the query used to get the list of tables with the SQL LIKE + /// syntax that will be transformed into entities. + /// NOTE: Currently this query only returns Tables, support for Views will come later. + /// + /// Pattern for tables that will be included. + /// Pattern for tables that will be excluded. + /// Pattern for naming the entities. + public string BuildGetAutoentitiesQuery() + { + string query = @$" + DECLARE @exclude_invalid_types BIT = 1; + + SET NOCOUNT ON; + + WITH + {IncludeAndExcludeSplitQuery(true)}, + {IncludeAndExcludeSplitQuery(false)}, + all_tables AS + ( + SELECT + s.name AS schema_name, + t.name AS object_name, + s.name + N'.' + t.name AS full_name, + N'table' AS object_type, + t.object_id + FROM sys.tables AS t + JOIN sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE EXISTS + ( + SELECT 1 + FROM sys.key_constraints AS kc + WHERE kc.parent_object_id = t.object_id + AND kc.type = 'PK' + ) + ), + eligible_tables AS + ( + SELECT + o.schema_name, + o.object_name, + o.full_name, + o.object_type, + o.object_id, + CASE + WHEN so.is_ms_shipped = 1 THEN 1 + WHEN o.schema_name IN (N'sys', N'INFORMATION_SCHEMA') THEN 1 + WHEN o.object_name IN + ( + N'__EFMigrationsHistory', + N'__MigrationHistory', + N'__FlywayHistory', + N'sysdiagrams' + ) THEN 1 + WHEN o.object_name LIKE N'service_broker_%' THEN 1 + WHEN o.object_name LIKE N'queue_messages_%' THEN 1 + WHEN o.object_name LIKE N'MSmerge_%' THEN 1 + WHEN o.object_name LIKE N'MSreplication_%' THEN 1 + WHEN o.object_name LIKE N'FileTableUpdates$%' THEN 1 + WHEN o.object_name LIKE N'graph_%' THEN 1 + WHEN EXISTS + ( + SELECT 1 + FROM sys.tables AS t + WHERE t.object_id = o.object_id + AND + ( + t.is_tracked_by_cdc = 1 + OR t.temporal_type > 0 + OR t.is_filetable = 1 + OR t.is_memory_optimized = 1 + ) + ) THEN 1 + ELSE 0 + END AS is_system_object + FROM all_tables AS o + JOIN sys.objects AS so + ON so.object_id = o.object_id + ) + SELECT + a.schema_name AS [schema], + a.object_name AS [object], + CASE + WHEN LTRIM(RTRIM(ISNULL(@name_pattern, N''))) = N'' THEN a.object_name + ELSE REPLACE( + REPLACE(@name_pattern, N'{{schema}}', a.schema_name), + N'{{object}}', a.object_name + ) + END AS entity_name, + CASE + WHEN EXISTS + ( + SELECT 1 + FROM sys.columns AS c + JOIN sys.types AS ty + ON c.user_type_id = ty.user_type_id + WHERE c.object_id = a.object_id + AND ty.name IN + ( + N'geography', + N'geometry', + N'hierarchyid', + N'sql_variant', + N'xml', + N'rowversion', + N'vector' + ) + ) THEN 1 + ELSE 0 + END AS contains_invalid_types + FROM eligible_tables AS a + WHERE + a.is_system_object = 0 + AND + ( + NOT EXISTS (SELECT 1 FROM exclude_patterns) + OR NOT EXISTS + ( + SELECT 1 + FROM exclude_patterns AS ep + WHERE a.full_name LIKE ep.pattern COLLATE DATABASE_DEFAULT ESCAPE '\' + ) + ) + AND + ( + NOT EXISTS (SELECT 1 FROM include_patterns) + OR EXISTS + ( + SELECT 1 + FROM include_patterns AS ip + WHERE a.full_name LIKE ip.pattern COLLATE DATABASE_DEFAULT ESCAPE '\' + ) + ) + AND + ( + @exclude_invalid_types = 0 + OR NOT EXISTS + ( + SELECT 1 + FROM sys.columns AS c + JOIN sys.types AS ty + ON c.user_type_id = ty.user_type_id + WHERE c.object_id = a.object_id + AND ty.name IN + ( + N'geography', + N'geometry', + N'hierarchyid', + N'sql_variant', + N'xml', + N'rowversion', + N'vector' + ) + ) + ) + ORDER BY + a.schema_name, + a.object_name;"; + + return query; + } + + /// + /// Generates a SQL query segment for splitting include or exclude patterns. + /// + /// Indicates whether to generate the include or exclude pattern query. + /// An SQL query segment as a string. + public static string IncludeAndExcludeSplitQuery(bool isInclude) + { + string pattern = isInclude ? "include" : "exclude"; + + string query = $@" + {pattern}_patterns AS + ( + SELECT LTRIM(RTRIM(value)) AS pattern + FROM STRING_SPLIT(ISNULL(@{pattern}_pattern, N''), N',') + WHERE LTRIM(RTRIM(value)) <> N'' + )"; + + return query; + } } } diff --git a/src/Core/Resolvers/MsSqlQueryExecutor.cs b/src/Core/Resolvers/MsSqlQueryExecutor.cs index 5cbe9f6a76..6a900fb7ff 100644 --- a/src/Core/Resolvers/MsSqlQueryExecutor.cs +++ b/src/Core/Resolvers/MsSqlQueryExecutor.cs @@ -82,7 +82,7 @@ public MsSqlQueryExecutor( _dataSourceToSessionContextUsage = new Dictionary(); _accessTokensFromConfiguration = runtimeConfigProvider.ManagedIdentityAccessToken; _runtimeConfigProvider = runtimeConfigProvider; - ConfigureMsSqlQueryEecutor(); + ConfigureMsSqlQueryExecutor(); } /// @@ -138,7 +138,7 @@ public override SqlConnection CreateConnection(string dataSourceName) /// /// Configure during construction or a hot-reload scenario. /// - private void ConfigureMsSqlQueryEecutor() + private void ConfigureMsSqlQueryExecutor() { IEnumerable> mssqldbs = _runtimeConfigProvider.GetConfig().GetDataSourceNamesToDataSourcesIterator().Where(x => x.Value.DatabaseType is DatabaseType.MSSQL || x.Value.DatabaseType is DatabaseType.DWSQL); diff --git a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs index 7d02798427..4e3be0548a 100644 --- a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs @@ -290,5 +290,48 @@ private bool TryResolveDbType(string sqlDbTypeName, out DbType dbType) return false; } } + + /// + // TODO: Finish implementation of autoentities generation in task #3052 + protected override async Task GenerateAutoentitiesIntoEntities() + { + await Task.CompletedTask; // Temporary await to suppress build errors. + + /*RuntimeConfig runtimeConfig = _runtimeConfigProvider.GetConfig(); + if (runtimeConfig.Autoentities is not null) + { + foreach ((string name, Autoentity autoentity) in runtimeConfig.Autoentities.AutoEntities) + { + JsonArray? resultArray = await QueryAutoentitiesAsync(autoentity); + } + }*/ + } + + public async Task QueryAutoentitiesAsync(Autoentity autoentity) + { + string include = string.Join(",", autoentity.Patterns.Include); + string exclude = string.Join(",", autoentity.Patterns.Exclude); + string namePattern = autoentity.Patterns.Name; + string getAutoentitiesQuery = SqlQueryBuilder.BuildGetAutoentitiesQuery(); + Dictionary parameters = new() + { + { $"{BaseQueryStructure.PARAM_NAME_PREFIX}include_pattern", new(include, null, SqlDbType.NVarChar) }, + { $"{BaseQueryStructure.PARAM_NAME_PREFIX}exclude_pattern", new(exclude, null, SqlDbType.NVarChar) }, + { $"{BaseQueryStructure.PARAM_NAME_PREFIX}name_pattern", new(namePattern, null, SqlDbType.NVarChar) } + }; + + _logger.LogInformation("Query for Autoentities is being executed with the following parameters."); + _logger.LogInformation($"Autoentities include pattern: {include}"); + _logger.LogInformation($"Autoentities exclude pattern: {exclude}"); + _logger.LogInformation($"Autoentities name pattern: {namePattern}"); + + JsonArray? resultArray = await QueryExecutor.ExecuteQueryAsync( + sqltext: getAutoentitiesQuery, + parameters: parameters, + dataReaderHandler: QueryExecutor.GetJsonArrayAsync, + dataSourceName: _dataSourceName); + + return resultArray; + } } } diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index 8553e08136..c9a62ca470 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -307,6 +307,11 @@ public string GetEntityName(string graphQLType) public async Task InitializeAsync() { System.Diagnostics.Stopwatch timer = System.Diagnostics.Stopwatch.StartNew(); + if (GetDatabaseType() == DatabaseType.MSSQL) + { + await GenerateAutoentitiesIntoEntities(); + } + GenerateDatabaseObjectForEntities(); if (_isValidateOnly) { @@ -686,6 +691,15 @@ private void GenerateDatabaseObjectForEntities() } } + /// + /// Creates entities for each table that is found, based on the autoentity configuration. + /// This method is only called for tables in MsSql. + /// + protected virtual Task GenerateAutoentitiesIntoEntities() + { + throw new NotSupportedException($"{GetType().Name} does not support Autoentities yet."); + } + protected void PopulateDatabaseObjectForEntity( Entity entity, string entityName, diff --git a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs index 8b4ed68f60..30969918c0 100644 --- a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs +++ b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs @@ -588,5 +588,68 @@ private static async Task SetupTestFixtureAndInferMetadata() await ResetDbStateAsync(); await _sqlMetadataProvider.InitializeAsync(); } + + /// + /// Ensures that the query that returns the tables that will be generated + /// into entities from the autoentities configuration returns the expected result. + /// + [DataTestMethod, TestCategory(TestCategory.MSSQL)] + [DataRow(new string[] { "dbo.%book%" }, new string[] { }, "{schema}.{object}.books", new string[] { "book" }, "")] + [DataRow(new string[] { "dbo.%publish%" }, new string[] { }, "{schema}.{object}", new string[] { "publish" }, "")] + [DataRow(new string[] { "dbo.%book%" }, new string[] { "dbo.%books%" }, "{schema}_{object}_exclude_books", new string[] { "book" }, "books")] + [DataRow(new string[] { "dbo.%book%", "dbo.%publish%" }, new string[] { }, "{object}", new string[] { "book", "publish" }, "")] + [DataRow(new string[] { }, new string[] { "dbo.%book%" }, "{object}", new string[] { "" }, "book")] + public async Task CheckAutoentitiesQuery(string[] include, string[] exclude, string name, string[] includeObject, string excludeObject) + { + // Arrange + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + RuntimeConfig runtimeConfig = SqlTestHelper.SetupRuntimeConfig(); + Autoentity autoentity = new(new AutoentityPatterns(include, exclude, name), null, null); + Dictionary dictAutoentity = new() + { + { "autoentity", autoentity } + }; + RuntimeConfig configWithAutoentity = runtimeConfig with + { + Autoentities = new RuntimeAutoentities(dictAutoentity) + }; + RuntimeConfigProvider runtimeConfigProvider = TestHelper.GenerateInMemoryRuntimeConfigProvider(configWithAutoentity); + SetUpSQLMetadataProvider(runtimeConfigProvider); + + await _sqlMetadataProvider.InitializeAsync(); + + // Act + MsSqlMetadataProvider metadataProvider = (MsSqlMetadataProvider)_sqlMetadataProvider; + JsonArray resultArray = await metadataProvider.QueryAutoentitiesAsync(autoentity); + + // Assert + Assert.IsNotNull(resultArray); + foreach (JsonObject resultObject in resultArray) + { + bool includedObjectExists = false; + foreach (string included in includeObject) + { + if (resultObject["object"].ToString().Contains(included)) + { + includedObjectExists = true; + Assert.AreNotEqual(name, resultObject["entity_name"].ToString(), "Name returned by query should not include {schema} or {object}."); + if (include.Length > 0) + { + Assert.AreEqual(expected: "dbo", actual: resultObject["schema"].ToString(), "Query does not return expected schema."); + } + + if (exclude.Length > 0) + { + Assert.IsTrue(!resultObject["object"].ToString().Contains(excludeObject), "Query returns pattern that should be excluded."); + } + } + } + + Assert.IsTrue(includedObjectExists, "Query does not return expected object."); + } + + TestHelper.UnsetAllDABEnvironmentVariables(); + } } }