From 3dfd4f94f0ede0196646f8cd62388278a6baf937 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 13:15:25 -0700 Subject: [PATCH 01/32] Add PackageVersion for Kusto.Data and Kusto.Ingest --- Directory.Packages.props | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index ef11ad3a56..b11d7c6f43 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -56,7 +56,8 @@ - + + From 1342541936d7b38cc07b17fec2ae7ebfa9e93d6f Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 13:15:57 -0700 Subject: [PATCH 02/32] Add Id to shared McpModels. --- eng/tools/ToolDescriptionEvaluator/Models/McpModels.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/eng/tools/ToolDescriptionEvaluator/Models/McpModels.cs b/eng/tools/ToolDescriptionEvaluator/Models/McpModels.cs index ad6ce6c59f..b4a68bb118 100644 --- a/eng/tools/ToolDescriptionEvaluator/Models/McpModels.cs +++ b/eng/tools/ToolDescriptionEvaluator/Models/McpModels.cs @@ -81,8 +81,11 @@ public class ToolAnnotations } // Tool definition for azmcp tools list response -public class Tool -{ +public class Tool { + + [JsonPropertyName("id")] + public string? Id { get; set; } + [JsonPropertyName("name")] public required string Name { get; set; } From 3cba80c004f694db0315115cb0b12a399482b75b Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 13:16:21 -0700 Subject: [PATCH 03/32] Initial commit --- .../ToolMetadataExporter/AppConfiguration.cs | 17 ++ .../ToolMetadataExporter/GlobalUsings.cs | 5 + .../Models/AzureMcpTool.cs | 6 + .../Models/Kusto/McpToolEvent.cs | 56 ++++ .../Models/Kusto/McpToolEventType.cs | 11 + .../Models/ModelsSerializationContext.cs | 14 + eng/tools/ToolMetadataExporter/Program.cs | 111 ++++++++ .../Resources/queries/GetAvailableTools.kql | 4 + .../Services/AzmcpProgram.cs | 19 ++ .../Services/AzureMcpKustoDatastore.cs | 177 ++++++++++++ .../Services/IAzureMcpDatastore.cs | 14 + .../ToolMetadataExporter.csproj | 32 +++ eng/tools/ToolMetadataExporter/Utility.cs | 254 ++++++++++++++++++ .../ToolMetadataExporter/appsettings.json | 5 + 14 files changed, 725 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter/AppConfiguration.cs create mode 100644 eng/tools/ToolMetadataExporter/GlobalUsings.cs create mode 100644 eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs create mode 100644 eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs create mode 100644 eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs create mode 100644 eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs create mode 100644 eng/tools/ToolMetadataExporter/Program.cs create mode 100644 eng/tools/ToolMetadataExporter/Resources/queries/GetAvailableTools.kql create mode 100644 eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs create mode 100644 eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs create mode 100644 eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs create mode 100644 eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj create mode 100644 eng/tools/ToolMetadataExporter/Utility.cs create mode 100644 eng/tools/ToolMetadataExporter/appsettings.json diff --git a/eng/tools/ToolMetadataExporter/AppConfiguration.cs b/eng/tools/ToolMetadataExporter/AppConfiguration.cs new file mode 100644 index 0000000000..3e9142a5d0 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/AppConfiguration.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace ToolMetadataExporter; + +public class AppConfiguration +{ + public string? ClusterName { get; set; } + + public string? DatabaseName { get; set; } + + public string? McpToolEventsTableName { get; set; } + + public string? QueriesFolder { get; set; } + + public string? WorkDirectory { get; set; } +} diff --git a/eng/tools/ToolMetadataExporter/GlobalUsings.cs b/eng/tools/ToolMetadataExporter/GlobalUsings.cs new file mode 100644 index 0000000000..38bac6b955 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/GlobalUsings.cs @@ -0,0 +1,5 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +global using System; +global using System.Text.Json; diff --git a/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs b/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs new file mode 100644 index 0000000000..60cc77c698 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs @@ -0,0 +1,6 @@ +namespace ToolMetadataExporter.Models; + +public record AzureMcpTool( + string ToolId, + string ToolName, + string ToolArea); diff --git a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs new file mode 100644 index 0000000000..0adc12a18e --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; +using Kusto.Data.Common; + +namespace ToolMetadataExporter.Models.Kusto; + +public class McpToolEvent +{ + private const string EventTimeColumn = "EventTime"; + private const string EventTypeColumn = "EventType"; + private const string ServerVersionColumn = "ServerVersion"; + private const string ToolIdColumn = "ToolId"; + private const string ToolNameColumn = "ToolName"; + private const string ToolAreaColumn = "ToolArea"; + private const string ReplacedByToolNameColumn = "ReplacedByToolName"; + private const string ReplacedByToolAreaColumn = "ReplacedByToolArea"; + + [JsonPropertyName(EventTimeColumn)] + public DateTimeOffset? EventTime { get; set; } + + [JsonPropertyName(EventTypeColumn)] + public McpToolEventType? EventType { get; set; } + + [JsonPropertyName(ServerVersionColumn)] + public string? ServerVersion { get; set; } + + [JsonPropertyName(ToolIdColumn)] + public string? ToolId { get; set; } + + [JsonPropertyName(ToolNameColumn)] + public string? ToolName { get; set; } + + [JsonPropertyName(ToolAreaColumn)] + public string? ToolArea { get; set; } + + [JsonPropertyName(ReplacedByToolNameColumn)] + public string? ReplacedByToolName { get; set; } + + [JsonPropertyName(ReplacedByToolAreaColumn)] + public string? ReplacedByToolArea { get; set; } + + public static ColumnMapping[] GetColumnMappings() + { + return [ + new ColumnMapping { ColumnName = EventTypeColumn, ColumnType = "datetime" }, + new ColumnMapping { ColumnName = ServerVersionColumn, ColumnType = "string" }, + new ColumnMapping { ColumnName = ToolIdColumn, ColumnType = "string"}, + new ColumnMapping { ColumnName = ToolNameColumn, ColumnType = "string" }, + new ColumnMapping { ColumnName = ToolAreaColumn , ColumnType = "string" }, + new ColumnMapping { ColumnName = ReplacedByToolNameColumn, ColumnType = "string"}, + new ColumnMapping { ColumnName = ReplacedByToolAreaColumn, ColumnType = "string"}, + ]; + } +} diff --git a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs new file mode 100644 index 0000000000..3c595bbc1b --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace ToolMetadataExporter.Models.Kusto; + +public enum McpToolEventType +{ + Created, + Updated, + Deleted +} diff --git a/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs b/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs new file mode 100644 index 0000000000..e2b4805f98 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; +using ToolMetadataExporter.Models.Kusto; + +namespace ToolMetadataExporter.Models; + +[JsonSerializable(typeof(McpToolEvent))] +[JsonSerializable(typeof(McpToolEventType))] +[JsonSerializable(typeof(List))] +public partial class ModelsSerializationContext : JsonSerializerContext +{ +} diff --git a/eng/tools/ToolMetadataExporter/Program.cs b/eng/tools/ToolMetadataExporter/Program.cs new file mode 100644 index 0000000000..8a6261d112 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Program.cs @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using ToolMetadataExporter.Models.Kusto; +using ToolMetadataExporter.Services; + +namespace ToolMetadataExporter; + +public class Program +{ + private readonly AzmcpProgram _azmcpExe; + private readonly IAzureMcpDatastore _azureMcpDatastore; + private readonly ILogger _logger; + + public Program(AzmcpProgram program, IAzureMcpDatastore azureMcpDatastore, ILogger logger) + { + _azmcpExe = program; + _azureMcpDatastore = azureMcpDatastore; + _logger = logger; + } + + public static async Task Main(string[] args) + { + var builder = Host.CreateApplicationBuilder(args); + ConfigureServices(builder.Services); + + var host = builder.Build(); + + var program = host.Services.GetRequiredService(); + + await program.RunAsync(DateTimeOffset.UtcNow, isDryRun: true); + await host.RunAsync(); + } + + private static void ConfigureServices(IServiceCollection services) + { + + services.AddSingleton() + .AddSingleton(); + + services.AddOptions() + .PostConfigure(existing => + { + if (existing.WorkDirectory == null) + { + string exeDir = AppContext.BaseDirectory; + var repoRoot = Utility.FindRepoRoot(exeDir); + existing.WorkDirectory = Path.Combine(repoRoot, ".work"); + } + }); + + services.AddLogging(builder => + { + builder.AddConsole(); + }); + } + + private async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, CancellationToken cancellationToken = default) + { + _logger.LogInformation("Starting analysis."); + + var currentTools = await _azmcpExe.LoadToolsDynamicallyAsync(); + + if (currentTools == null) + { + _logger.LogError("LoadToolsDynamicallyAsync did not return a result."); + return; + } + else if (currentTools.Tools == null || currentTools.Tools.Count == 0) + { + _logger.LogWarning("azmcp program did not return any tools."); + return; + } + + var existingTools = (await _azureMcpDatastore.GetAvailableToolsAsync(cancellationToken)).ToDictionary(x => x.ToolId); + + if (cancellationToken.IsCancellationRequested) + { + _logger.LogInformation("Analysis was cancelled."); + return; + } + + var changes = new List(); + + foreach (var tool in currentTools.Tools) + { + if (string.IsNullOrEmpty(tool.Id)) + { + throw new InvalidOperationException($"Tool without an id. Name: {tool.Name}. Command: {tool.Command}"); + } + + if (!existingTools.TryGetValue(tool.Id, out var knownValue)) + { + + } + else + { + changes.Add(new McpToolEvent + { + EventTime = analysisTime, + EventType = McpToolEventType.Created, + }); + } + } + + await _azureMcpDatastore.AddToolEventsAsync(changes, cancellationToken); + } +} diff --git a/eng/tools/ToolMetadataExporter/Resources/queries/GetAvailableTools.kql b/eng/tools/ToolMetadataExporter/Resources/queries/GetAvailableTools.kql new file mode 100644 index 0000000000..26a901adc8 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Resources/queries/GetAvailableTools.kql @@ -0,0 +1,4 @@ +McpToolEvents + | summarize arg_max(EventTime, *) by ToolId + | where EventType != 'Deleted' + | project EventTime, ToolId, ToolName, ToolArea, ServerVersion, EventType, ReplacedByToolName, ReplacedByToolArea diff --git a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs new file mode 100644 index 0000000000..f58b0558e2 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.Options; +using ToolSelection.Models; + +namespace ToolMetadataExporter.Services; + +public class AzmcpProgram +{ + private readonly string _toolDirectory; + + public AzmcpProgram(IOptions options) + { + _toolDirectory = options.Value.WorkDirectory ?? throw new ArgumentNullException(nameof(AppConfiguration.WorkDirectory)); + } + + public virtual Task LoadToolsDynamicallyAsync() + { + return Utility.LoadToolsDynamicallyAsync(_toolDirectory, false); + } +} diff --git a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs new file mode 100644 index 0000000000..beb756234a --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Runtime.CompilerServices; +using Kusto.Data.Common; +using Kusto.Data.Ingestion; +using Kusto.Ingest; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ToolMetadataExporter.Models; +using ToolMetadataExporter.Models.Kusto; + +namespace ToolMetadataExporter.Services; + +public class AzureMcpKustoDatastore : IAzureMcpDatastore +{ + private readonly ICslQueryProvider _kustoClient; + private readonly IKustoIngestClient _ingestClient; + private readonly ILogger _logger; + private readonly DirectoryInfo _queriesDirectory; + private readonly string _databaseName; + private readonly string _tableName; + + public AzureMcpKustoDatastore( + ICslQueryProvider kustoClient, + IKustoIngestClient ingestClient, + IOptions configuration, + ILogger logger) + { + _kustoClient = kustoClient; + _ingestClient = ingestClient; + _logger = logger; + + _databaseName = configuration.Value.DatabaseName ?? throw new ArgumentNullException(nameof(AppConfiguration.DatabaseName)); + _tableName = configuration.Value.McpToolEventsTableName ?? throw new ArgumentNullException(nameof(AppConfiguration.McpToolEventsTableName)); + _queriesDirectory = configuration.Value.QueriesFolder == null + ? throw new ArgumentNullException(nameof(configuration.Value.QueriesFolder)) + : new DirectoryInfo(configuration.Value.QueriesFolder); + + if (!_queriesDirectory.Exists) + { + throw new ArgumentException($"'{_queriesDirectory.FullName}' does not exist. Value: {configuration.Value.QueriesFolder}"); + } + } + + public async Task> GetAvailableToolsAsync(CancellationToken cancellationToken = default) + { + var queryFile = _queriesDirectory.GetFiles("GetAvailableTools.kql").FirstOrDefault(); + + if (queryFile == null) + { + throw new InvalidOperationException($"Could not find GetAvailableTools.kql in {_queriesDirectory.FullName}"); + } + + var results = new List(); + + await foreach (var latestEvent in GetLatestToolEventsAsync(queryFile.FullName, cancellationToken)) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (string.IsNullOrEmpty(latestEvent.ToolId)) + { + throw new InvalidOperationException( + $"Cannot have an event with no id. Name: {latestEvent.ToolArea}, Area: {latestEvent.ToolArea}"); + } + + string? toolName; + string? toolArea; + switch (latestEvent.EventType) + { + case McpToolEventType.Created: + toolName = latestEvent.ToolName; + toolArea = latestEvent.ToolArea; + break; + case McpToolEventType.Updated: + toolName = latestEvent.ReplacedByToolName; + toolArea = latestEvent.ReplacedByToolArea; + break; + default: + throw new InvalidOperationException($"Tool '{latestEvent.ToolId}' has unsupported event type: {latestEvent.EventType}"); + } + + if (string.IsNullOrEmpty(toolName) || string.IsNullOrEmpty(toolArea)) + { + throw new InvalidOperationException($"Tool '{latestEvent.ToolId}' without tool name and/or a tool area."); + } + + results.Add(new AzureMcpTool(latestEvent.ToolId, toolName, toolArea)); + } + + return results; + } + + public async Task AddToolEventsAsync(IList toolEvents, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + using MemoryStream stream = new MemoryStream(); + + await JsonSerializer.SerializeAsync(stream, toolEvents, ModelsSerializationContext.Default.ListMcpToolEvent, cancellationToken); + stream.Seek(0, SeekOrigin.Begin); + + cancellationToken.ThrowIfCancellationRequested(); + + var ingestionProperties = new KustoIngestionProperties(_databaseName, _tableName) + { + Format = DataSourceFormat.json, + IngestionMapping = new IngestionMapping() + { + IngestionMappingKind = IngestionMappingKind.Json, + IngestionMappings = McpToolEvent.GetColumnMappings() + } + }; + + await _ingestClient.IngestFromStreamAsync(stream, ingestionProperties); + } + + internal async IAsyncEnumerable GetLatestToolEventsAsync(string kqlFilePath, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + if (!File.Exists(kqlFilePath)) + { + throw new FileNotFoundException($"KQL file not found: {kqlFilePath}"); + } + + var kql = await File.ReadAllTextAsync(kqlFilePath); + + var clientRequestProperties = new ClientRequestProperties(); + var reader = await _kustoClient.ExecuteQueryAsync(_databaseName, kql, clientRequestProperties, cancellationToken); + + var eventTimeOrdinal = reader.GetOrdinal("EventTime"); + var eventTypeOrdinal = reader.GetOrdinal("EventType"); + var serverVersionOrdinal = reader.GetOrdinal("ServerVersion"); + var toolIdOrdinal = reader.GetOrdinal("ToolId"); + var toolNameOrdinal = reader.GetOrdinal("ToolName"); + var toolAreaOrdinal = reader.GetOrdinal("ToolArea"); + var replacedByToolNameOrdinal = reader.GetOrdinal("ReplacedByToolName"); + var replacedByToolAreaOrdinal = reader.GetOrdinal("ReplacedByToolArea"); + + while (reader.Read()) + { + cancellationToken.ThrowIfCancellationRequested(); + + var eventTime = reader.GetDateTime(eventTimeOrdinal); + var eventTypeString = reader.GetString(eventTypeOrdinal); + var serverVersion = reader.GetString(serverVersionOrdinal); + var toolId = reader.GetString(toolIdOrdinal); + var toolName = reader.GetString(toolNameOrdinal); + var toolArea = reader.GetString(toolAreaOrdinal); + var replacedByToolName = reader.IsDBNull(replacedByToolNameOrdinal) + ? null + : reader.GetString(replacedByToolNameOrdinal); + var replacedByToolArea = reader.IsDBNull(replacedByToolAreaOrdinal) + ? null + : reader.GetString(replacedByToolAreaOrdinal); + + if (!Enum.TryParse(eventTypeString, ignoreCase: true, out var eventType)) + { + throw new InvalidOperationException($"Invalid EventType value: '{eventTypeString}'. EventTime: '{eventTime}', ToolName: '{toolName}', ToolArea: '{toolArea}'"); + } + + var tool = new McpToolEvent + { + EventTime = eventTime, + EventType = eventType, + ServerVersion = serverVersion, + ToolId = toolId, + ToolName = toolName, + ToolArea = toolArea, + ReplacedByToolName = replacedByToolName, + ReplacedByToolArea = replacedByToolArea, + }; + + yield return tool; + } + } +} diff --git a/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs b/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs new file mode 100644 index 0000000000..ace467be80 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using ToolMetadataExporter.Models; +using ToolMetadataExporter.Models.Kusto; + +namespace ToolMetadataExporter.Services; + +public interface IAzureMcpDatastore +{ + Task> GetAvailableToolsAsync(CancellationToken cancellationToken = default); + + Task AddToolEventsAsync(IList toolEvents, CancellationToken cancellationToken = default); +} diff --git a/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj b/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj new file mode 100644 index 0000000000..66f7146ca9 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj @@ -0,0 +1,32 @@ + + + + Exe + net9.0 + enable + enable + true + true + + + + + + + + + + + + + + + + + + + PreserveNewest + + + + diff --git a/eng/tools/ToolMetadataExporter/Utility.cs b/eng/tools/ToolMetadataExporter/Utility.cs new file mode 100644 index 0000000000..bc2acabd75 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Utility.cs @@ -0,0 +1,254 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using System.Text.Json; +using ToolSelection.Models; + +namespace ToolMetadataExporter; + +internal class Utility +{ + internal static async Task LoadToolsDynamicallyAsync(string workDirectory, bool isCiMode = false) + { + try + { + // Locate azmcp artifact across common build outputs (servers/core, Debug/Release) + var exeDir = AppContext.BaseDirectory; + var repoRoot = FindRepoRoot(exeDir); + var searchRoots = new List + { + Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Debug"), + Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Release") + }; + + var candidateNames = new[] { "azmcp.exe", "azmcp", "azmcp.dll" }; + FileInfo? cliArtifact = null; + + foreach (var root in searchRoots.Where(Directory.Exists)) + { + foreach (var name in candidateNames) + { + var found = new DirectoryInfo(root) + .EnumerateFiles(name, SearchOption.AllDirectories) + .FirstOrDefault(); + if (found != null) + { + cliArtifact = found; + break; + } + } + + if (cliArtifact != null) + { + break; + } + } + + if (cliArtifact == null) + { + if (isCiMode) + { + return null; // Graceful fallback in CI + } + + throw new FileNotFoundException("Could not locate azmcp CLI artifact in Debug/Release outputs under servers."); + } + + var isDll = string.Equals(cliArtifact.Extension, ".dll", StringComparison.OrdinalIgnoreCase); + var fileName = isDll ? "dotnet" : cliArtifact.FullName; + var arguments = isDll ? $"{cliArtifact.FullName} tools list" : "tools list"; + + var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + } + }; + + process.Start(); + + var output = await process.StandardOutput.ReadToEndAsync(); + var error = await process.StandardError.ReadToEndAsync(); + + await process.WaitForExitAsync(); + + if (process.ExitCode != 0) + { + if (isCiMode) + { + return null; // Graceful fallback in CI + } + + throw new InvalidOperationException($"Failed to get tools from azmcp: {error}"); + } + + // Filter out non-JSON lines (like launch settings messages) + var lines = output.Split('\n'); + var jsonStartIndex = -1; + + for (int i = 0; i < lines.Length; i++) + { + if (lines[i].Trim().StartsWith("{")) + { + jsonStartIndex = i; + + break; + } + } + + if (jsonStartIndex == -1) + { + if (isCiMode) + { + return null; // Graceful fallback in CI + } + + throw new InvalidOperationException("No JSON output found from azmcp command."); + } + + var jsonOutput = string.Join('\n', lines.Skip(jsonStartIndex)); + + // Parse the JSON output + var result = JsonSerializer.Deserialize(jsonOutput, SourceGenerationContext.Default.ListToolsResult); + + // Save the dynamically loaded tools to tools.json for future use + if (result != null) + { + await SaveToolsToJsonAsync(result, Path.Combine(workDirectory, "tools.json")); + + Console.WriteLine($"💾 Saved {result.Tools?.Count} tools to tools.json"); + } + + return result; + } + catch (Exception) + { + if (isCiMode) + { + return null; // Graceful fallback in CI + } + + throw; + } + } + + private static async Task LoadToolsFromJsonAsync(string filePath, bool isCiMode = false) + { + if (!File.Exists(filePath)) + { + if (isCiMode) + { + return null; // Let caller handle this gracefully + } + + throw new FileNotFoundException($"Tools file not found: {filePath}"); + } + + var json = await File.ReadAllTextAsync(filePath); + + // Process the JSON + if (json.StartsWith('\'') && json.EndsWith('\'')) + { + json = json[1..^1]; // Remove first and last characters (quotes) + json = json.Replace("\\'", "'"); // Convert \' --> ' + json = json.Replace("\\\\\"", "'"); // Convert \\" --> ' + } + + var result = JsonSerializer.Deserialize(json, SourceGenerationContext.Default.ListToolsResult); + + return result; + } + + private static async Task SaveToolsToJsonAsync(ListToolsResult toolsResult, string filePath) + { + try + { + // Normalize only tool and option descriptions instead of escaping the entire JSON document + if (toolsResult.Tools != null) + { + foreach (var tool in toolsResult.Tools) + { + if (!string.IsNullOrEmpty(tool.Description)) + { + tool.Description = EscapeCharacters(tool.Description); + } + + if (tool.Options != null) + { + foreach (var opt in tool.Options) + { + if (!string.IsNullOrEmpty(opt.Description)) + { + opt.Description = EscapeCharacters(opt.Description); + } + } + } + } + } + + var writerOptions = new JsonWriterOptions + { + Indented = true, + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + using var stream = new MemoryStream(); + using (var jsonWriter = new Utf8JsonWriter(stream, writerOptions)) + { + JsonSerializer.Serialize(jsonWriter, toolsResult, SourceGenerationContext.Default.ListToolsResult); + } + + await File.WriteAllBytesAsync(filePath, stream.ToArray()); + } + catch (Exception ex) + { + Console.WriteLine($"⚠️ Warning: Failed to save tools to {filePath}: {ex.Message}"); + Console.WriteLine(ex.StackTrace); + } + } + + private static string EscapeCharacters(string text) + { + if (string.IsNullOrEmpty(text)) + return text; + + // Normalize only the fancy “curly” quotes to straight ASCII. Identity replacements were removed. + return text.Replace(UnicodeChars.LeftSingleQuote, "'") + .Replace(UnicodeChars.RightSingleQuote, "'") + .Replace(UnicodeChars.LeftDoubleQuote, "\"") + .Replace(UnicodeChars.RightDoubleQuote, "\""); + } + + // Traverse up from a starting directory to find the repo root (containing AzureMcp.sln or .git) + internal static string FindRepoRoot(string startDir) + { + var dir = new DirectoryInfo(startDir); + + while (dir != null) + { + if (File.Exists(Path.Combine(dir.FullName, "AzureMcp.sln")) || + Directory.Exists(Path.Combine(dir.FullName, ".git"))) + { + return dir.FullName; + } + dir = dir.Parent; + } + + throw new InvalidOperationException("Could not find repo root (AzureMcp.sln or .git)."); + } + + internal static class UnicodeChars + { + public const string LeftSingleQuote = "\u2018"; + public const string RightSingleQuote = "\u2019"; + public const string LeftDoubleQuote = "\u201C"; + public const string RightDoubleQuote = "\u201D"; + } +} diff --git a/eng/tools/ToolMetadataExporter/appsettings.json b/eng/tools/ToolMetadataExporter/appsettings.json new file mode 100644 index 0000000000..00edbc5dff --- /dev/null +++ b/eng/tools/ToolMetadataExporter/appsettings.json @@ -0,0 +1,5 @@ +{ + "ClusterName": "McpClusterName", + "DatabaseName": "McpDatastore", + "McpToolEventsTableName": "McpToolEvents" +} From 5c9fd415bb8ea94d4d75dbff7468d98d6a241b2a Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 13:20:35 -0700 Subject: [PATCH 04/32] Adding tool to AzureMcp.sln --- AzureMcp.sln | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/AzureMcp.sln b/AzureMcp.sln index cbafdf58b7..a41c722a84 100644 --- a/AzureMcp.sln +++ b/AzureMcp.sln @@ -561,6 +561,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.Mcp.Tools.AzureAIBest Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{319B94CD-694C-16E8-9E3A-9577B99158DD}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.Mcp.Server.UnitTests", "servers\Azure.Mcp.Server\tests\Azure.Mcp.Server.UnitTests\Azure.Mcp.Server.UnitTests.csproj", "{ADF14627-FCB5-4BD3-B65F-DDCC3A3F727C}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolMetadataExporter", "eng\tools\ToolMetadataExporter\ToolMetadataExporter.csproj", "{66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -2132,6 +2133,18 @@ Global {ADF14627-FCB5-4BD3-B65F-DDCC3A3F727C}.Release|x64.Build.0 = Release|Any CPU {ADF14627-FCB5-4BD3-B65F-DDCC3A3F727C}.Release|x86.ActiveCfg = Release|Any CPU {ADF14627-FCB5-4BD3-B65F-DDCC3A3F727C}.Release|x86.Build.0 = Release|Any CPU + {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Debug|x64.ActiveCfg = Debug|Any CPU + {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Debug|x64.Build.0 = Debug|Any CPU + {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Debug|x86.ActiveCfg = Debug|Any CPU + {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Debug|x86.Build.0 = Debug|Any CPU + {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Release|Any CPU.Build.0 = Release|Any CPU + {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Release|x64.ActiveCfg = Release|Any CPU + {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Release|x64.Build.0 = Release|Any CPU + {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Release|x86.ActiveCfg = Release|Any CPU + {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -2413,6 +2426,7 @@ Global {BE8CFF4C-E536-43DB-9D01-001E9A052D37} = {50124EEC-97B0-320E-80D4-8464D7692B22} {319B94CD-694C-16E8-9E3A-9577B99158DD} = {F7E192D1-DE6C-42A2-B52F-02849D482450} {ADF14627-FCB5-4BD3-B65F-DDCC3A3F727C} = {319B94CD-694C-16E8-9E3A-9577B99158DD} + {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7} = {DAAE2FFB-70A9-DCEF-23A0-0ABAED0A9720} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {926577F9-9246-44E4-BCE9-25DB003F1C51} From 4af70a6bb70f9d763742ca74c1e65f7a5255e6fd Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 15:02:19 -0700 Subject: [PATCH 05/32] Remove AoT --- eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj b/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj index 66f7146ca9..ed5ce8d082 100644 --- a/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj +++ b/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj @@ -5,7 +5,6 @@ net9.0 enable enable - true true From 4e3adf790344c58eac09c2e6c07e82e726eef4a1 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 15:02:33 -0700 Subject: [PATCH 06/32] Add initial mock up --- .../ToolMetadataExporter/AppConfiguration.cs | 2 +- eng/tools/ToolMetadataExporter/Program.cs | 110 +++++------ .../Services/AzmcpProgram.cs | 5 + .../ToolMetadataExporter/ToolAnalyzer.cs | 172 ++++++++++++++++++ eng/tools/ToolMetadataExporter/Utility.cs | 164 +++++++++-------- .../ToolMetadataExporter/appsettings.json | 13 +- 6 files changed, 318 insertions(+), 148 deletions(-) create mode 100644 eng/tools/ToolMetadataExporter/ToolAnalyzer.cs diff --git a/eng/tools/ToolMetadataExporter/AppConfiguration.cs b/eng/tools/ToolMetadataExporter/AppConfiguration.cs index 3e9142a5d0..7f7d8fbd6c 100644 --- a/eng/tools/ToolMetadataExporter/AppConfiguration.cs +++ b/eng/tools/ToolMetadataExporter/AppConfiguration.cs @@ -5,7 +5,7 @@ namespace ToolMetadataExporter; public class AppConfiguration { - public string? ClusterName { get; set; } + public string? ClusterEndpoint { get; set; } public string? DatabaseName { get; set; } diff --git a/eng/tools/ToolMetadataExporter/Program.cs b/eng/tools/ToolMetadataExporter/Program.cs index 8a6261d112..8de4b203e5 100644 --- a/eng/tools/ToolMetadataExporter/Program.cs +++ b/eng/tools/ToolMetadataExporter/Program.cs @@ -1,48 +1,50 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Azure.Core; +using Azure.Identity; +using Kusto.Data; +using Kusto.Data.Common; +using Kusto.Data.Net.Client; +using Kusto.Ingest; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using ToolMetadataExporter.Models.Kusto; +using Microsoft.Extensions.Options; using ToolMetadataExporter.Services; namespace ToolMetadataExporter; public class Program { - private readonly AzmcpProgram _azmcpExe; - private readonly IAzureMcpDatastore _azureMcpDatastore; - private readonly ILogger _logger; - - public Program(AzmcpProgram program, IAzureMcpDatastore azureMcpDatastore, ILogger logger) - { - _azmcpExe = program; - _azureMcpDatastore = azureMcpDatastore; - _logger = logger; - } - public static async Task Main(string[] args) { var builder = Host.CreateApplicationBuilder(args); - ConfigureServices(builder.Services); + ConfigureServices(builder.Services, builder.Configuration); + ConfigureAzureServices(builder.Services); var host = builder.Build(); - var program = host.Services.GetRequiredService(); + var analyzer = host.Services.GetRequiredService(); - await program.RunAsync(DateTimeOffset.UtcNow, isDryRun: true); + await analyzer.RunAsync(DateTimeOffset.UtcNow, isDryRun: true); await host.RunAsync(); } - private static void ConfigureServices(IServiceCollection services) + private static void ConfigureServices(IServiceCollection services, IConfiguration configuration) { + services.AddLogging(builder => + { + builder.AddConsole(); + }); services.AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton(); - services.AddOptions() - .PostConfigure(existing => + services.Configure(configuration.GetSection("AppConfig")) + .PostConfigure(existing => { if (existing.WorkDirectory == null) { @@ -51,61 +53,33 @@ private static void ConfigureServices(IServiceCollection services) existing.WorkDirectory = Path.Combine(repoRoot, ".work"); } }); - - services.AddLogging(builder => - { - builder.AddConsole(); - }); } - private async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, CancellationToken cancellationToken = default) + private static void ConfigureAzureServices(IServiceCollection services) { - _logger.LogInformation("Starting analysis."); - - var currentTools = await _azmcpExe.LoadToolsDynamicallyAsync(); + services.AddScoped(sp => { + var credential = new ChainedTokenCredential( + new ManagedIdentityCredential(), + new DefaultAzureCredential() + ); - if (currentTools == null) - { - _logger.LogError("LoadToolsDynamicallyAsync did not return a result."); - return; - } - else if (currentTools.Tools == null || currentTools.Tools.Count == 0) - { - _logger.LogWarning("azmcp program did not return any tools."); - return; - } - - var existingTools = (await _azureMcpDatastore.GetAvailableToolsAsync(cancellationToken)).ToDictionary(x => x.ToolId); - - if (cancellationToken.IsCancellationRequested) - { - _logger.LogInformation("Analysis was cancelled."); - return; - } - - var changes = new List(); - - foreach (var tool in currentTools.Tools) + return credential; + }); + services.AddSingleton(sp => { - if (string.IsNullOrEmpty(tool.Id)) - { - throw new InvalidOperationException($"Tool without an id. Name: {tool.Name}. Command: {tool.Command}"); - } + var config = sp.GetRequiredService>(); + var credential = sp.GetRequiredService(); + var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.ClusterEndpoint) + .WithAadAzureTokenCredentialsAuthentication(credential); - if (!existingTools.TryGetValue(tool.Id, out var knownValue)) - { - - } - else - { - changes.Add(new McpToolEvent - { - EventTime = analysisTime, - EventType = McpToolEventType.Created, - }); - } - } + return KustoClientFactory.CreateCslQueryProvider(connectionStringBuilder); + }); + services.AddSingleton(sp => { + var config = sp.GetRequiredService>(); + var credential = sp.GetRequiredService(); + var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.ClusterEndpoint); - await _azureMcpDatastore.AddToolEventsAsync(changes, cancellationToken); + return KustoIngestFactory.CreateQueuedIngestClient(connectionStringBuilder); + }); } } diff --git a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs index f58b0558e2..6a645bb567 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs @@ -16,4 +16,9 @@ public AzmcpProgram(IOptions options) { return Utility.LoadToolsDynamicallyAsync(_toolDirectory, false); } + + public Task GetVersionAsync() + { + return Utility.GetVersionAsync(); + } } diff --git a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs new file mode 100644 index 0000000000..6e1b67a978 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs @@ -0,0 +1,172 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ToolMetadataExporter.Models; +using ToolMetadataExporter.Models.Kusto; +using ToolMetadataExporter.Services; +using ToolSelection.Models; + +namespace ToolMetadataExporter; + +public class ToolAnalyzer +{ + private readonly AzmcpProgram _azmcpExe; + private readonly IAzureMcpDatastore _azureMcpDatastore; + private readonly ILogger _logger; + private readonly string _workingDirectory; + + public ToolAnalyzer(AzmcpProgram program, IAzureMcpDatastore azureMcpDatastore, + IOptions configuration, ILogger logger) + { + _azmcpExe = program; + _azureMcpDatastore = azureMcpDatastore; + _logger = logger; + + _workingDirectory = configuration.Value.WorkDirectory ?? throw new ArgumentNullException(nameof(AppConfiguration.WorkDirectory)); + ; + } + + public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, CancellationToken cancellationToken = default) + { + _logger.LogInformation("Starting analysis."); + + var serverVersion = await _azmcpExe.GetVersionAsync(); + var currentTools = await _azmcpExe.LoadToolsDynamicallyAsync(); + + if (currentTools == null) + { + _logger.LogError("LoadToolsDynamicallyAsync did not return a result."); + return; + } + else if (currentTools.Tools == null || currentTools.Tools.Count == 0) + { + _logger.LogWarning("azmcp program did not return any tools."); + return; + } + + var existingTools = (await _azureMcpDatastore.GetAvailableToolsAsync(cancellationToken)).ToDictionary(x => x.ToolId); + + if (cancellationToken.IsCancellationRequested) + { + _logger.LogInformation("Analysis was cancelled."); + return; + } + + // Iterate through all the current tools and match them against the + // state Kusto knows about. + // For each tool, if there is no matching Tool, it is a new tool. + // Else, check the ToolName and ToolArea. If either of those are different + // then there is an update. + var changes = new List(); + + foreach (var tool in currentTools.Tools) + { + if (string.IsNullOrEmpty(tool.Id)) + { + throw new InvalidOperationException($"Tool without an id. Name: {tool.Name}. Command: {tool.Command}"); + } + + var toolArea = GetToolArea(tool); + if (string.IsNullOrEmpty(toolArea)) + { + throw new InvalidOperationException($"Tool without a tool area. Name: {tool.Name}. Id: {tool.Id}"); + } + + var changeEvent = new McpToolEvent + { + EventTime = analysisTime, + ToolId = tool.Id, + ServerVersion = serverVersion, + }; + + var hasChange = false; + if (existingTools.Remove(tool.Id, out var knownValue)) + { + if (!string.Equals(tool.Name, knownValue.ToolName, StringComparison.OrdinalIgnoreCase) + || !string.Equals(toolArea, knownValue.ToolArea, StringComparison.OrdinalIgnoreCase)) + { + hasChange = true; + + changeEvent.EventType = McpToolEventType.Updated; + changeEvent.ToolName = knownValue.ToolName; + changeEvent.ToolArea = knownValue.ToolArea; + changeEvent.ReplacedByToolName = tool.Name; + changeEvent.ReplacedByToolArea = toolArea; + } + } + else + { + hasChange = true; + changeEvent.EventType = McpToolEventType.Created; + changeEvent.ToolName = tool.Name; + changeEvent.ToolArea = toolArea; + } + + if (hasChange) + { + changes.Add(changeEvent); + } + } + + // We're done iterating through the newest available tools. + // Any remaining entries in `existingTool` are ones that got deleted. + var removals = existingTools.Select(x => new McpToolEvent + { + ServerVersion = serverVersion, + EventTime = analysisTime, + EventType = McpToolEventType.Deleted, + ToolId = x.Key, + ToolName = x.Value.ToolName, + ToolArea = x.Value.ToolArea, + ReplacedByToolName = null, + ReplacedByToolArea = null, + }); + + changes.AddRange(removals); + + cancellationToken.ThrowIfCancellationRequested(); + + if (!changes.Any()) + { + _logger.LogInformation("No changes made."); + return; + } + + var outputFile = Path.Combine(_workingDirectory, "tool_changes.json"); + + _logger.LogInformation("Tool updates. Writing output to : {FileName}", outputFile); + + + var writerOptions = new JsonWriterOptions + { + Indented = true, + }; + + using (var ms = new MemoryStream()) + using (var jsonWriter = new Utf8JsonWriter(ms, writerOptions)) + { + JsonSerializer.Serialize(jsonWriter, changes, ModelsSerializationContext.Default.ListMcpToolEvent); + + try + { + await File.WriteAllBytesAsync(outputFile, ms.ToArray()); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error writing to {FileName}", outputFile); + } + } + + if (!isDryRun) + { + _logger.LogInformation("Updating datastore."); + await _azureMcpDatastore.AddToolEventsAsync(changes, cancellationToken); + } + } + + private static string? GetToolArea(Tool tool) + { + var split = tool.Command?.Split(" ", 2); + return split == null ? null : split[0]; + } + +} diff --git a/eng/tools/ToolMetadataExporter/Utility.cs b/eng/tools/ToolMetadataExporter/Utility.cs index bc2acabd75..a7c027b0e5 100644 --- a/eng/tools/ToolMetadataExporter/Utility.cs +++ b/eng/tools/ToolMetadataExporter/Utility.cs @@ -13,82 +13,7 @@ internal class Utility { try { - // Locate azmcp artifact across common build outputs (servers/core, Debug/Release) - var exeDir = AppContext.BaseDirectory; - var repoRoot = FindRepoRoot(exeDir); - var searchRoots = new List - { - Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Debug"), - Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Release") - }; - - var candidateNames = new[] { "azmcp.exe", "azmcp", "azmcp.dll" }; - FileInfo? cliArtifact = null; - - foreach (var root in searchRoots.Where(Directory.Exists)) - { - foreach (var name in candidateNames) - { - var found = new DirectoryInfo(root) - .EnumerateFiles(name, SearchOption.AllDirectories) - .FirstOrDefault(); - if (found != null) - { - cliArtifact = found; - break; - } - } - - if (cliArtifact != null) - { - break; - } - } - - if (cliArtifact == null) - { - if (isCiMode) - { - return null; // Graceful fallback in CI - } - - throw new FileNotFoundException("Could not locate azmcp CLI artifact in Debug/Release outputs under servers."); - } - - var isDll = string.Equals(cliArtifact.Extension, ".dll", StringComparison.OrdinalIgnoreCase); - var fileName = isDll ? "dotnet" : cliArtifact.FullName; - var arguments = isDll ? $"{cliArtifact.FullName} tools list" : "tools list"; - - var process = new Process - { - StartInfo = new ProcessStartInfo - { - FileName = fileName, - Arguments = arguments, - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true - } - }; - - process.Start(); - - var output = await process.StandardOutput.ReadToEndAsync(); - var error = await process.StandardError.ReadToEndAsync(); - - await process.WaitForExitAsync(); - - if (process.ExitCode != 0) - { - if (isCiMode) - { - return null; // Graceful fallback in CI - } - - throw new InvalidOperationException($"Failed to get tools from azmcp: {error}"); - } - + var output = await ExecuteAzmcpAsync("tools list", isCiMode); // Filter out non-JSON lines (like launch settings messages) var lines = output.Split('\n'); var jsonStartIndex = -1; @@ -139,6 +64,92 @@ internal class Utility } } + internal static async Task GetVersionAsync() + { + return await ExecuteAzmcpAsync("version"); + } + + internal static async Task ExecuteAzmcpAsync(string arguments, bool isCiMode = false) + { + // Locate azmcp artifact across common build outputs (servers/core, Debug/Release) + var exeDir = AppContext.BaseDirectory; + var repoRoot = FindRepoRoot(exeDir); + var searchRoots = new List + { + Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Debug"), + Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Release") + }; + + var candidateNames = new[] { "azmcp.exe", "azmcp", "azmcp.dll" }; + FileInfo? cliArtifact = null; + + foreach (var root in searchRoots.Where(Directory.Exists)) + { + foreach (var name in candidateNames) + { + var found = new DirectoryInfo(root) + .EnumerateFiles(name, SearchOption.AllDirectories) + .FirstOrDefault(); + if (found != null) + { + cliArtifact = found; + break; + } + } + + if (cliArtifact != null) + { + break; + } + } + + if (cliArtifact == null) + { + if (isCiMode) + { + return string.Empty; // Graceful fallback in CI + } + + throw new FileNotFoundException("Could not locate azmcp CLI artifact in Debug/Release outputs under servers."); + } + + var isDll = string.Equals(cliArtifact.Extension, ".dll", StringComparison.OrdinalIgnoreCase); + var fileName = isDll ? "dotnet" : cliArtifact.FullName; + var argumentsToUse = isDll ? $"{cliArtifact.FullName} " : "tools list"; + + var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = fileName, + Arguments = argumentsToUse, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + } + }; + + process.Start(); + + var output = await process.StandardOutput.ReadToEndAsync(); + var error = await process.StandardError.ReadToEndAsync(); + + await process.WaitForExitAsync(); + + if (process.ExitCode != 0) + { + if (isCiMode) + { + return string.Empty; // Graceful fallback in CI + } + + throw new InvalidOperationException($"Failed to execute operation '{arguments}' from azmcp: {error}"); + } + + return output; + } + private static async Task LoadToolsFromJsonAsync(string filePath, bool isCiMode = false) { if (!File.Exists(filePath)) @@ -166,6 +177,7 @@ internal class Utility return result; } + private static async Task SaveToolsToJsonAsync(ListToolsResult toolsResult, string filePath) { try diff --git a/eng/tools/ToolMetadataExporter/appsettings.json b/eng/tools/ToolMetadataExporter/appsettings.json index 00edbc5dff..456c353a6b 100644 --- a/eng/tools/ToolMetadataExporter/appsettings.json +++ b/eng/tools/ToolMetadataExporter/appsettings.json @@ -1,5 +1,12 @@ { - "ClusterName": "McpClusterName", - "DatabaseName": "McpDatastore", - "McpToolEventsTableName": "McpToolEvents" + "Logging": { + "LogLevel": { + "Default": "Debug" + } + }, + "AppConfig": { + "ClusterName": "McpClusterName", + "DatabaseName": "McpDatastore", + "McpToolEventsTableName": "McpToolEvents" + } } From 0bbad1aa8e737555338409650087776c72f9737a Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 15:06:49 -0700 Subject: [PATCH 07/32] Split to use Query and Ingestion endpoint. --- eng/tools/ToolMetadataExporter/AppConfiguration.cs | 4 +++- eng/tools/ToolMetadataExporter/appsettings.json | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/AppConfiguration.cs b/eng/tools/ToolMetadataExporter/AppConfiguration.cs index 7f7d8fbd6c..881679e435 100644 --- a/eng/tools/ToolMetadataExporter/AppConfiguration.cs +++ b/eng/tools/ToolMetadataExporter/AppConfiguration.cs @@ -5,7 +5,9 @@ namespace ToolMetadataExporter; public class AppConfiguration { - public string? ClusterEndpoint { get; set; } + public string? IngestionEndpoint { get; set; } + + public string? QueryEndpoint { get; set; } public string? DatabaseName { get; set; } diff --git a/eng/tools/ToolMetadataExporter/appsettings.json b/eng/tools/ToolMetadataExporter/appsettings.json index 456c353a6b..fd78b76088 100644 --- a/eng/tools/ToolMetadataExporter/appsettings.json +++ b/eng/tools/ToolMetadataExporter/appsettings.json @@ -5,7 +5,8 @@ } }, "AppConfig": { - "ClusterName": "McpClusterName", + "IngestionEndpoint": "https://ingest-your-server.region.kusto.windows.net", + "QueryEndpoint": "https://your-server.region.kusto.windows.net", "DatabaseName": "McpDatastore", "McpToolEventsTableName": "McpToolEvents" } From fe07ce8e1a514ed240d9061b4f17a76d54e33ec2 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 15:17:30 -0700 Subject: [PATCH 08/32] Add Query to create table. --- .../Resources/queries/CreateTable.kql | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter/Resources/queries/CreateTable.kql diff --git a/eng/tools/ToolMetadataExporter/Resources/queries/CreateTable.kql b/eng/tools/ToolMetadataExporter/Resources/queries/CreateTable.kql new file mode 100644 index 0000000000..74a14c1c0d --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Resources/queries/CreateTable.kql @@ -0,0 +1,10 @@ +.create table McpToolEvents ( + EventTime: datetime, + EventType: string, + ServerVersion: string, + ToolId: string, + ToolName: string, + ToolArea: string, + ReplacedByToolName: string, + ReplacedByToolArea: string +) From 49e88c34b71f571abe0bf3ce593558054527dc6d Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 15:38:44 -0700 Subject: [PATCH 09/32] Add LaunchSettings. --- .../Properties/launchSettings.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter/Properties/launchSettings.json diff --git a/eng/tools/ToolMetadataExporter/Properties/launchSettings.json b/eng/tools/ToolMetadataExporter/Properties/launchSettings.json new file mode 100644 index 0000000000..7f48f76e44 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Properties/launchSettings.json @@ -0,0 +1,10 @@ +{ + "profiles": { + "ToolMetadataExporter": { + "commandName": "Project", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + } + } +} From 725dc59d70297e694b5045a5e9a3f459124c7a80 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 15:38:57 -0700 Subject: [PATCH 10/32] Set Default queries folder. --- eng/tools/ToolMetadataExporter/AppConfiguration.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/AppConfiguration.cs b/eng/tools/ToolMetadataExporter/AppConfiguration.cs index 881679e435..915802ae62 100644 --- a/eng/tools/ToolMetadataExporter/AppConfiguration.cs +++ b/eng/tools/ToolMetadataExporter/AppConfiguration.cs @@ -13,7 +13,7 @@ public class AppConfiguration public string? McpToolEventsTableName { get; set; } - public string? QueriesFolder { get; set; } + public string? QueriesFolder { get; set; } = "Resources/queries"; public string? WorkDirectory { get; set; } } From c9ba1c17fad47af9bdc569cdd25179d757c45ef5 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 28 Oct 2025 09:40:17 -0700 Subject: [PATCH 11/32] Adding Appsettings.json. Copy to output --- .../ToolAnalyzerTests.cs | 8 ++++++++ eng/tools/ToolMetadataExporter/Program.cs | 12 +++++++----- .../ToolMetadataExporter/ToolMetadataExporter.csproj | 6 ++++++ eng/tools/ToolMetadataExporter/appsettings.json | 5 ----- 4 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs diff --git a/eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs b/eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs new file mode 100644 index 0000000000..4882f1233f --- /dev/null +++ b/eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace ToolMetadataExporter.UnitTests; + +public class ToolAnalyzerTests +{ +} diff --git a/eng/tools/ToolMetadataExporter/Program.cs b/eng/tools/ToolMetadataExporter/Program.cs index 8de4b203e5..2ff963d969 100644 --- a/eng/tools/ToolMetadataExporter/Program.cs +++ b/eng/tools/ToolMetadataExporter/Program.cs @@ -21,6 +21,7 @@ public class Program public static async Task Main(string[] args) { var builder = Host.CreateApplicationBuilder(args); + ConfigureServices(builder.Services, builder.Configuration); ConfigureAzureServices(builder.Services); @@ -68,16 +69,17 @@ private static void ConfigureAzureServices(IServiceCollection services) services.AddSingleton(sp => { var config = sp.GetRequiredService>(); - var credential = sp.GetRequiredService(); - var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.ClusterEndpoint) - .WithAadAzureTokenCredentialsAuthentication(credential); + //var credential = sp.GetRequiredService(); + var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.QueryEndpoint) + .WithAadAzCliAuthentication(interactive: true); return KustoClientFactory.CreateCslQueryProvider(connectionStringBuilder); }); services.AddSingleton(sp => { var config = sp.GetRequiredService>(); - var credential = sp.GetRequiredService(); - var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.ClusterEndpoint); + //var credential = sp.GetRequiredService(); + var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.IngestionEndpoint) + .WithAadAzCliAuthentication(interactive: true); return KustoIngestFactory.CreateQueuedIngestClient(connectionStringBuilder); }); diff --git a/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj b/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj index ed5ce8d082..39d1ece9b1 100644 --- a/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj +++ b/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj @@ -23,6 +23,12 @@ + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/eng/tools/ToolMetadataExporter/appsettings.json b/eng/tools/ToolMetadataExporter/appsettings.json index fd78b76088..fde5467e3f 100644 --- a/eng/tools/ToolMetadataExporter/appsettings.json +++ b/eng/tools/ToolMetadataExporter/appsettings.json @@ -1,9 +1,4 @@ { - "Logging": { - "LogLevel": { - "Default": "Debug" - } - }, "AppConfig": { "IngestionEndpoint": "https://ingest-your-server.region.kusto.windows.net", "QueryEndpoint": "https://your-server.region.kusto.windows.net", From 7207b1ab7945c5cdebd600059ab23b63b3258c41 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 11:58:55 -0700 Subject: [PATCH 12/32] Update Json serialization to output Enum names. --- .../ToolMetadataExporter/Models/Kusto/McpToolEventType.cs | 5 +++++ .../Models/ModelsSerializationContext.cs | 1 + 2 files changed, 6 insertions(+) diff --git a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs index 3c595bbc1b..6bc1ea3c54 100644 --- a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs +++ b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs @@ -1,11 +1,16 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Text.Json.Serialization; + namespace ToolMetadataExporter.Models.Kusto; public enum McpToolEventType { + [JsonStringEnumMemberName("Created")] Created, + [JsonStringEnumMemberName("Updated")] Updated, + [JsonStringEnumMemberName("Deleted")] Deleted } diff --git a/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs b/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs index e2b4805f98..4b4facbd1b 100644 --- a/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs +++ b/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs @@ -9,6 +9,7 @@ namespace ToolMetadataExporter.Models; [JsonSerializable(typeof(McpToolEvent))] [JsonSerializable(typeof(McpToolEventType))] [JsonSerializable(typeof(List))] +[JsonSourceGenerationOptions(Converters = [ typeof(JsonStringEnumConverter )])] public partial class ModelsSerializationContext : JsonSerializerContext { } From b27c7866c16b4372712f84d2f54e027bb9634b6f Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 11:59:10 -0700 Subject: [PATCH 13/32] Start host --- eng/tools/ToolMetadataExporter/Program.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/Program.cs b/eng/tools/ToolMetadataExporter/Program.cs index 2ff963d969..1f1545a607 100644 --- a/eng/tools/ToolMetadataExporter/Program.cs +++ b/eng/tools/ToolMetadataExporter/Program.cs @@ -29,8 +29,9 @@ public static async Task Main(string[] args) var analyzer = host.Services.GetRequiredService(); + await host.StartAsync(); + await analyzer.RunAsync(DateTimeOffset.UtcNow, isDryRun: true); - await host.RunAsync(); } private static void ConfigureServices(IServiceCollection services, IConfiguration configuration) From ebd87336ec4dd42aa190ea56ff017b36ee473bf7 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 11:59:42 -0700 Subject: [PATCH 14/32] Move JsonOutput parsing. Fix command line arguments to pass to exe --- eng/tools/ToolMetadataExporter/Utility.cs | 55 ++++++++++++++--------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/Utility.cs b/eng/tools/ToolMetadataExporter/Utility.cs index a7c027b0e5..ff67dda914 100644 --- a/eng/tools/ToolMetadataExporter/Utility.cs +++ b/eng/tools/ToolMetadataExporter/Utility.cs @@ -14,21 +14,9 @@ internal class Utility try { var output = await ExecuteAzmcpAsync("tools list", isCiMode); - // Filter out non-JSON lines (like launch settings messages) - var lines = output.Split('\n'); - var jsonStartIndex = -1; + var jsonOutput = GetJsonFromOutput(output); - for (int i = 0; i < lines.Length; i++) - { - if (lines[i].Trim().StartsWith("{")) - { - jsonStartIndex = i; - - break; - } - } - - if (jsonStartIndex == -1) + if (jsonOutput == null) { if (isCiMode) { @@ -38,8 +26,6 @@ internal class Utility throw new InvalidOperationException("No JSON output found from azmcp command."); } - var jsonOutput = string.Join('\n', lines.Skip(jsonStartIndex)); - // Parse the JSON output var result = JsonSerializer.Deserialize(jsonOutput, SourceGenerationContext.Default.ListToolsResult); @@ -66,10 +52,11 @@ internal class Utility internal static async Task GetVersionAsync() { - return await ExecuteAzmcpAsync("version"); + var output = await ExecuteAzmcpAsync("--version", checkErrorCode: false); + return output.Trim(); } - internal static async Task ExecuteAzmcpAsync(string arguments, bool isCiMode = false) + internal static async Task ExecuteAzmcpAsync(string arguments, bool isCiMode = false, bool checkErrorCode = true) { // Locate azmcp artifact across common build outputs (servers/core, Debug/Release) var exeDir = AppContext.BaseDirectory; @@ -115,7 +102,7 @@ internal static async Task ExecuteAzmcpAsync(string arguments, bool isCi var isDll = string.Equals(cliArtifact.Extension, ".dll", StringComparison.OrdinalIgnoreCase); var fileName = isDll ? "dotnet" : cliArtifact.FullName; - var argumentsToUse = isDll ? $"{cliArtifact.FullName} " : "tools list"; + var argumentsToUse = isDll ? $"{cliArtifact.FullName} " : arguments; var process = new Process { @@ -137,7 +124,7 @@ internal static async Task ExecuteAzmcpAsync(string arguments, bool isCi await process.WaitForExitAsync(); - if (process.ExitCode != 0) + if (checkErrorCode && process.ExitCode != 0) { if (isCiMode) { @@ -177,6 +164,34 @@ internal static async Task ExecuteAzmcpAsync(string arguments, bool isCi return result; } + private static string? GetJsonFromOutput(string? output) + { + if (output == null) + { + return null; + } + + // Filter out non-JSON lines (like launch settings messages) + var lines = output.Split('\n'); + var jsonStartIndex = -1; + + for (int i = 0; i < lines.Length; i++) + { + if (lines[i].Trim().StartsWith("{")) + { + jsonStartIndex = i; + + break; + } + } + + if (jsonStartIndex == -1) + { + return null; + } + + return string.Join('\n', lines.Skip(jsonStartIndex)); + } private static async Task SaveToolsToJsonAsync(ListToolsResult toolsResult, string filePath) { From ec1e1490228d7733a3b385c331aab4a87174c8ce Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 12:00:05 -0700 Subject: [PATCH 15/32] Pass in command rather than just name in changes. --- eng/tools/ToolMetadataExporter/ToolAnalyzer.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs index 6e1b67a978..57d95a1674 100644 --- a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs +++ b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs @@ -9,6 +9,8 @@ namespace ToolMetadataExporter; public class ToolAnalyzer { + private const string Separator = "_"; + private readonly AzmcpProgram _azmcpExe; private readonly IAzureMcpDatastore _azureMcpDatastore; private readonly ILogger _logger; @@ -68,7 +70,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat var toolArea = GetToolArea(tool); if (string.IsNullOrEmpty(toolArea)) { - throw new InvalidOperationException($"Tool without a tool area. Name: {tool.Name}. Id: {tool.Id}"); + throw new InvalidOperationException($"Tool without a tool area. Name: {tool.Name}. Command: {tool.Command} Id: {tool.Id}"); } var changeEvent = new McpToolEvent @@ -78,10 +80,12 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat ServerVersion = serverVersion, }; + var commandWithSeparator = tool.Command?.Replace(" ", Separator); + var hasChange = false; if (existingTools.Remove(tool.Id, out var knownValue)) { - if (!string.Equals(tool.Name, knownValue.ToolName, StringComparison.OrdinalIgnoreCase) + if (!string.Equals(commandWithSeparator, knownValue.ToolName, StringComparison.OrdinalIgnoreCase) || !string.Equals(toolArea, knownValue.ToolArea, StringComparison.OrdinalIgnoreCase)) { hasChange = true; @@ -89,7 +93,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat changeEvent.EventType = McpToolEventType.Updated; changeEvent.ToolName = knownValue.ToolName; changeEvent.ToolArea = knownValue.ToolArea; - changeEvent.ReplacedByToolName = tool.Name; + changeEvent.ReplacedByToolName = commandWithSeparator; changeEvent.ReplacedByToolArea = toolArea; } } @@ -97,7 +101,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat { hasChange = true; changeEvent.EventType = McpToolEventType.Created; - changeEvent.ToolName = tool.Name; + changeEvent.ToolName = commandWithSeparator; changeEvent.ToolArea = toolArea; } From 901cb277b8704790356526279d6763675802adee Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 13:54:20 -0700 Subject: [PATCH 16/32] Update Logging --- eng/tools/ToolMetadataExporter/ToolAnalyzer.cs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs index 57d95a1674..c96503d913 100644 --- a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs +++ b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs @@ -29,7 +29,7 @@ public ToolAnalyzer(AzmcpProgram program, IAzureMcpDatastore azureMcpDatastore, public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, CancellationToken cancellationToken = default) { - _logger.LogInformation("Starting analysis."); + _logger.LogInformation("Starting analysis. IsDryRun: {IsDryRun}", isDryRun); var serverVersion = await _azmcpExe.GetVersionAsync(); var currentTools = await _azmcpExe.LoadToolsDynamicallyAsync(); @@ -67,7 +67,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat throw new InvalidOperationException($"Tool without an id. Name: {tool.Name}. Command: {tool.Command}"); } - var toolArea = GetToolArea(tool); + var toolArea = GetToolArea(tool)?.ToLowerInvariant(); if (string.IsNullOrEmpty(toolArea)) { throw new InvalidOperationException($"Tool without a tool area. Name: {tool.Name}. Command: {tool.Command} Id: {tool.Id}"); @@ -80,7 +80,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat ServerVersion = serverVersion, }; - var commandWithSeparator = tool.Command?.Replace(" ", Separator); + var commandWithSeparator = tool.Command?.Replace(" ", Separator).ToLowerInvariant(); var hasChange = false; if (existingTools.Remove(tool.Id, out var knownValue)) @@ -137,8 +137,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat var outputFile = Path.Combine(_workingDirectory, "tool_changes.json"); - _logger.LogInformation("Tool updates. Writing output to : {FileName}", outputFile); - + _logger.LogInformation("Tool updates. Writing output to: {FileName}", outputFile); var writerOptions = new JsonWriterOptions { @@ -152,7 +151,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat try { - await File.WriteAllBytesAsync(outputFile, ms.ToArray()); + await File.WriteAllBytesAsync(outputFile, ms.ToArray(), cancellationToken); } catch (Exception ex) { @@ -172,5 +171,4 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat var split = tool.Command?.Split(" ", 2); return split == null ? null : split[0]; } - } From 238c33b96c5b0b3e0cae5a09d5a94b41df708526 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 13:57:01 -0700 Subject: [PATCH 17/32] Change interface to use List. Update Ingestion settings for singlejson --- .../Services/AzureMcpKustoDatastore.cs | 23 ++++++++++++++++--- .../Services/IAzureMcpDatastore.cs | 2 +- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs index beb756234a..258e4010a0 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs @@ -91,7 +91,7 @@ public async Task> GetAvailableToolsAsync(CancellationToken return results; } - public async Task AddToolEventsAsync(IList toolEvents, CancellationToken cancellationToken = default) + public async Task AddToolEventsAsync(List toolEvents, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -104,7 +104,7 @@ public async Task AddToolEventsAsync(IList toolEvents, Cancellatio var ingestionProperties = new KustoIngestionProperties(_databaseName, _tableName) { - Format = DataSourceFormat.json, + Format = DataSourceFormat.singlejson, IngestionMapping = new IngestionMapping() { IngestionMappingKind = IngestionMappingKind.Json, @@ -112,7 +112,24 @@ public async Task AddToolEventsAsync(IList toolEvents, Cancellatio } }; - await _ingestClient.IngestFromStreamAsync(stream, ingestionProperties); + var result = await _ingestClient.IngestFromStreamAsync(stream, ingestionProperties); + + if (result != null) + { + _logger.LogInformation("Ingestion results."); + foreach(var item in result.GetIngestionStatusCollection()) + { + _logger.LogInformation("- {IngestionSourceId}\t{Table}\t{Status}\t{Details}", + item.IngestionSourceId, + item.Table, + item.Status, + item.Details); + } + } + else + { + _logger.LogWarning("Ingestion client did not produce any results."); + } } internal async IAsyncEnumerable GetLatestToolEventsAsync(string kqlFilePath, diff --git a/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs b/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs index ace467be80..6a20206c51 100644 --- a/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs +++ b/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs @@ -10,5 +10,5 @@ public interface IAzureMcpDatastore { Task> GetAvailableToolsAsync(CancellationToken cancellationToken = default); - Task AddToolEventsAsync(IList toolEvents, CancellationToken cancellationToken = default); + Task AddToolEventsAsync(List toolEvents, CancellationToken cancellationToken = default); } From 46e1d4f6807afce28075d816aad1a07647e8a312 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 13:57:22 -0700 Subject: [PATCH 18/32] Fix column mappings --- .../ToolMetadataExporter/Models/Kusto/McpToolEvent.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs index 0adc12a18e..fbbe3b90b1 100644 --- a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs +++ b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs @@ -44,13 +44,14 @@ public class McpToolEvent public static ColumnMapping[] GetColumnMappings() { return [ - new ColumnMapping { ColumnName = EventTypeColumn, ColumnType = "datetime" }, + new ColumnMapping { ColumnName = EventTimeColumn, ColumnType = "datetime" }, + new ColumnMapping { ColumnName = EventTypeColumn, ColumnType = "string"}, + new ColumnMapping { ColumnName = ReplacedByToolAreaColumn, ColumnType = "string"}, + new ColumnMapping { ColumnName = ReplacedByToolNameColumn, ColumnType = "string"}, new ColumnMapping { ColumnName = ServerVersionColumn, ColumnType = "string" }, + new ColumnMapping { ColumnName = ToolAreaColumn , ColumnType = "string" }, new ColumnMapping { ColumnName = ToolIdColumn, ColumnType = "string"}, new ColumnMapping { ColumnName = ToolNameColumn, ColumnType = "string" }, - new ColumnMapping { ColumnName = ToolAreaColumn , ColumnType = "string" }, - new ColumnMapping { ColumnName = ReplacedByToolNameColumn, ColumnType = "string"}, - new ColumnMapping { ColumnName = ReplacedByToolAreaColumn, ColumnType = "string"}, ]; } } From 5bb8bcc01d3591bd96626e6b0ecdfce2fb2d526d Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 13:58:36 -0700 Subject: [PATCH 19/32] Change to use Direct Ingestion client --- eng/tools/ToolMetadataExporter/Program.cs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/Program.cs b/eng/tools/ToolMetadataExporter/Program.cs index 1f1545a607..190adfa557 100644 --- a/eng/tools/ToolMetadataExporter/Program.cs +++ b/eng/tools/ToolMetadataExporter/Program.cs @@ -31,7 +31,14 @@ public static async Task Main(string[] args) await host.StartAsync(); - await analyzer.RunAsync(DateTimeOffset.UtcNow, isDryRun: true); + var isDryRunValue = builder.Configuration["IsDryRun"]; + var isDryRun = false; + if (bool.TryParse(isDryRunValue, out var parsed)) + { + isDryRun = parsed; + } + + await analyzer.RunAsync(DateTimeOffset.UtcNow, isDryRun); } private static void ConfigureServices(IServiceCollection services, IConfiguration configuration) @@ -70,19 +77,21 @@ private static void ConfigureAzureServices(IServiceCollection services) services.AddSingleton(sp => { var config = sp.GetRequiredService>(); - //var credential = sp.GetRequiredService(); + var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.QueryEndpoint) + .WithAadUserPromptAuthentication() .WithAadAzCliAuthentication(interactive: true); return KustoClientFactory.CreateCslQueryProvider(connectionStringBuilder); }); services.AddSingleton(sp => { var config = sp.GetRequiredService>(); - //var credential = sp.GetRequiredService(); + var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.IngestionEndpoint) - .WithAadAzCliAuthentication(interactive: true); + .WithAadUserPromptAuthentication() + .WithAadAzCliAuthentication(interactive: true); - return KustoIngestFactory.CreateQueuedIngestClient(connectionStringBuilder); + return KustoIngestFactory.CreateDirectIngestClient(connectionStringBuilder); }); } } From 08dd0f56feb7b949973794dd47665d45db6f48d3 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 19:59:18 -0700 Subject: [PATCH 20/32] Fix formatting issues --- eng/tools/ToolDescriptionEvaluator/Models/McpModels.cs | 4 ++-- eng/tools/ToolMetadataExporter/Program.cs | 6 ++++-- .../ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/eng/tools/ToolDescriptionEvaluator/Models/McpModels.cs b/eng/tools/ToolDescriptionEvaluator/Models/McpModels.cs index b4a68bb118..781922e64e 100644 --- a/eng/tools/ToolDescriptionEvaluator/Models/McpModels.cs +++ b/eng/tools/ToolDescriptionEvaluator/Models/McpModels.cs @@ -81,8 +81,8 @@ public class ToolAnnotations } // Tool definition for azmcp tools list response -public class Tool { - +public class Tool +{ [JsonPropertyName("id")] public string? Id { get; set; } diff --git a/eng/tools/ToolMetadataExporter/Program.cs b/eng/tools/ToolMetadataExporter/Program.cs index 190adfa557..01151c9e5a 100644 --- a/eng/tools/ToolMetadataExporter/Program.cs +++ b/eng/tools/ToolMetadataExporter/Program.cs @@ -66,7 +66,8 @@ private static void ConfigureServices(IServiceCollection services, IConfiguratio private static void ConfigureAzureServices(IServiceCollection services) { - services.AddScoped(sp => { + services.AddScoped(sp => + { var credential = new ChainedTokenCredential( new ManagedIdentityCredential(), new DefaultAzureCredential() @@ -84,7 +85,8 @@ private static void ConfigureAzureServices(IServiceCollection services) return KustoClientFactory.CreateCslQueryProvider(connectionStringBuilder); }); - services.AddSingleton(sp => { + services.AddSingleton(sp => + { var config = sp.GetRequiredService>(); var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.IngestionEndpoint) diff --git a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs index 258e4010a0..0e19a151f7 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs @@ -117,7 +117,7 @@ public async Task AddToolEventsAsync(List toolEvents, Cancellation if (result != null) { _logger.LogInformation("Ingestion results."); - foreach(var item in result.GetIngestionStatusCollection()) + foreach (var item in result.GetIngestionStatusCollection()) { _logger.LogInformation("- {IngestionSourceId}\t{Table}\t{Status}\t{Details}", item.IngestionSourceId, From 12fd8b5b7cd55705d63bbd69520c62c396a57380 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 20:50:13 -0700 Subject: [PATCH 21/32] Fix formatting --- .../ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs index 0e19a151f7..45fbec2f0e 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs @@ -119,7 +119,7 @@ public async Task AddToolEventsAsync(List toolEvents, Cancellation _logger.LogInformation("Ingestion results."); foreach (var item in result.GetIngestionStatusCollection()) { - _logger.LogInformation("- {IngestionSourceId}\t{Table}\t{Status}\t{Details}", + _logger.LogInformation("Id: {IngestionSourceId}\tTable: {Table}\tStatus: {Status}\tDetails: {Details}", item.IngestionSourceId, item.Table, item.Status, From 2d0f10bc0007631305c2c8c35056000914b162d7 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 20:50:23 -0700 Subject: [PATCH 22/32] Add Friend assembly --- eng/tools/ToolMetadataExporter/AssemblyInfo.cs | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter/AssemblyInfo.cs diff --git a/eng/tools/ToolMetadataExporter/AssemblyInfo.cs b/eng/tools/ToolMetadataExporter/AssemblyInfo.cs new file mode 100644 index 0000000000..e0e0f77159 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/AssemblyInfo.cs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("ToolMetadataExporter.UnitTests")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] From dc326cec0c1e772f9a8937a10732e3b3911a8dbd Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 20:50:33 -0700 Subject: [PATCH 23/32] Add unit test project --- AzureMcp.sln | 15 +++++++++++++++ .../ToolMetadataExporter.UnitTests.csproj | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj diff --git a/AzureMcp.sln b/AzureMcp.sln index a41c722a84..4993c0c6a7 100644 --- a/AzureMcp.sln +++ b/AzureMcp.sln @@ -563,6 +563,8 @@ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.Mcp.Server.UnitTests", "servers\Azure.Mcp.Server\tests\Azure.Mcp.Server.UnitTests\Azure.Mcp.Server.UnitTests.csproj", "{ADF14627-FCB5-4BD3-B65F-DDCC3A3F727C}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolMetadataExporter", "eng\tools\ToolMetadataExporter\ToolMetadataExporter.csproj", "{66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolMetadataExporter.UnitTests", "eng\tools\ToolMetadataExporter.UnitTests\ToolMetadataExporter.UnitTests.csproj", "{C1AEE42D-62AD-DF8D-1459-5B9FBE6BEDEA}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -2145,6 +2147,18 @@ Global {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Release|x64.Build.0 = Release|Any CPU {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Release|x86.ActiveCfg = Release|Any CPU {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7}.Release|x86.Build.0 = Release|Any CPU + {C1AEE42D-62AD-DF8D-1459-5B9FBE6BEDEA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C1AEE42D-62AD-DF8D-1459-5B9FBE6BEDEA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C1AEE42D-62AD-DF8D-1459-5B9FBE6BEDEA}.Debug|x64.ActiveCfg = Debug|Any CPU + {C1AEE42D-62AD-DF8D-1459-5B9FBE6BEDEA}.Debug|x64.Build.0 = Debug|Any CPU + {C1AEE42D-62AD-DF8D-1459-5B9FBE6BEDEA}.Debug|x86.ActiveCfg = Debug|Any CPU + {C1AEE42D-62AD-DF8D-1459-5B9FBE6BEDEA}.Debug|x86.Build.0 = Debug|Any CPU + {C1AEE42D-62AD-DF8D-1459-5B9FBE6BEDEA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C1AEE42D-62AD-DF8D-1459-5B9FBE6BEDEA}.Release|Any CPU.Build.0 = Release|Any CPU + {C1AEE42D-62AD-DF8D-1459-5B9FBE6BEDEA}.Release|x64.ActiveCfg = Release|Any CPU + {C1AEE42D-62AD-DF8D-1459-5B9FBE6BEDEA}.Release|x64.Build.0 = Release|Any CPU + {C1AEE42D-62AD-DF8D-1459-5B9FBE6BEDEA}.Release|x86.ActiveCfg = Release|Any CPU + {C1AEE42D-62AD-DF8D-1459-5B9FBE6BEDEA}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -2427,6 +2441,7 @@ Global {319B94CD-694C-16E8-9E3A-9577B99158DD} = {F7E192D1-DE6C-42A2-B52F-02849D482450} {ADF14627-FCB5-4BD3-B65F-DDCC3A3F727C} = {319B94CD-694C-16E8-9E3A-9577B99158DD} {66A2052D-99BD-4F5E-B031-F69E3E7FEDC7} = {DAAE2FFB-70A9-DCEF-23A0-0ABAED0A9720} + {C1AEE42D-62AD-DF8D-1459-5B9FBE6BEDEA} = {DAAE2FFB-70A9-DCEF-23A0-0ABAED0A9720} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {926577F9-9246-44E4-BCE9-25DB003F1C51} diff --git a/eng/tools/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj b/eng/tools/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj new file mode 100644 index 0000000000..08c835a03d --- /dev/null +++ b/eng/tools/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj @@ -0,0 +1,16 @@ + + + true + Exe + + + + + + + + + + + + From 792c53ef0fcc154dd22767a740379ca2c7b820c6 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 21:00:12 -0700 Subject: [PATCH 24/32] Make method virtual --- eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs index 6a645bb567..c4491ece4c 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs @@ -17,7 +17,7 @@ public AzmcpProgram(IOptions options) return Utility.LoadToolsDynamicallyAsync(_toolDirectory, false); } - public Task GetVersionAsync() + public virtual Task GetVersionAsync() { return Utility.GetVersionAsync(); } From e13fd7e8b3f98fde558d2383a2a36b20779a2bf2 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 28 Oct 2025 02:39:27 -0700 Subject: [PATCH 25/32] Fix formatting issue. --- .../ToolMetadataExporter/Models/ModelsSerializationContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs b/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs index 4b4facbd1b..e4ccddbb36 100644 --- a/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs +++ b/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs @@ -9,7 +9,7 @@ namespace ToolMetadataExporter.Models; [JsonSerializable(typeof(McpToolEvent))] [JsonSerializable(typeof(McpToolEventType))] [JsonSerializable(typeof(List))] -[JsonSourceGenerationOptions(Converters = [ typeof(JsonStringEnumConverter )])] +[JsonSourceGenerationOptions(Converters = [typeof(JsonStringEnumConverter)])] public partial class ModelsSerializationContext : JsonSerializerContext { } From bdd8fb5b642346a773f559f4afb362dbedb1fcbc Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 28 Oct 2025 02:39:42 -0700 Subject: [PATCH 26/32] Add header --- eng/tools/ToolMetadataExporter/ToolAnalyzer.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs index c96503d913..22c7764347 100644 --- a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs +++ b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs @@ -1,4 +1,7 @@ -using Microsoft.Extensions.Logging; +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ToolMetadataExporter.Models; using ToolMetadataExporter.Models.Kusto; From a454034e68c3b77f37a7b74653c8d4761650269d Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 28 Oct 2025 09:41:57 -0700 Subject: [PATCH 27/32] Add dev settings --- .../appsettings.Development.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter/appsettings.Development.json diff --git a/eng/tools/ToolMetadataExporter/appsettings.Development.json b/eng/tools/ToolMetadataExporter/appsettings.Development.json new file mode 100644 index 0000000000..0203ba7a47 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/appsettings.Development.json @@ -0,0 +1,12 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Debug" + } + }, + "AppConfig": { + "IngestionEndpoint": "https://ingest-test.westus.kusto.windows.net", + "QueryEndpoint": "https://test.westus.kusto.windows.net", + "DatabaseName": "McpToolTest" + } +} From be54a7f17777875bd3704e34eb189e0014805f26 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Wed, 29 Oct 2025 01:51:55 -0700 Subject: [PATCH 28/32] Add header --- eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs b/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs index 60cc77c698..26dfecd3f6 100644 --- a/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs +++ b/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs @@ -1,4 +1,7 @@ -namespace ToolMetadataExporter.Models; +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace ToolMetadataExporter.Models; public record AzureMcpTool( string ToolId, From 901c5a9b01c20eed1b7c949a7db56fc104f7f69f Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 11 Nov 2025 11:28:00 -0800 Subject: [PATCH 29/32] Add ServerName support. --- .../ToolMetadataExporter/Models/Kusto/McpToolEvent.cs | 4 ++++ eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs | 5 +++++ eng/tools/ToolMetadataExporter/ToolAnalyzer.cs | 3 +++ eng/tools/ToolMetadataExporter/Utility.cs | 7 ++++++- 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs index fbbe3b90b1..c034594429 100644 --- a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs +++ b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs @@ -10,6 +10,7 @@ public class McpToolEvent { private const string EventTimeColumn = "EventTime"; private const string EventTypeColumn = "EventType"; + private const string ServerNameColumn = "ServerName"; private const string ServerVersionColumn = "ServerVersion"; private const string ToolIdColumn = "ToolId"; private const string ToolNameColumn = "ToolName"; @@ -23,6 +24,9 @@ public class McpToolEvent [JsonPropertyName(EventTypeColumn)] public McpToolEventType? EventType { get; set; } + [JsonPropertyName(ServerNameColumn)] + public string? ServerName { get; set; } + [JsonPropertyName(ServerVersionColumn)] public string? ServerVersion { get; set; } diff --git a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs index c4491ece4c..8a0d433766 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs @@ -7,6 +7,11 @@ public class AzmcpProgram { private readonly string _toolDirectory; + public Task GetServerNameAsync() + { + return Task.FromResult(""); + } + public AzmcpProgram(IOptions options) { _toolDirectory = options.Value.WorkDirectory ?? throw new ArgumentNullException(nameof(AppConfiguration.WorkDirectory)); diff --git a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs index 22c7764347..101fbd5f08 100644 --- a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs +++ b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs @@ -34,6 +34,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat { _logger.LogInformation("Starting analysis. IsDryRun: {IsDryRun}", isDryRun); + var serverName = await _azmcpExe.GetServerNameAsync(); var serverVersion = await _azmcpExe.GetVersionAsync(); var currentTools = await _azmcpExe.LoadToolsDynamicallyAsync(); @@ -80,6 +81,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat { EventTime = analysisTime, ToolId = tool.Id, + ServerName = serverName, ServerVersion = serverVersion, }; @@ -118,6 +120,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat // Any remaining entries in `existingTool` are ones that got deleted. var removals = existingTools.Select(x => new McpToolEvent { + ServerName = serverName, ServerVersion = serverVersion, EventTime = analysisTime, EventType = McpToolEventType.Deleted, diff --git a/eng/tools/ToolMetadataExporter/Utility.cs b/eng/tools/ToolMetadataExporter/Utility.cs index ff67dda914..02021da318 100644 --- a/eng/tools/ToolMetadataExporter/Utility.cs +++ b/eng/tools/ToolMetadataExporter/Utility.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. using System.Diagnostics; -using System.Text.Json; using ToolSelection.Models; namespace ToolMetadataExporter; @@ -50,6 +49,12 @@ internal class Utility } } + internal static async Task GetServerName() + { + var output = await ExecuteAzmcpAsync("", checkErrorCode: false); + return output.Trim(); + } + internal static async Task GetVersionAsync() { var output = await ExecuteAzmcpAsync("--version", checkErrorCode: false); From bf8aee9ea108296fa82d5a5f3a4b64c4251e0fc9 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 11 Nov 2025 11:28:22 -0800 Subject: [PATCH 30/32] Remove test template. --- .../ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs diff --git a/eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs b/eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs deleted file mode 100644 index 4882f1233f..0000000000 --- a/eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace ToolMetadataExporter.UnitTests; - -public class ToolAnalyzerTests -{ -} From d244e0e7dfce9b7d9ce979f2deac76a33f265389 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 11 Nov 2025 11:32:47 -0800 Subject: [PATCH 31/32] Add ServerName --- .../Services/AzmcpProgram.cs | 2 +- eng/tools/ToolMetadataExporter/Utility.cs | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs index 8a0d433766..19585d61e9 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs @@ -9,7 +9,7 @@ public class AzmcpProgram public Task GetServerNameAsync() { - return Task.FromResult(""); + return Utility.GetServerName(); } public AzmcpProgram(IOptions options) diff --git a/eng/tools/ToolMetadataExporter/Utility.cs b/eng/tools/ToolMetadataExporter/Utility.cs index 02021da318..f09c63c6d1 100644 --- a/eng/tools/ToolMetadataExporter/Utility.cs +++ b/eng/tools/ToolMetadataExporter/Utility.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System.Diagnostics; +using System.Text.RegularExpressions; using ToolSelection.Models; namespace ToolMetadataExporter; @@ -51,8 +52,19 @@ internal class Utility internal static async Task GetServerName() { - var output = await ExecuteAzmcpAsync("", checkErrorCode: false); - return output.Trim(); + var output = await ExecuteAzmcpAsync("--help", checkErrorCode: false); + + string[] array = Regex.Split(output, "\n\r"); + for (int i = 0; i < array.Length; i++) + { + string? line = array[i]; + if (line.StartsWith("Description:")) + { + return array[i + 1].Trim(); + } + } + + throw new InvalidOperationException("Could not find server name"); } internal static async Task GetVersionAsync() From bbded7417f74f1ca86b628711035a3ff369f5622 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 11 Nov 2025 11:35:46 -0800 Subject: [PATCH 32/32] Propagate cancellation --- .../ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs index 45fbec2f0e..5cb7cf42a5 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs @@ -140,7 +140,7 @@ internal async IAsyncEnumerable GetLatestToolEventsAsync(string kq throw new FileNotFoundException($"KQL file not found: {kqlFilePath}"); } - var kql = await File.ReadAllTextAsync(kqlFilePath); + var kql = await File.ReadAllTextAsync(kqlFilePath, cancellationToken); var clientRequestProperties = new ClientRequestProperties(); var reader = await _kustoClient.ExecuteQueryAsync(_databaseName, kql, clientRequestProperties, cancellationToken);