diff --git a/docs/bug-bash/scenarios/deployment.md b/docs/bug-bash/scenarios/deployment.md index fe3c632ee9..c727a0cf47 100644 --- a/docs/bug-bash/scenarios/deployment.md +++ b/docs/bug-bash/scenarios/deployment.md @@ -397,4 +397,4 @@ When logging issues, include: --- -**Next**: [Full Stack Applications Testing](https://github.com/microsoft/mcp/tree/main/docs/bug-bash/scenarios/full-stack-apps.md) +**Next**: [Full Stack Applications Testing](https://github.com/microsoft/mcp/tree/main/docs/bug-bash/scenarios/full-stack-apps.md) \ No newline at end of file diff --git a/eng/tools/ToolDescriptionEvaluator/namespace-tools.json b/eng/tools/ToolDescriptionEvaluator/namespace-tools.json index 8ef1788abe..90688827c0 100644 --- a/eng/tools/ToolDescriptionEvaluator/namespace-tools.json +++ b/eng/tools/ToolDescriptionEvaluator/namespace-tools.json @@ -83,7 +83,7 @@ { "id": "", "name": "deploy", - "description": "Deploy commands for deploying applications to Azure, including sub commands: - plan get: generates a deployment plan to construct the infrastructure and deploy the application on Azure. Agent should read its output and generate a deploy plan in '.azure/plan.copilotmd' for execution steps, recommended azure services based on the information agent detected from project. Before calling this tool, please scan this workspace to detect the services to deploy and their dependent services; - iac rules get: offers guidelines for creating Bicep/Terraform files to deploy applications on Azure; - app logs get: fetch logs from log analytics workspace for Container Apps, App Services, function apps that were deployed through azd; - pipeline guidance get: guidance to create a CI/CD pipeline which provision Azure resources and build and deploy applications to Azure; - architecture diagram generate: generates an azure service architecture diagram for the application based on the provided app topology; ", + "description": "Deploy commands for deploying applications to Azure, including sub commands: - plan get: generates a deployment plan to construct the infrastructure and deploy the application on Azure. Agent should read its output and generate a deploy plan in 'docs/plan.copilotmd' for execution steps, recommended azure services based on the information agent detected from project. Before calling this tool, please scan this workspace to detect the services to deploy and their dependent services; - iac rules get: offers guidelines for creating Bicep/Terraform files to deploy applications on Azure; - app logs get: fetch logs from log analytics workspace for Container Apps, App Services, function apps that were deployed through az cli; - pipeline guidance get: guidance to create a CI/CD pipeline which provision Azure resources and build and deploy applications to Azure; - architecture diagram generate: generates an azure service architecture diagram for the application based on the provided app topology; ", "command": "deploy" }, { diff --git a/eng/tools/ToolDescriptionEvaluator/prompts.json b/eng/tools/ToolDescriptionEvaluator/prompts.json index b1a85d89b7..20740e1c15 100644 --- a/eng/tools/ToolDescriptionEvaluator/prompts.json +++ b/eng/tools/ToolDescriptionEvaluator/prompts.json @@ -6,72 +6,6 @@ "Create an AI app that helps me to manage travel queries.", "Create an AI app that helps me to manage travel queries in Microsoft Foundry" ], - "foundry_agents_connect": [ - "Query an agent in my Microsoft Foundry resource" - ], - "foundry_agents_create": [ - "Create a new Microsoft Foundry agent using instructions in the active editor" - ], - "foundry_agents_evaluate": [ - "Evaluate the full query and response I got from my agent for task_adherence" - ], - "foundry_agents_get-sdk-sample": [ - "Create a CLI app that can talk to a Microsoft Foundry Agent using Python SDK" - ], - "foundry_agents_list": [ - "List all agents in my Microsoft Foundry resource", - "Show me the available agents in my Microsoft Foundry resource" - ], - "foundry_agents_query-and-evaluate": [ - "Query and evaluate an agent in my Microsoft Foundry resource for task_adherence" - ], - "foundry_knowledge_index_list": [ - "List all knowledge indexes in my Microsoft Foundry project", - "Show me the knowledge indexes in my Microsoft Foundry project" - ], - "foundry_knowledge_index_schema": [ - "Show me the schema for knowledge index in my Microsoft Foundry resource", - "Get the schema configuration for knowledge index " - ], - "foundry_models_deploy": [ - "Deploy a GPT4o instance on my resource " - ], - "foundry_models_deployments_list": [ - "List all Microsoft Foundry model deployments", - "Show me all Microsoft Foundry model deployments" - ], - "foundry_models_list": [ - "List all Microsoft Foundry models", - "Show me the available Microsoft Foundry models" - ], - "foundry_openai_chat-completions-create": [ - "Create a chat completion with the message \"Hello, how are you today?\" using my Microsoft Foundry resource" - ], - "foundry_openai_create-completion": [ - "Create a completion with the prompt \"What is Azure?\" using my Microsoft Foundry resource" - ], - "foundry_openai_embeddings-create": [ - "Generate embeddings for the text \"Azure OpenAI Service\" using my Microsoft Foundry resource", - "Create vector embeddings for my text using my Microsoft Foundry resource" - ], - "foundry_openai_models-list": [ - "List all available OpenAI models in my Microsoft Foundry resource", - "Show me the OpenAI model deployments in my Microsoft Foundry resource" - ], - "foundry_resource_get": [ - "List all Microsoft Foundry resources in my subscription", - "Show me the Microsoft Foundry resources in resource group ", - "Get details for Microsoft Foundry resource in resource group " - ], - "foundry_threads_create": [ - "Create a Microsoft Foundry thread to hold the conversation" - ], - "foundry_threads_get-messages": [ - "Show me the messages in the Microsoft Foundry thread with id " - ], - "foundry_threads_list": [ - "List my Microsoft Foundry threads" - ], "search_knowledge_base_get": [ "List all knowledge bases in the Azure AI Search service ", "Show me the knowledge bases in the Azure AI Search service ", @@ -331,13 +265,13 @@ "Show me the schema of table in the PostgreSQL database in server " ], "deploy_app_logs_get": [ - "Show me the log of the application deployed by azd" + "Show me the log of the application in resource group " ], "deploy_architecture_diagram_generate": [ "Generate the azure architecture diagram for this application" ], "deploy_iac_rules_get": [ - "Show me the rules to generate bicep scripts" + "Show me the rules to generate scripts" ], "deploy_pipeline_guidance_get": [ "How can I create a CI/CD pipeline to deploy this app to Azure?" @@ -838,5 +772,71 @@ "Help me design an Azure cloud service that will serve as an ATM for users", "I want to design a cloud app for ordering groceries", "How can I design a cloud service in Azure that will store and present videos for users?" + ], + "foundry_agents_connect": [ + "Query an agent in my Microsoft Foundry resource" + ], + "foundry_agents_create": [ + "Create a new Microsoft Foundry agent using instructions in the active editor" + ], + "foundry_agents_evaluate": [ + "Evaluate the full query and response I got from my agent for task_adherence" + ], + "foundry_agents_get-sdk-sample": [ + "Create a CLI app that can talk to a Microsoft Foundry Agent using Python SDK" + ], + "foundry_agents_list": [ + "List all agents in my Microsoft Foundry resource", + "Show me the available agents in my Microsoft Foundry resource" + ], + "foundry_agents_query-and-evaluate": [ + "Query and evaluate an agent in my Microsoft Foundry resource for task_adherence" + ], + "foundry_knowledge_index_list": [ + "List all knowledge indexes in my Microsoft Foundry project", + "Show me the knowledge indexes in my Microsoft Foundry project" + ], + "foundry_knowledge_index_schema": [ + "Show me the schema for knowledge index in my Microsoft Foundry resource", + "Get the schema configuration for knowledge index " + ], + "foundry_models_deploy": [ + "Deploy a GPT4o instance on my resource " + ], + "foundry_models_deployments_list": [ + "List all Microsoft Foundry model deployments", + "Show me all Microsoft Foundry model deployments" + ], + "foundry_models_list": [ + "List all Microsoft Foundry models", + "Show me the available Microsoft Foundry models" + ], + "foundry_openai_chat-completions-create": [ + "Create a chat completion with the message \"Hello, how are you today?\" using my Microsoft Foundry resource" + ], + "foundry_openai_create-completion": [ + "Create a completion with the prompt \"What is Azure?\" using my Microsoft Foundry resource" + ], + "foundry_openai_embeddings-create": [ + "Generate embeddings for the text \"Azure OpenAI Service\" using my Microsoft Foundry resource", + "Create vector embeddings for my text using my Microsoft Foundry resource" + ], + "foundry_openai_models-list": [ + "List all available OpenAI models in my Microsoft Foundry resource", + "Show me the OpenAI model deployments in my Microsoft Foundry resource" + ], + "foundry_resource_get": [ + "List all Microsoft Foundry resources in my subscription", + "Show me the Microsoft Foundry resources in resource group ", + "Get details for Microsoft Foundry resource in resource group " + ], + "foundry_threads_create": [ + "Create a Microsoft Foundry thread to hold the conversation" + ], + "foundry_threads_get-messages": [ + "Show me the messages in the Microsoft Foundry thread with id " + ], + "foundry_threads_list": [ + "List my Microsoft Foundry threads" ] } \ No newline at end of file diff --git a/eng/tools/ToolDescriptionEvaluator/results.md b/eng/tools/ToolDescriptionEvaluator/results.md index e4d269cb88..9ce1bb2a43 100644 --- a/eng/tools/ToolDescriptionEvaluator/results.md +++ b/eng/tools/ToolDescriptionEvaluator/results.md @@ -1,15 +1,15 @@ # Tool Selection Analysis Setup -**Setup completed:** 2025-11-17 14:34:53 -**Tool count:** 180 -**Database setup time:** 1.1892593s +**Setup completed:** 2025-11-25 11:10:49 +**Tool count:** 181 +**Database setup time:** 20.1270870s --- # Tool Selection Analysis Results -**Analysis Date:** 2025-11-17 14:34:53 -**Tool count:** 180 +**Analysis Date:** 2025-11-25 11:10:49 +**Tool count:** 181 ## Table of Contents @@ -18,479 +18,479 @@ - [Test 3: azureaibestpractices_get](#test-3) - [Test 4: azureaibestpractices_get](#test-4) - [Test 5: azureaibestpractices_get](#test-5) -- [Test 6: foundry_agents_connect](#test-6) -- [Test 7: foundry_agents_create](#test-7) -- [Test 8: foundry_agents_evaluate](#test-8) -- [Test 9: foundry_agents_get-sdk-sample](#test-9) -- [Test 10: foundry_agents_list](#test-10) -- [Test 11: foundry_agents_list](#test-11) -- [Test 12: foundry_agents_query-and-evaluate](#test-12) -- [Test 13: foundry_knowledge_index_list](#test-13) -- [Test 14: foundry_knowledge_index_list](#test-14) -- [Test 15: foundry_knowledge_index_schema](#test-15) -- [Test 16: foundry_knowledge_index_schema](#test-16) -- [Test 17: foundry_models_deploy](#test-17) -- [Test 18: foundry_models_deployments_list](#test-18) -- [Test 19: foundry_models_deployments_list](#test-19) -- [Test 20: foundry_models_list](#test-20) -- [Test 21: foundry_models_list](#test-21) -- [Test 22: foundry_openai_chat-completions-create](#test-22) -- [Test 23: foundry_openai_create-completion](#test-23) -- [Test 24: foundry_openai_embeddings-create](#test-24) -- [Test 25: foundry_openai_embeddings-create](#test-25) -- [Test 26: foundry_openai_models-list](#test-26) -- [Test 27: foundry_openai_models-list](#test-27) -- [Test 28: foundry_resource_get](#test-28) -- [Test 29: foundry_resource_get](#test-29) -- [Test 30: foundry_resource_get](#test-30) -- [Test 31: foundry_threads_create](#test-31) -- [Test 32: foundry_threads_get-messages](#test-32) -- [Test 33: foundry_threads_list](#test-33) -- [Test 34: search_knowledge_base_get](#test-34) -- [Test 35: search_knowledge_base_get](#test-35) -- [Test 36: search_knowledge_base_get](#test-36) -- [Test 37: search_knowledge_base_get](#test-37) -- [Test 38: search_knowledge_base_get](#test-38) -- [Test 39: search_knowledge_base_get](#test-39) -- [Test 40: search_knowledge_base_retrieve](#test-40) -- [Test 41: search_knowledge_base_retrieve](#test-41) -- [Test 42: search_knowledge_base_retrieve](#test-42) -- [Test 43: search_knowledge_base_retrieve](#test-43) -- [Test 44: search_knowledge_base_retrieve](#test-44) -- [Test 45: search_knowledge_base_retrieve](#test-45) -- [Test 46: search_knowledge_base_retrieve](#test-46) -- [Test 47: search_knowledge_base_retrieve](#test-47) -- [Test 48: search_knowledge_source_get](#test-48) -- [Test 49: search_knowledge_source_get](#test-49) -- [Test 50: search_knowledge_source_get](#test-50) -- [Test 51: search_knowledge_source_get](#test-51) -- [Test 52: search_knowledge_source_get](#test-52) -- [Test 53: search_knowledge_source_get](#test-53) -- [Test 54: search_index_get](#test-54) -- [Test 55: search_index_get](#test-55) -- [Test 56: search_index_get](#test-56) -- [Test 57: search_index_query](#test-57) -- [Test 58: search_service_list](#test-58) -- [Test 59: search_service_list](#test-59) -- [Test 60: search_service_list](#test-60) -- [Test 61: speech_stt_recognize](#test-61) -- [Test 62: speech_stt_recognize](#test-62) -- [Test 63: speech_stt_recognize](#test-63) -- [Test 64: speech_stt_recognize](#test-64) -- [Test 65: speech_stt_recognize](#test-65) -- [Test 66: speech_stt_recognize](#test-66) -- [Test 67: speech_stt_recognize](#test-67) -- [Test 68: speech_stt_recognize](#test-68) -- [Test 69: speech_stt_recognize](#test-69) -- [Test 70: speech_stt_recognize](#test-70) -- [Test 71: speech_tts_synthesize](#test-71) -- [Test 72: speech_tts_synthesize](#test-72) -- [Test 73: speech_tts_synthesize](#test-73) -- [Test 74: speech_tts_synthesize](#test-74) -- [Test 75: speech_tts_synthesize](#test-75) -- [Test 76: speech_tts_synthesize](#test-76) -- [Test 77: speech_tts_synthesize](#test-77) -- [Test 78: speech_tts_synthesize](#test-78) -- [Test 79: speech_tts_synthesize](#test-79) -- [Test 80: speech_tts_synthesize](#test-80) -- [Test 81: appconfig_account_list](#test-81) -- [Test 82: appconfig_account_list](#test-82) -- [Test 83: appconfig_account_list](#test-83) -- [Test 84: appconfig_kv_delete](#test-84) -- [Test 85: appconfig_kv_get](#test-85) -- [Test 86: appconfig_kv_get](#test-86) -- [Test 87: appconfig_kv_get](#test-87) -- [Test 88: appconfig_kv_get](#test-88) -- [Test 89: appconfig_kv_lock_set](#test-89) -- [Test 90: appconfig_kv_lock_set](#test-90) -- [Test 91: appconfig_kv_set](#test-91) -- [Test 92: applens_resource_diagnose](#test-92) -- [Test 93: applens_resource_diagnose](#test-93) -- [Test 94: applens_resource_diagnose](#test-94) -- [Test 95: appservice_database_add](#test-95) -- [Test 96: appservice_database_add](#test-96) -- [Test 97: appservice_database_add](#test-97) -- [Test 98: appservice_database_add](#test-98) -- [Test 99: appservice_database_add](#test-99) -- [Test 100: appservice_database_add](#test-100) -- [Test 101: appservice_database_add](#test-101) -- [Test 102: appservice_database_add](#test-102) -- [Test 103: appservice_database_add](#test-103) -- [Test 104: appservice_database_add](#test-104) -- [Test 105: applicationinsights_recommendation_list](#test-105) -- [Test 106: applicationinsights_recommendation_list](#test-106) -- [Test 107: applicationinsights_recommendation_list](#test-107) -- [Test 108: applicationinsights_recommendation_list](#test-108) -- [Test 109: extension_cli_generate](#test-109) -- [Test 110: extension_cli_generate](#test-110) -- [Test 111: extension_cli_generate](#test-111) -- [Test 112: extension_cli_install](#test-112) -- [Test 113: extension_cli_install](#test-113) -- [Test 114: extension_cli_install](#test-114) -- [Test 115: acr_registry_list](#test-115) -- [Test 116: acr_registry_list](#test-116) -- [Test 117: acr_registry_list](#test-117) -- [Test 118: acr_registry_list](#test-118) -- [Test 119: acr_registry_list](#test-119) -- [Test 120: acr_registry_repository_list](#test-120) -- [Test 121: acr_registry_repository_list](#test-121) -- [Test 122: acr_registry_repository_list](#test-122) -- [Test 123: acr_registry_repository_list](#test-123) -- [Test 124: communication_email_send](#test-124) -- [Test 125: communication_email_send](#test-125) -- [Test 126: communication_email_send](#test-126) -- [Test 127: communication_email_send](#test-127) -- [Test 128: communication_email_send](#test-128) -- [Test 129: communication_email_send](#test-129) -- [Test 130: communication_email_send](#test-130) -- [Test 131: communication_email_send](#test-131) -- [Test 132: communication_sms_send](#test-132) -- [Test 133: communication_sms_send](#test-133) -- [Test 134: communication_sms_send](#test-134) -- [Test 135: communication_sms_send](#test-135) -- [Test 136: communication_sms_send](#test-136) -- [Test 137: communication_sms_send](#test-137) -- [Test 138: communication_sms_send](#test-138) -- [Test 139: communication_sms_send](#test-139) -- [Test 140: confidentialledger_entries_append](#test-140) -- [Test 141: confidentialledger_entries_append](#test-141) -- [Test 142: confidentialledger_entries_append](#test-142) -- [Test 143: confidentialledger_entries_append](#test-143) -- [Test 144: confidentialledger_entries_append](#test-144) -- [Test 145: confidentialledger_entries_get](#test-145) -- [Test 146: confidentialledger_entries_get](#test-146) -- [Test 147: cosmos_account_list](#test-147) -- [Test 148: cosmos_account_list](#test-148) -- [Test 149: cosmos_account_list](#test-149) -- [Test 150: cosmos_database_container_item_query](#test-150) -- [Test 151: cosmos_database_container_list](#test-151) -- [Test 152: cosmos_database_container_list](#test-152) -- [Test 153: cosmos_database_list](#test-153) -- [Test 154: cosmos_database_list](#test-154) -- [Test 155: kusto_cluster_get](#test-155) -- [Test 156: kusto_cluster_list](#test-156) -- [Test 157: kusto_cluster_list](#test-157) -- [Test 158: kusto_cluster_list](#test-158) -- [Test 159: kusto_database_list](#test-159) -- [Test 160: kusto_database_list](#test-160) -- [Test 161: kusto_query](#test-161) -- [Test 162: kusto_sample](#test-162) -- [Test 163: kusto_table_list](#test-163) -- [Test 164: kusto_table_list](#test-164) -- [Test 165: kusto_table_schema](#test-165) -- [Test 166: mysql_database_list](#test-166) -- [Test 167: mysql_database_list](#test-167) -- [Test 168: mysql_database_query](#test-168) -- [Test 169: mysql_server_config_get](#test-169) -- [Test 170: mysql_server_list](#test-170) -- [Test 171: mysql_server_list](#test-171) -- [Test 172: mysql_server_list](#test-172) -- [Test 173: mysql_server_param_get](#test-173) -- [Test 174: mysql_server_param_set](#test-174) -- [Test 175: mysql_table_list](#test-175) -- [Test 176: mysql_table_list](#test-176) -- [Test 177: mysql_table_schema_get](#test-177) -- [Test 178: postgres_database_list](#test-178) -- [Test 179: postgres_database_list](#test-179) -- [Test 180: postgres_database_query](#test-180) -- [Test 181: postgres_server_config_get](#test-181) -- [Test 182: postgres_server_list](#test-182) -- [Test 183: postgres_server_list](#test-183) -- [Test 184: postgres_server_list](#test-184) -- [Test 185: postgres_server_param_get](#test-185) -- [Test 186: postgres_server_param_set](#test-186) -- [Test 187: postgres_table_list](#test-187) -- [Test 188: postgres_table_list](#test-188) -- [Test 189: postgres_table_schema_get](#test-189) -- [Test 190: deploy_app_logs_get](#test-190) -- [Test 191: deploy_architecture_diagram_generate](#test-191) -- [Test 192: deploy_iac_rules_get](#test-192) -- [Test 193: deploy_pipeline_guidance_get](#test-193) -- [Test 194: deploy_plan_get](#test-194) -- [Test 195: eventgrid_events_publish](#test-195) -- [Test 196: eventgrid_events_publish](#test-196) -- [Test 197: eventgrid_events_publish](#test-197) -- [Test 198: eventgrid_topic_list](#test-198) -- [Test 199: eventgrid_topic_list](#test-199) -- [Test 200: eventgrid_topic_list](#test-200) -- [Test 201: eventgrid_topic_list](#test-201) -- [Test 202: eventgrid_subscription_list](#test-202) -- [Test 203: eventgrid_subscription_list](#test-203) -- [Test 204: eventgrid_subscription_list](#test-204) -- [Test 205: eventgrid_subscription_list](#test-205) -- [Test 206: eventgrid_subscription_list](#test-206) -- [Test 207: eventgrid_subscription_list](#test-207) -- [Test 208: eventgrid_subscription_list](#test-208) -- [Test 209: eventhubs_eventhub_consumergroup_delete](#test-209) -- [Test 210: eventhubs_eventhub_consumergroup_get](#test-210) -- [Test 211: eventhubs_eventhub_consumergroup_get](#test-211) -- [Test 212: eventhubs_eventhub_consumergroup_update](#test-212) -- [Test 213: eventhubs_eventhub_consumergroup_update](#test-213) -- [Test 214: eventhubs_eventhub_delete](#test-214) -- [Test 215: eventhubs_eventhub_get](#test-215) -- [Test 216: eventhubs_eventhub_get](#test-216) -- [Test 217: eventhubs_eventhub_update](#test-217) -- [Test 218: eventhubs_eventhub_update](#test-218) -- [Test 219: eventhubs_namespace_delete](#test-219) -- [Test 220: eventhubs_namespace_get](#test-220) -- [Test 221: eventhubs_namespace_get](#test-221) -- [Test 222: eventhubs_namespace_update](#test-222) -- [Test 223: eventhubs_namespace_update](#test-223) -- [Test 224: functionapp_get](#test-224) -- [Test 225: functionapp_get](#test-225) -- [Test 226: functionapp_get](#test-226) -- [Test 227: functionapp_get](#test-227) -- [Test 228: functionapp_get](#test-228) -- [Test 229: functionapp_get](#test-229) -- [Test 230: functionapp_get](#test-230) -- [Test 231: functionapp_get](#test-231) -- [Test 232: functionapp_get](#test-232) -- [Test 233: functionapp_get](#test-233) -- [Test 234: functionapp_get](#test-234) -- [Test 235: functionapp_get](#test-235) -- [Test 236: keyvault_admin_settings_get](#test-236) -- [Test 237: keyvault_admin_settings_get](#test-237) -- [Test 238: keyvault_admin_settings_get](#test-238) -- [Test 239: keyvault_certificate_create](#test-239) -- [Test 240: keyvault_certificate_create](#test-240) -- [Test 241: keyvault_certificate_create](#test-241) -- [Test 242: keyvault_certificate_create](#test-242) -- [Test 243: keyvault_certificate_create](#test-243) -- [Test 244: keyvault_certificate_get](#test-244) -- [Test 245: keyvault_certificate_get](#test-245) -- [Test 246: keyvault_certificate_get](#test-246) -- [Test 247: keyvault_certificate_get](#test-247) -- [Test 248: keyvault_certificate_get](#test-248) -- [Test 249: keyvault_certificate_import](#test-249) -- [Test 250: keyvault_certificate_import](#test-250) -- [Test 251: keyvault_certificate_import](#test-251) -- [Test 252: keyvault_certificate_import](#test-252) -- [Test 253: keyvault_certificate_import](#test-253) -- [Test 254: keyvault_certificate_list](#test-254) -- [Test 255: keyvault_certificate_list](#test-255) -- [Test 256: keyvault_certificate_list](#test-256) -- [Test 257: keyvault_certificate_list](#test-257) -- [Test 258: keyvault_certificate_list](#test-258) -- [Test 259: keyvault_certificate_list](#test-259) -- [Test 260: keyvault_key_create](#test-260) -- [Test 261: keyvault_key_create](#test-261) -- [Test 262: keyvault_key_create](#test-262) -- [Test 263: keyvault_key_create](#test-263) -- [Test 264: keyvault_key_create](#test-264) -- [Test 265: keyvault_key_get](#test-265) -- [Test 266: keyvault_key_get](#test-266) -- [Test 267: keyvault_key_get](#test-267) -- [Test 268: keyvault_key_get](#test-268) -- [Test 269: keyvault_key_get](#test-269) -- [Test 270: keyvault_key_list](#test-270) -- [Test 271: keyvault_key_list](#test-271) -- [Test 272: keyvault_key_list](#test-272) -- [Test 273: keyvault_key_list](#test-273) -- [Test 274: keyvault_key_list](#test-274) -- [Test 275: keyvault_key_list](#test-275) -- [Test 276: keyvault_secret_create](#test-276) -- [Test 277: keyvault_secret_create](#test-277) -- [Test 278: keyvault_secret_create](#test-278) -- [Test 279: keyvault_secret_create](#test-279) -- [Test 280: keyvault_secret_create](#test-280) -- [Test 281: keyvault_secret_get](#test-281) -- [Test 282: keyvault_secret_get](#test-282) -- [Test 283: keyvault_secret_get](#test-283) -- [Test 284: keyvault_secret_get](#test-284) -- [Test 285: keyvault_secret_get](#test-285) -- [Test 286: keyvault_secret_list](#test-286) -- [Test 287: keyvault_secret_list](#test-287) -- [Test 288: keyvault_secret_list](#test-288) -- [Test 289: keyvault_secret_list](#test-289) -- [Test 290: keyvault_secret_list](#test-290) -- [Test 291: keyvault_secret_list](#test-291) -- [Test 292: aks_cluster_get](#test-292) -- [Test 293: aks_cluster_get](#test-293) -- [Test 294: aks_cluster_get](#test-294) -- [Test 295: aks_cluster_get](#test-295) -- [Test 296: aks_cluster_get](#test-296) -- [Test 297: aks_cluster_get](#test-297) -- [Test 298: aks_cluster_get](#test-298) -- [Test 299: aks_nodepool_get](#test-299) -- [Test 300: aks_nodepool_get](#test-300) -- [Test 301: aks_nodepool_get](#test-301) -- [Test 302: aks_nodepool_get](#test-302) -- [Test 303: aks_nodepool_get](#test-303) -- [Test 304: aks_nodepool_get](#test-304) -- [Test 305: loadtesting_test_create](#test-305) -- [Test 306: loadtesting_test_get](#test-306) -- [Test 307: loadtesting_testresource_create](#test-307) -- [Test 308: loadtesting_testresource_list](#test-308) -- [Test 309: loadtesting_testrun_create](#test-309) -- [Test 310: loadtesting_testrun_get](#test-310) -- [Test 311: loadtesting_testrun_list](#test-311) -- [Test 312: loadtesting_testrun_update](#test-312) -- [Test 313: grafana_list](#test-313) -- [Test 314: managedlustre_fs_create](#test-314) -- [Test 315: managedlustre_fs_list](#test-315) -- [Test 316: managedlustre_fs_list](#test-316) -- [Test 317: managedlustre_fs_sku_get](#test-317) -- [Test 318: managedlustre_fs_subnetsize_ask](#test-318) -- [Test 319: managedlustre_fs_subnetsize_validate](#test-319) -- [Test 320: managedlustre_fs_update](#test-320) -- [Test 321: marketplace_product_get](#test-321) -- [Test 322: marketplace_product_list](#test-322) -- [Test 323: marketplace_product_list](#test-323) -- [Test 324: get_bestpractices_get](#test-324) -- [Test 325: get_bestpractices_get](#test-325) -- [Test 326: get_bestpractices_get](#test-326) -- [Test 327: get_bestpractices_get](#test-327) -- [Test 328: get_bestpractices_get](#test-328) -- [Test 329: get_bestpractices_get](#test-329) -- [Test 330: get_bestpractices_get](#test-330) -- [Test 331: get_bestpractices_get](#test-331) -- [Test 332: get_bestpractices_get](#test-332) -- [Test 333: monitor_activitylog_list](#test-333) -- [Test 334: monitor_healthmodels_entity_get](#test-334) -- [Test 335: monitor_metrics_definitions](#test-335) -- [Test 336: monitor_metrics_definitions](#test-336) -- [Test 337: monitor_metrics_definitions](#test-337) -- [Test 338: monitor_metrics_query](#test-338) -- [Test 339: monitor_metrics_query](#test-339) -- [Test 340: monitor_metrics_query](#test-340) -- [Test 341: monitor_metrics_query](#test-341) -- [Test 342: monitor_metrics_query](#test-342) -- [Test 343: monitor_metrics_query](#test-343) -- [Test 344: monitor_resource_log_query](#test-344) -- [Test 345: monitor_table_list](#test-345) -- [Test 346: monitor_table_list](#test-346) -- [Test 347: monitor_table_type_list](#test-347) -- [Test 348: monitor_table_type_list](#test-348) -- [Test 349: monitor_webtests_create](#test-349) -- [Test 350: monitor_webtests_get](#test-350) -- [Test 351: monitor_webtests_list](#test-351) -- [Test 352: monitor_webtests_list](#test-352) -- [Test 353: monitor_webtests_update](#test-353) -- [Test 354: monitor_workspace_list](#test-354) -- [Test 355: monitor_workspace_list](#test-355) -- [Test 356: monitor_workspace_list](#test-356) -- [Test 357: monitor_workspace_log_query](#test-357) -- [Test 358: datadog_monitoredresources_list](#test-358) -- [Test 359: datadog_monitoredresources_list](#test-359) -- [Test 360: extension_azqr](#test-360) -- [Test 361: extension_azqr](#test-361) -- [Test 362: extension_azqr](#test-362) -- [Test 363: quota_region_availability_list](#test-363) -- [Test 364: quota_usage_check](#test-364) -- [Test 365: role_assignment_list](#test-365) -- [Test 366: role_assignment_list](#test-366) -- [Test 367: redis_create](#test-367) -- [Test 368: redis_create](#test-368) -- [Test 369: redis_create](#test-369) -- [Test 370: redis_create](#test-370) -- [Test 371: redis_list](#test-371) -- [Test 372: redis_list](#test-372) -- [Test 373: redis_list](#test-373) -- [Test 374: redis_list](#test-374) -- [Test 375: redis_list](#test-375) -- [Test 376: group_list](#test-376) -- [Test 377: group_list](#test-377) -- [Test 378: group_list](#test-378) -- [Test 379: resourcehealth_availability-status_get](#test-379) -- [Test 380: resourcehealth_availability-status_get](#test-380) -- [Test 381: resourcehealth_availability-status_get](#test-381) -- [Test 382: resourcehealth_availability-status_list](#test-382) -- [Test 383: resourcehealth_availability-status_list](#test-383) -- [Test 384: resourcehealth_availability-status_list](#test-384) -- [Test 385: resourcehealth_health-events_list](#test-385) -- [Test 386: resourcehealth_health-events_list](#test-386) -- [Test 387: resourcehealth_health-events_list](#test-387) -- [Test 388: resourcehealth_health-events_list](#test-388) -- [Test 389: resourcehealth_health-events_list](#test-389) -- [Test 390: servicebus_queue_details](#test-390) -- [Test 391: servicebus_topic_details](#test-391) -- [Test 392: servicebus_topic_subscription_details](#test-392) -- [Test 393: signalr_runtime_get](#test-393) -- [Test 394: signalr_runtime_get](#test-394) -- [Test 395: signalr_runtime_get](#test-395) -- [Test 396: signalr_runtime_get](#test-396) -- [Test 397: signalr_runtime_get](#test-397) -- [Test 398: signalr_runtime_get](#test-398) -- [Test 399: sql_db_create](#test-399) -- [Test 400: sql_db_create](#test-400) -- [Test 401: sql_db_create](#test-401) -- [Test 402: sql_db_delete](#test-402) -- [Test 403: sql_db_delete](#test-403) -- [Test 404: sql_db_delete](#test-404) -- [Test 405: sql_db_list](#test-405) -- [Test 406: sql_db_list](#test-406) -- [Test 407: sql_db_rename](#test-407) -- [Test 408: sql_db_rename](#test-408) -- [Test 409: sql_db_show](#test-409) -- [Test 410: sql_db_show](#test-410) -- [Test 411: sql_db_update](#test-411) -- [Test 412: sql_db_update](#test-412) -- [Test 413: sql_elastic-pool_list](#test-413) -- [Test 414: sql_elastic-pool_list](#test-414) -- [Test 415: sql_elastic-pool_list](#test-415) -- [Test 416: sql_server_create](#test-416) -- [Test 417: sql_server_create](#test-417) -- [Test 418: sql_server_create](#test-418) -- [Test 419: sql_server_delete](#test-419) -- [Test 420: sql_server_delete](#test-420) -- [Test 421: sql_server_delete](#test-421) -- [Test 422: sql_server_entra-admin_list](#test-422) -- [Test 423: sql_server_entra-admin_list](#test-423) -- [Test 424: sql_server_entra-admin_list](#test-424) -- [Test 425: sql_server_firewall-rule_create](#test-425) -- [Test 426: sql_server_firewall-rule_create](#test-426) -- [Test 427: sql_server_firewall-rule_create](#test-427) -- [Test 428: sql_server_firewall-rule_delete](#test-428) -- [Test 429: sql_server_firewall-rule_delete](#test-429) -- [Test 430: sql_server_firewall-rule_delete](#test-430) -- [Test 431: sql_server_firewall-rule_list](#test-431) -- [Test 432: sql_server_firewall-rule_list](#test-432) -- [Test 433: sql_server_firewall-rule_list](#test-433) -- [Test 434: sql_server_list](#test-434) -- [Test 435: sql_server_list](#test-435) -- [Test 436: sql_server_show](#test-436) -- [Test 437: sql_server_show](#test-437) -- [Test 438: sql_server_show](#test-438) -- [Test 439: storage_account_create](#test-439) -- [Test 440: storage_account_create](#test-440) -- [Test 441: storage_account_create](#test-441) -- [Test 442: storage_account_get](#test-442) -- [Test 443: storage_account_get](#test-443) -- [Test 444: storage_account_get](#test-444) -- [Test 445: storage_account_get](#test-445) -- [Test 446: storage_account_get](#test-446) -- [Test 447: storage_blob_container_create](#test-447) -- [Test 448: storage_blob_container_create](#test-448) -- [Test 449: storage_blob_container_create](#test-449) -- [Test 450: storage_blob_container_get](#test-450) -- [Test 451: storage_blob_container_get](#test-451) -- [Test 452: storage_blob_container_get](#test-452) -- [Test 453: storage_blob_get](#test-453) -- [Test 454: storage_blob_get](#test-454) -- [Test 455: storage_blob_get](#test-455) -- [Test 456: storage_blob_get](#test-456) -- [Test 457: storage_blob_upload](#test-457) -- [Test 458: subscription_list](#test-458) -- [Test 459: subscription_list](#test-459) -- [Test 460: subscription_list](#test-460) -- [Test 461: subscription_list](#test-461) -- [Test 462: azureterraformbestpractices_get](#test-462) -- [Test 463: azureterraformbestpractices_get](#test-463) -- [Test 464: virtualdesktop_hostpool_list](#test-464) -- [Test 465: virtualdesktop_hostpool_host_list](#test-465) -- [Test 466: virtualdesktop_hostpool_host_user-list](#test-466) -- [Test 467: workbooks_create](#test-467) -- [Test 468: workbooks_delete](#test-468) -- [Test 469: workbooks_list](#test-469) -- [Test 470: workbooks_list](#test-470) -- [Test 471: workbooks_show](#test-471) -- [Test 472: workbooks_show](#test-472) -- [Test 473: workbooks_update](#test-473) -- [Test 474: bicepschema_get](#test-474) -- [Test 475: cloudarchitect_design](#test-475) -- [Test 476: cloudarchitect_design](#test-476) -- [Test 477: cloudarchitect_design](#test-477) -- [Test 478: cloudarchitect_design](#test-478) +- [Test 6: search_knowledge_base_get](#test-6) +- [Test 7: search_knowledge_base_get](#test-7) +- [Test 8: search_knowledge_base_get](#test-8) +- [Test 9: search_knowledge_base_get](#test-9) +- [Test 10: search_knowledge_base_get](#test-10) +- [Test 11: search_knowledge_base_get](#test-11) +- [Test 12: search_knowledge_base_retrieve](#test-12) +- [Test 13: search_knowledge_base_retrieve](#test-13) +- [Test 14: search_knowledge_base_retrieve](#test-14) +- [Test 15: search_knowledge_base_retrieve](#test-15) +- [Test 16: search_knowledge_base_retrieve](#test-16) +- [Test 17: search_knowledge_base_retrieve](#test-17) +- [Test 18: search_knowledge_base_retrieve](#test-18) +- [Test 19: search_knowledge_base_retrieve](#test-19) +- [Test 20: search_knowledge_source_get](#test-20) +- [Test 21: search_knowledge_source_get](#test-21) +- [Test 22: search_knowledge_source_get](#test-22) +- [Test 23: search_knowledge_source_get](#test-23) +- [Test 24: search_knowledge_source_get](#test-24) +- [Test 25: search_knowledge_source_get](#test-25) +- [Test 26: search_index_get](#test-26) +- [Test 27: search_index_get](#test-27) +- [Test 28: search_index_get](#test-28) +- [Test 29: search_index_query](#test-29) +- [Test 30: search_service_list](#test-30) +- [Test 31: search_service_list](#test-31) +- [Test 32: search_service_list](#test-32) +- [Test 33: speech_stt_recognize](#test-33) +- [Test 34: speech_stt_recognize](#test-34) +- [Test 35: speech_stt_recognize](#test-35) +- [Test 36: speech_stt_recognize](#test-36) +- [Test 37: speech_stt_recognize](#test-37) +- [Test 38: speech_stt_recognize](#test-38) +- [Test 39: speech_stt_recognize](#test-39) +- [Test 40: speech_stt_recognize](#test-40) +- [Test 41: speech_stt_recognize](#test-41) +- [Test 42: speech_stt_recognize](#test-42) +- [Test 43: speech_tts_synthesize](#test-43) +- [Test 44: speech_tts_synthesize](#test-44) +- [Test 45: speech_tts_synthesize](#test-45) +- [Test 46: speech_tts_synthesize](#test-46) +- [Test 47: speech_tts_synthesize](#test-47) +- [Test 48: speech_tts_synthesize](#test-48) +- [Test 49: speech_tts_synthesize](#test-49) +- [Test 50: speech_tts_synthesize](#test-50) +- [Test 51: speech_tts_synthesize](#test-51) +- [Test 52: speech_tts_synthesize](#test-52) +- [Test 53: appconfig_account_list](#test-53) +- [Test 54: appconfig_account_list](#test-54) +- [Test 55: appconfig_account_list](#test-55) +- [Test 56: appconfig_kv_delete](#test-56) +- [Test 57: appconfig_kv_get](#test-57) +- [Test 58: appconfig_kv_get](#test-58) +- [Test 59: appconfig_kv_get](#test-59) +- [Test 60: appconfig_kv_get](#test-60) +- [Test 61: appconfig_kv_lock_set](#test-61) +- [Test 62: appconfig_kv_lock_set](#test-62) +- [Test 63: appconfig_kv_set](#test-63) +- [Test 64: applens_resource_diagnose](#test-64) +- [Test 65: applens_resource_diagnose](#test-65) +- [Test 66: applens_resource_diagnose](#test-66) +- [Test 67: appservice_database_add](#test-67) +- [Test 68: appservice_database_add](#test-68) +- [Test 69: appservice_database_add](#test-69) +- [Test 70: appservice_database_add](#test-70) +- [Test 71: appservice_database_add](#test-71) +- [Test 72: appservice_database_add](#test-72) +- [Test 73: appservice_database_add](#test-73) +- [Test 74: appservice_database_add](#test-74) +- [Test 75: appservice_database_add](#test-75) +- [Test 76: appservice_database_add](#test-76) +- [Test 77: applicationinsights_recommendation_list](#test-77) +- [Test 78: applicationinsights_recommendation_list](#test-78) +- [Test 79: applicationinsights_recommendation_list](#test-79) +- [Test 80: applicationinsights_recommendation_list](#test-80) +- [Test 81: extension_cli_generate](#test-81) +- [Test 82: extension_cli_generate](#test-82) +- [Test 83: extension_cli_generate](#test-83) +- [Test 84: extension_cli_install](#test-84) +- [Test 85: extension_cli_install](#test-85) +- [Test 86: extension_cli_install](#test-86) +- [Test 87: acr_registry_list](#test-87) +- [Test 88: acr_registry_list](#test-88) +- [Test 89: acr_registry_list](#test-89) +- [Test 90: acr_registry_list](#test-90) +- [Test 91: acr_registry_list](#test-91) +- [Test 92: acr_registry_repository_list](#test-92) +- [Test 93: acr_registry_repository_list](#test-93) +- [Test 94: acr_registry_repository_list](#test-94) +- [Test 95: acr_registry_repository_list](#test-95) +- [Test 96: communication_email_send](#test-96) +- [Test 97: communication_email_send](#test-97) +- [Test 98: communication_email_send](#test-98) +- [Test 99: communication_email_send](#test-99) +- [Test 100: communication_email_send](#test-100) +- [Test 101: communication_email_send](#test-101) +- [Test 102: communication_email_send](#test-102) +- [Test 103: communication_email_send](#test-103) +- [Test 104: communication_sms_send](#test-104) +- [Test 105: communication_sms_send](#test-105) +- [Test 106: communication_sms_send](#test-106) +- [Test 107: communication_sms_send](#test-107) +- [Test 108: communication_sms_send](#test-108) +- [Test 109: communication_sms_send](#test-109) +- [Test 110: communication_sms_send](#test-110) +- [Test 111: communication_sms_send](#test-111) +- [Test 112: confidentialledger_entries_append](#test-112) +- [Test 113: confidentialledger_entries_append](#test-113) +- [Test 114: confidentialledger_entries_append](#test-114) +- [Test 115: confidentialledger_entries_append](#test-115) +- [Test 116: confidentialledger_entries_append](#test-116) +- [Test 117: confidentialledger_entries_get](#test-117) +- [Test 118: confidentialledger_entries_get](#test-118) +- [Test 119: cosmos_account_list](#test-119) +- [Test 120: cosmos_account_list](#test-120) +- [Test 121: cosmos_account_list](#test-121) +- [Test 122: cosmos_database_container_item_query](#test-122) +- [Test 123: cosmos_database_container_list](#test-123) +- [Test 124: cosmos_database_container_list](#test-124) +- [Test 125: cosmos_database_list](#test-125) +- [Test 126: cosmos_database_list](#test-126) +- [Test 127: kusto_cluster_get](#test-127) +- [Test 128: kusto_cluster_list](#test-128) +- [Test 129: kusto_cluster_list](#test-129) +- [Test 130: kusto_cluster_list](#test-130) +- [Test 131: kusto_database_list](#test-131) +- [Test 132: kusto_database_list](#test-132) +- [Test 133: kusto_query](#test-133) +- [Test 134: kusto_sample](#test-134) +- [Test 135: kusto_table_list](#test-135) +- [Test 136: kusto_table_list](#test-136) +- [Test 137: kusto_table_schema](#test-137) +- [Test 138: mysql_database_list](#test-138) +- [Test 139: mysql_database_list](#test-139) +- [Test 140: mysql_database_query](#test-140) +- [Test 141: mysql_server_config_get](#test-141) +- [Test 142: mysql_server_list](#test-142) +- [Test 143: mysql_server_list](#test-143) +- [Test 144: mysql_server_list](#test-144) +- [Test 145: mysql_server_param_get](#test-145) +- [Test 146: mysql_server_param_set](#test-146) +- [Test 147: mysql_table_list](#test-147) +- [Test 148: mysql_table_list](#test-148) +- [Test 149: mysql_table_schema_get](#test-149) +- [Test 150: postgres_database_list](#test-150) +- [Test 151: postgres_database_list](#test-151) +- [Test 152: postgres_database_query](#test-152) +- [Test 153: postgres_server_config_get](#test-153) +- [Test 154: postgres_server_list](#test-154) +- [Test 155: postgres_server_list](#test-155) +- [Test 156: postgres_server_list](#test-156) +- [Test 157: postgres_server_param_get](#test-157) +- [Test 158: postgres_server_param_set](#test-158) +- [Test 159: postgres_table_list](#test-159) +- [Test 160: postgres_table_list](#test-160) +- [Test 161: postgres_table_schema_get](#test-161) +- [Test 162: deploy_app_logs_get](#test-162) +- [Test 163: deploy_architecture_diagram_generate](#test-163) +- [Test 164: deploy_iac_rules_get](#test-164) +- [Test 165: deploy_pipeline_guidance_get](#test-165) +- [Test 166: deploy_plan_get](#test-166) +- [Test 167: eventgrid_events_publish](#test-167) +- [Test 168: eventgrid_events_publish](#test-168) +- [Test 169: eventgrid_events_publish](#test-169) +- [Test 170: eventgrid_topic_list](#test-170) +- [Test 171: eventgrid_topic_list](#test-171) +- [Test 172: eventgrid_topic_list](#test-172) +- [Test 173: eventgrid_topic_list](#test-173) +- [Test 174: eventgrid_subscription_list](#test-174) +- [Test 175: eventgrid_subscription_list](#test-175) +- [Test 176: eventgrid_subscription_list](#test-176) +- [Test 177: eventgrid_subscription_list](#test-177) +- [Test 178: eventgrid_subscription_list](#test-178) +- [Test 179: eventgrid_subscription_list](#test-179) +- [Test 180: eventgrid_subscription_list](#test-180) +- [Test 181: eventhubs_eventhub_consumergroup_delete](#test-181) +- [Test 182: eventhubs_eventhub_consumergroup_get](#test-182) +- [Test 183: eventhubs_eventhub_consumergroup_get](#test-183) +- [Test 184: eventhubs_eventhub_consumergroup_update](#test-184) +- [Test 185: eventhubs_eventhub_consumergroup_update](#test-185) +- [Test 186: eventhubs_eventhub_delete](#test-186) +- [Test 187: eventhubs_eventhub_get](#test-187) +- [Test 188: eventhubs_eventhub_get](#test-188) +- [Test 189: eventhubs_eventhub_update](#test-189) +- [Test 190: eventhubs_eventhub_update](#test-190) +- [Test 191: eventhubs_namespace_delete](#test-191) +- [Test 192: eventhubs_namespace_get](#test-192) +- [Test 193: eventhubs_namespace_get](#test-193) +- [Test 194: eventhubs_namespace_update](#test-194) +- [Test 195: eventhubs_namespace_update](#test-195) +- [Test 196: functionapp_get](#test-196) +- [Test 197: functionapp_get](#test-197) +- [Test 198: functionapp_get](#test-198) +- [Test 199: functionapp_get](#test-199) +- [Test 200: functionapp_get](#test-200) +- [Test 201: functionapp_get](#test-201) +- [Test 202: functionapp_get](#test-202) +- [Test 203: functionapp_get](#test-203) +- [Test 204: functionapp_get](#test-204) +- [Test 205: functionapp_get](#test-205) +- [Test 206: functionapp_get](#test-206) +- [Test 207: functionapp_get](#test-207) +- [Test 208: keyvault_admin_settings_get](#test-208) +- [Test 209: keyvault_admin_settings_get](#test-209) +- [Test 210: keyvault_admin_settings_get](#test-210) +- [Test 211: keyvault_certificate_create](#test-211) +- [Test 212: keyvault_certificate_create](#test-212) +- [Test 213: keyvault_certificate_create](#test-213) +- [Test 214: keyvault_certificate_create](#test-214) +- [Test 215: keyvault_certificate_create](#test-215) +- [Test 216: keyvault_certificate_get](#test-216) +- [Test 217: keyvault_certificate_get](#test-217) +- [Test 218: keyvault_certificate_get](#test-218) +- [Test 219: keyvault_certificate_get](#test-219) +- [Test 220: keyvault_certificate_get](#test-220) +- [Test 221: keyvault_certificate_import](#test-221) +- [Test 222: keyvault_certificate_import](#test-222) +- [Test 223: keyvault_certificate_import](#test-223) +- [Test 224: keyvault_certificate_import](#test-224) +- [Test 225: keyvault_certificate_import](#test-225) +- [Test 226: keyvault_certificate_list](#test-226) +- [Test 227: keyvault_certificate_list](#test-227) +- [Test 228: keyvault_certificate_list](#test-228) +- [Test 229: keyvault_certificate_list](#test-229) +- [Test 230: keyvault_certificate_list](#test-230) +- [Test 231: keyvault_certificate_list](#test-231) +- [Test 232: keyvault_key_create](#test-232) +- [Test 233: keyvault_key_create](#test-233) +- [Test 234: keyvault_key_create](#test-234) +- [Test 235: keyvault_key_create](#test-235) +- [Test 236: keyvault_key_create](#test-236) +- [Test 237: keyvault_key_get](#test-237) +- [Test 238: keyvault_key_get](#test-238) +- [Test 239: keyvault_key_get](#test-239) +- [Test 240: keyvault_key_get](#test-240) +- [Test 241: keyvault_key_get](#test-241) +- [Test 242: keyvault_key_list](#test-242) +- [Test 243: keyvault_key_list](#test-243) +- [Test 244: keyvault_key_list](#test-244) +- [Test 245: keyvault_key_list](#test-245) +- [Test 246: keyvault_key_list](#test-246) +- [Test 247: keyvault_key_list](#test-247) +- [Test 248: keyvault_secret_create](#test-248) +- [Test 249: keyvault_secret_create](#test-249) +- [Test 250: keyvault_secret_create](#test-250) +- [Test 251: keyvault_secret_create](#test-251) +- [Test 252: keyvault_secret_create](#test-252) +- [Test 253: keyvault_secret_get](#test-253) +- [Test 254: keyvault_secret_get](#test-254) +- [Test 255: keyvault_secret_get](#test-255) +- [Test 256: keyvault_secret_get](#test-256) +- [Test 257: keyvault_secret_get](#test-257) +- [Test 258: keyvault_secret_list](#test-258) +- [Test 259: keyvault_secret_list](#test-259) +- [Test 260: keyvault_secret_list](#test-260) +- [Test 261: keyvault_secret_list](#test-261) +- [Test 262: keyvault_secret_list](#test-262) +- [Test 263: keyvault_secret_list](#test-263) +- [Test 264: aks_cluster_get](#test-264) +- [Test 265: aks_cluster_get](#test-265) +- [Test 266: aks_cluster_get](#test-266) +- [Test 267: aks_cluster_get](#test-267) +- [Test 268: aks_cluster_get](#test-268) +- [Test 269: aks_cluster_get](#test-269) +- [Test 270: aks_cluster_get](#test-270) +- [Test 271: aks_nodepool_get](#test-271) +- [Test 272: aks_nodepool_get](#test-272) +- [Test 273: aks_nodepool_get](#test-273) +- [Test 274: aks_nodepool_get](#test-274) +- [Test 275: aks_nodepool_get](#test-275) +- [Test 276: aks_nodepool_get](#test-276) +- [Test 277: loadtesting_test_create](#test-277) +- [Test 278: loadtesting_test_get](#test-278) +- [Test 279: loadtesting_testresource_create](#test-279) +- [Test 280: loadtesting_testresource_list](#test-280) +- [Test 281: loadtesting_testrun_create](#test-281) +- [Test 282: loadtesting_testrun_get](#test-282) +- [Test 283: loadtesting_testrun_list](#test-283) +- [Test 284: loadtesting_testrun_update](#test-284) +- [Test 285: grafana_list](#test-285) +- [Test 286: managedlustre_fs_create](#test-286) +- [Test 287: managedlustre_fs_list](#test-287) +- [Test 288: managedlustre_fs_list](#test-288) +- [Test 289: managedlustre_fs_sku_get](#test-289) +- [Test 290: managedlustre_fs_subnetsize_ask](#test-290) +- [Test 291: managedlustre_fs_subnetsize_validate](#test-291) +- [Test 292: managedlustre_fs_update](#test-292) +- [Test 293: marketplace_product_get](#test-293) +- [Test 294: marketplace_product_list](#test-294) +- [Test 295: marketplace_product_list](#test-295) +- [Test 296: get_bestpractices_get](#test-296) +- [Test 297: get_bestpractices_get](#test-297) +- [Test 298: get_bestpractices_get](#test-298) +- [Test 299: get_bestpractices_get](#test-299) +- [Test 300: get_bestpractices_get](#test-300) +- [Test 301: get_bestpractices_get](#test-301) +- [Test 302: get_bestpractices_get](#test-302) +- [Test 303: get_bestpractices_get](#test-303) +- [Test 304: get_bestpractices_get](#test-304) +- [Test 305: monitor_activitylog_list](#test-305) +- [Test 306: monitor_healthmodels_entity_get](#test-306) +- [Test 307: monitor_metrics_definitions](#test-307) +- [Test 308: monitor_metrics_definitions](#test-308) +- [Test 309: monitor_metrics_definitions](#test-309) +- [Test 310: monitor_metrics_query](#test-310) +- [Test 311: monitor_metrics_query](#test-311) +- [Test 312: monitor_metrics_query](#test-312) +- [Test 313: monitor_metrics_query](#test-313) +- [Test 314: monitor_metrics_query](#test-314) +- [Test 315: monitor_metrics_query](#test-315) +- [Test 316: monitor_resource_log_query](#test-316) +- [Test 317: monitor_table_list](#test-317) +- [Test 318: monitor_table_list](#test-318) +- [Test 319: monitor_table_type_list](#test-319) +- [Test 320: monitor_table_type_list](#test-320) +- [Test 321: monitor_webtests_create](#test-321) +- [Test 322: monitor_webtests_get](#test-322) +- [Test 323: monitor_webtests_list](#test-323) +- [Test 324: monitor_webtests_list](#test-324) +- [Test 325: monitor_webtests_update](#test-325) +- [Test 326: monitor_workspace_list](#test-326) +- [Test 327: monitor_workspace_list](#test-327) +- [Test 328: monitor_workspace_list](#test-328) +- [Test 329: monitor_workspace_log_query](#test-329) +- [Test 330: datadog_monitoredresources_list](#test-330) +- [Test 331: datadog_monitoredresources_list](#test-331) +- [Test 332: extension_azqr](#test-332) +- [Test 333: extension_azqr](#test-333) +- [Test 334: extension_azqr](#test-334) +- [Test 335: quota_region_availability_list](#test-335) +- [Test 336: quota_usage_check](#test-336) +- [Test 337: role_assignment_list](#test-337) +- [Test 338: role_assignment_list](#test-338) +- [Test 339: redis_create](#test-339) +- [Test 340: redis_create](#test-340) +- [Test 341: redis_create](#test-341) +- [Test 342: redis_create](#test-342) +- [Test 343: redis_list](#test-343) +- [Test 344: redis_list](#test-344) +- [Test 345: redis_list](#test-345) +- [Test 346: redis_list](#test-346) +- [Test 347: redis_list](#test-347) +- [Test 348: group_list](#test-348) +- [Test 349: group_list](#test-349) +- [Test 350: group_list](#test-350) +- [Test 351: resourcehealth_availability-status_get](#test-351) +- [Test 352: resourcehealth_availability-status_get](#test-352) +- [Test 353: resourcehealth_availability-status_get](#test-353) +- [Test 354: resourcehealth_availability-status_list](#test-354) +- [Test 355: resourcehealth_availability-status_list](#test-355) +- [Test 356: resourcehealth_availability-status_list](#test-356) +- [Test 357: resourcehealth_health-events_list](#test-357) +- [Test 358: resourcehealth_health-events_list](#test-358) +- [Test 359: resourcehealth_health-events_list](#test-359) +- [Test 360: resourcehealth_health-events_list](#test-360) +- [Test 361: resourcehealth_health-events_list](#test-361) +- [Test 362: servicebus_queue_details](#test-362) +- [Test 363: servicebus_topic_details](#test-363) +- [Test 364: servicebus_topic_subscription_details](#test-364) +- [Test 365: signalr_runtime_get](#test-365) +- [Test 366: signalr_runtime_get](#test-366) +- [Test 367: signalr_runtime_get](#test-367) +- [Test 368: signalr_runtime_get](#test-368) +- [Test 369: signalr_runtime_get](#test-369) +- [Test 370: signalr_runtime_get](#test-370) +- [Test 371: sql_db_create](#test-371) +- [Test 372: sql_db_create](#test-372) +- [Test 373: sql_db_create](#test-373) +- [Test 374: sql_db_delete](#test-374) +- [Test 375: sql_db_delete](#test-375) +- [Test 376: sql_db_delete](#test-376) +- [Test 377: sql_db_list](#test-377) +- [Test 378: sql_db_list](#test-378) +- [Test 379: sql_db_rename](#test-379) +- [Test 380: sql_db_rename](#test-380) +- [Test 381: sql_db_show](#test-381) +- [Test 382: sql_db_show](#test-382) +- [Test 383: sql_db_update](#test-383) +- [Test 384: sql_db_update](#test-384) +- [Test 385: sql_elastic-pool_list](#test-385) +- [Test 386: sql_elastic-pool_list](#test-386) +- [Test 387: sql_elastic-pool_list](#test-387) +- [Test 388: sql_server_create](#test-388) +- [Test 389: sql_server_create](#test-389) +- [Test 390: sql_server_create](#test-390) +- [Test 391: sql_server_delete](#test-391) +- [Test 392: sql_server_delete](#test-392) +- [Test 393: sql_server_delete](#test-393) +- [Test 394: sql_server_entra-admin_list](#test-394) +- [Test 395: sql_server_entra-admin_list](#test-395) +- [Test 396: sql_server_entra-admin_list](#test-396) +- [Test 397: sql_server_firewall-rule_create](#test-397) +- [Test 398: sql_server_firewall-rule_create](#test-398) +- [Test 399: sql_server_firewall-rule_create](#test-399) +- [Test 400: sql_server_firewall-rule_delete](#test-400) +- [Test 401: sql_server_firewall-rule_delete](#test-401) +- [Test 402: sql_server_firewall-rule_delete](#test-402) +- [Test 403: sql_server_firewall-rule_list](#test-403) +- [Test 404: sql_server_firewall-rule_list](#test-404) +- [Test 405: sql_server_firewall-rule_list](#test-405) +- [Test 406: sql_server_list](#test-406) +- [Test 407: sql_server_list](#test-407) +- [Test 408: sql_server_show](#test-408) +- [Test 409: sql_server_show](#test-409) +- [Test 410: sql_server_show](#test-410) +- [Test 411: storage_account_create](#test-411) +- [Test 412: storage_account_create](#test-412) +- [Test 413: storage_account_create](#test-413) +- [Test 414: storage_account_get](#test-414) +- [Test 415: storage_account_get](#test-415) +- [Test 416: storage_account_get](#test-416) +- [Test 417: storage_account_get](#test-417) +- [Test 418: storage_account_get](#test-418) +- [Test 419: storage_blob_container_create](#test-419) +- [Test 420: storage_blob_container_create](#test-420) +- [Test 421: storage_blob_container_create](#test-421) +- [Test 422: storage_blob_container_get](#test-422) +- [Test 423: storage_blob_container_get](#test-423) +- [Test 424: storage_blob_container_get](#test-424) +- [Test 425: storage_blob_get](#test-425) +- [Test 426: storage_blob_get](#test-426) +- [Test 427: storage_blob_get](#test-427) +- [Test 428: storage_blob_get](#test-428) +- [Test 429: storage_blob_upload](#test-429) +- [Test 430: subscription_list](#test-430) +- [Test 431: subscription_list](#test-431) +- [Test 432: subscription_list](#test-432) +- [Test 433: subscription_list](#test-433) +- [Test 434: azureterraformbestpractices_get](#test-434) +- [Test 435: azureterraformbestpractices_get](#test-435) +- [Test 436: virtualdesktop_hostpool_list](#test-436) +- [Test 437: virtualdesktop_hostpool_host_list](#test-437) +- [Test 438: virtualdesktop_hostpool_host_user-list](#test-438) +- [Test 439: workbooks_create](#test-439) +- [Test 440: workbooks_delete](#test-440) +- [Test 441: workbooks_list](#test-441) +- [Test 442: workbooks_list](#test-442) +- [Test 443: workbooks_show](#test-443) +- [Test 444: workbooks_show](#test-444) +- [Test 445: workbooks_update](#test-445) +- [Test 446: bicepschema_get](#test-446) +- [Test 447: cloudarchitect_design](#test-447) +- [Test 448: cloudarchitect_design](#test-448) +- [Test 449: cloudarchitect_design](#test-449) +- [Test 450: cloudarchitect_design](#test-450) +- [Test 451: foundry_agents_connect](#test-451) +- [Test 452: foundry_agents_create](#test-452) +- [Test 453: foundry_agents_evaluate](#test-453) +- [Test 454: foundry_agents_get-sdk-sample](#test-454) +- [Test 455: foundry_agents_list](#test-455) +- [Test 456: foundry_agents_list](#test-456) +- [Test 457: foundry_agents_query-and-evaluate](#test-457) +- [Test 458: foundry_knowledge_index_list](#test-458) +- [Test 459: foundry_knowledge_index_list](#test-459) +- [Test 460: foundry_knowledge_index_schema](#test-460) +- [Test 461: foundry_knowledge_index_schema](#test-461) +- [Test 462: foundry_models_deploy](#test-462) +- [Test 463: foundry_models_deployments_list](#test-463) +- [Test 464: foundry_models_deployments_list](#test-464) +- [Test 465: foundry_models_list](#test-465) +- [Test 466: foundry_models_list](#test-466) +- [Test 467: foundry_openai_chat-completions-create](#test-467) +- [Test 468: foundry_openai_create-completion](#test-468) +- [Test 469: foundry_openai_embeddings-create](#test-469) +- [Test 470: foundry_openai_embeddings-create](#test-470) +- [Test 471: foundry_openai_models-list](#test-471) +- [Test 472: foundry_openai_models-list](#test-472) +- [Test 473: foundry_resource_get](#test-473) +- [Test 474: foundry_resource_get](#test-474) +- [Test 475: foundry_resource_get](#test-475) +- [Test 476: foundry_threads_create](#test-476) +- [Test 477: foundry_threads_get-messages](#test-477) +- [Test 478: foundry_threads_list](#test-478) --- @@ -503,11 +503,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.665954 | `azureaibestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.555579 | `get_bestpractices_get` | ❌ | -| 3 | 0.501210 | `azureterraformbestpractices_get` | ❌ | -| 4 | 0.480235 | `deploy_pipeline_guidance_get` | ❌ | -| 5 | 0.477592 | `cloudarchitect_design` | ❌ | +| 1 | 0.901707 | `azureaibestpractices_get` | ✅ **EXPECTED** | +| 2 | 0.849188 | `get_bestpractices_get` | ❌ | +| 3 | 0.837780 | `cloudarchitect_design` | ❌ | +| 4 | 0.837172 | `deploy_iac_rules_get` | ❌ | +| 5 | 0.828440 | `deploy_pipeline_guidance_get` | ❌ | --- @@ -520,11 +520,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.686352 | `azureaibestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.603773 | `foundry_agents_get-sdk-sample` | ❌ | -| 3 | 0.534202 | `get_bestpractices_get` | ❌ | -| 4 | 0.520223 | `foundry_agents_list` | ❌ | -| 5 | 0.508727 | `azureterraformbestpractices_get` | ❌ | +| 1 | 0.902039 | `foundry_agents_get-sdk-sample` | ❌ | +| 2 | 0.852729 | `foundry_threads_list` | ❌ | +| 3 | 0.840573 | `foundry_threads_get-messages` | ❌ | +| 4 | 0.839788 | `azureaibestpractices_get` | ✅ **EXPECTED** | +| 5 | 0.826004 | `foundry_agents_create` | ❌ | --- @@ -537,11 +537,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.633128 | `azureaibestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.593216 | `foundry_agents_get-sdk-sample` | ❌ | -| 3 | 0.553662 | `foundry_agents_list` | ❌ | -| 4 | 0.534256 | `foundry_agents_create` | ❌ | -| 5 | 0.513217 | `foundry_agents_connect` | ❌ | +| 1 | 0.881602 | `foundry_agents_get-sdk-sample` | ❌ | +| 2 | 0.847658 | `foundry_threads_list` | ❌ | +| 3 | 0.844244 | `foundry_agents_create` | ❌ | +| 4 | 0.838805 | `foundry_threads_get-messages` | ❌ | +| 5 | 0.832386 | `azureaibestpractices_get` | ✅ **EXPECTED** | --- @@ -554,11 +554,11 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.427818 | `azureaibestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.343877 | `foundry_threads_create` | ❌ | -| 3 | 0.327503 | `foundry_openai_chat-completions-create` | ❌ | -| 4 | 0.320532 | `foundry_openai_create-completion` | ❌ | -| 5 | 0.311958 | `foundry_agents_connect` | ❌ | +| 1 | 0.814525 | `azureaibestpractices_get` | ✅ **EXPECTED** | +| 2 | 0.782747 | `cloudarchitect_design` | ❌ | +| 3 | 0.782239 | `deploy_app_logs_get` | ❌ | +| 4 | 0.776789 | `foundry_agents_get-sdk-sample` | ❌ | +| 5 | 0.774939 | `search_index_query` | ❌ | --- @@ -571,8084 +571,8084 @@ | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.527803 | `azureaibestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.478745 | `foundry_openai_embeddings-create` | ❌ | -| 3 | 0.469654 | `foundry_openai_chat-completions-create` | ❌ | -| 4 | 0.466216 | `foundry_openai_create-completion` | ❌ | -| 5 | 0.456719 | `foundry_resource_get` | ❌ | +| 1 | 0.851597 | `foundry_agents_get-sdk-sample` | ❌ | +| 2 | 0.847705 | `azureaibestpractices_get` | ✅ **EXPECTED** | +| 3 | 0.845440 | `foundry_openai_create-completion` | ❌ | +| 4 | 0.843237 | `foundry_openai_chat-completions-create` | ❌ | +| 5 | 0.833031 | `foundry_openai_embeddings-create` | ❌ | --- ## Test 6 -**Expected Tool:** `foundry_agents_connect` -**Prompt:** Query an agent in my Microsoft Foundry resource +**Expected Tool:** `search_knowledge_base_get` +**Prompt:** List all knowledge bases in the Azure AI Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.705410 | `foundry_agents_connect` | ✅ **EXPECTED** | -| 2 | 0.663568 | `foundry_agents_list` | ❌ | -| 3 | 0.617213 | `foundry_resource_get` | ❌ | -| 4 | 0.548108 | `foundry_agents_get-sdk-sample` | ❌ | -| 5 | 0.548044 | `foundry_openai_models-list` | ❌ | +| 1 | 0.883581 | `search_service_list` | ❌ | +| 2 | 0.874295 | `search_knowledge_base_get` | ✅ **EXPECTED** | +| 3 | 0.845190 | `search_knowledge_base_retrieve` | ❌ | +| 4 | 0.835427 | `search_index_query` | ❌ | +| 5 | 0.831530 | `search_knowledge_source_get` | ❌ | --- ## Test 7 -**Expected Tool:** `foundry_agents_create` -**Prompt:** Create a new Microsoft Foundry agent using instructions in the active editor +**Expected Tool:** `search_knowledge_base_get` +**Prompt:** Show me the knowledge bases in the Azure AI Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.587064 | `foundry_agents_create` | ✅ **EXPECTED** | -| 2 | 0.562087 | `foundry_agents_get-sdk-sample` | ❌ | -| 3 | 0.554032 | `foundry_threads_create` | ❌ | -| 4 | 0.525727 | `foundry_models_deploy` | ❌ | -| 5 | 0.525615 | `foundry_agents_list` | ❌ | +| 1 | 0.861872 | `search_service_list` | ❌ | +| 2 | 0.856137 | `search_knowledge_base_get` | ✅ **EXPECTED** | +| 3 | 0.843258 | `search_knowledge_base_retrieve` | ❌ | +| 4 | 0.839597 | `search_index_query` | ❌ | +| 5 | 0.823722 | `search_index_get` | ❌ | --- ## Test 8 -**Expected Tool:** `foundry_agents_evaluate` -**Prompt:** Evaluate the full query and response I got from my agent for task_adherence +**Expected Tool:** `search_knowledge_base_get` +**Prompt:** List all knowledge bases in the search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.544163 | `foundry_agents_query-and-evaluate` | ❌ | -| 2 | 0.469428 | `foundry_agents_evaluate` | ✅ **EXPECTED** | -| 3 | 0.445964 | `foundry_agents_connect` | ❌ | -| 4 | 0.297986 | `foundry_threads_list` | ❌ | -| 5 | 0.278920 | `foundry_agents_list` | ❌ | +| 1 | 0.851712 | `search_knowledge_base_get` | ✅ **EXPECTED** | +| 2 | 0.827051 | `search_knowledge_base_retrieve` | ❌ | +| 3 | 0.819336 | `search_service_list` | ❌ | +| 4 | 0.798747 | `search_knowledge_source_get` | ❌ | +| 5 | 0.796343 | `postgres_database_list` | ❌ | --- ## Test 9 -**Expected Tool:** `foundry_agents_get-sdk-sample` -**Prompt:** Create a CLI app that can talk to an Microsoft Foundry Agent using Python SDK +**Expected Tool:** `search_knowledge_base_get` +**Prompt:** Show me the knowledge bases in the search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.595844 | `foundry_agents_get-sdk-sample` | ✅ **EXPECTED** | -| 2 | 0.552348 | `foundry_threads_create` | ❌ | -| 3 | 0.522041 | `foundry_agents_connect` | ❌ | -| 4 | 0.518567 | `foundry_agents_create` | ❌ | -| 5 | 0.510001 | `foundry_agents_list` | ❌ | +| 1 | 0.830523 | `search_knowledge_base_get` | ✅ **EXPECTED** | +| 2 | 0.820975 | `search_knowledge_base_retrieve` | ❌ | +| 3 | 0.802298 | `search_service_list` | ❌ | +| 4 | 0.779583 | `search_knowledge_source_get` | ❌ | +| 5 | 0.778498 | `aks_cluster_get` | ❌ | --- ## Test 10 -**Expected Tool:** `foundry_agents_list` -**Prompt:** List all agents in my Microsoft Foundry resource +**Expected Tool:** `search_knowledge_base_get` +**Prompt:** Get the details of knowledge base in the Azure AI Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.797877 | `foundry_agents_list` | ✅ **EXPECTED** | -| 2 | 0.666021 | `foundry_resource_get` | ❌ | -| 3 | 0.654206 | `foundry_openai_models-list` | ❌ | -| 4 | 0.647246 | `foundry_threads_list` | ❌ | -| 5 | 0.575553 | `foundry_models_deployments_list` | ❌ | +| 1 | 0.857497 | `search_knowledge_base_get` | ✅ **EXPECTED** | +| 2 | 0.855167 | `search_service_list` | ❌ | +| 3 | 0.842705 | `search_index_query` | ❌ | +| 4 | 0.840920 | `search_index_get` | ❌ | +| 5 | 0.838583 | `search_knowledge_base_retrieve` | ❌ | --- ## Test 11 -**Expected Tool:** `foundry_agents_list` -**Prompt:** Show me the available agents in my Microsoft Foundry resource +**Expected Tool:** `search_knowledge_base_get` +**Prompt:** Show me the knowledge base in search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.749829 | `foundry_agents_list` | ✅ **EXPECTED** | -| 2 | 0.630288 | `foundry_resource_get` | ❌ | -| 3 | 0.611722 | `foundry_openai_models-list` | ❌ | -| 4 | 0.603689 | `foundry_threads_list` | ❌ | -| 5 | 0.556990 | `foundry_agents_get-sdk-sample` | ❌ | +| 1 | 0.818518 | `search_knowledge_base_retrieve` | ❌ | +| 2 | 0.805246 | `search_knowledge_base_get` | ✅ **EXPECTED** | +| 3 | 0.799337 | `search_service_list` | ❌ | +| 4 | 0.778408 | `foundry_agents_get-sdk-sample` | ❌ | +| 5 | 0.771652 | `foundry_agents_connect` | ❌ | --- ## Test 12 -**Expected Tool:** `foundry_agents_query-and-evaluate` -**Prompt:** Query and evaluate an agent in my Microsoft Foundry resource for task_adherence +**Expected Tool:** `search_knowledge_base_retrieve` +**Prompt:** Run a retrieval with knowledge base in Azure AI Search service for the query ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.652200 | `foundry_agents_connect` | ❌ | -| 2 | 0.570787 | `foundry_agents_list` | ❌ | -| 3 | 0.553282 | `foundry_agents_query-and-evaluate` | ✅ **EXPECTED** | -| 4 | 0.493778 | `foundry_agents_evaluate` | ❌ | -| 5 | 0.469431 | `foundry_threads_list` | ❌ | +| 1 | 0.867527 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | +| 2 | 0.845382 | `search_index_query` | ❌ | +| 3 | 0.817868 | `foundry_agents_connect` | ❌ | +| 4 | 0.810555 | `search_service_list` | ❌ | +| 5 | 0.804040 | `postgres_database_query` | ❌ | --- ## Test 13 -**Expected Tool:** `foundry_knowledge_index_list` -**Prompt:** List all knowledge indexes in my Microsoft Foundry project +**Expected Tool:** `search_knowledge_base_retrieve` +**Prompt:** Ask knowledge base in search service to retrieve information about ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.703772 | `foundry_knowledge_index_list` | ✅ **EXPECTED** | -| 2 | 0.537700 | `foundry_agents_list` | ❌ | -| 3 | 0.526528 | `foundry_knowledge_index_schema` | ❌ | -| 4 | 0.500786 | `foundry_threads_list` | ❌ | -| 5 | 0.475802 | `foundry_models_deployments_list` | ❌ | +| 1 | 0.834849 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | +| 2 | 0.805058 | `foundry_agents_connect` | ❌ | +| 3 | 0.794166 | `search_knowledge_base_get` | ❌ | +| 4 | 0.783138 | `search_index_query` | ❌ | +| 5 | 0.779341 | `foundry_agents_query-and-evaluate` | ❌ | --- ## Test 14 -**Expected Tool:** `foundry_knowledge_index_list` -**Prompt:** Show me the knowledge indexes in my Microsoft Foundry project +**Expected Tool:** `search_knowledge_base_retrieve` +**Prompt:** Run a retrieval with knowledge base in search service for the query ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.615458 | `foundry_knowledge_index_list` | ✅ **EXPECTED** | -| 2 | 0.489311 | `foundry_knowledge_index_schema` | ❌ | -| 3 | 0.484466 | `foundry_agents_list` | ❌ | -| 4 | 0.454174 | `foundry_threads_list` | ❌ | -| 5 | 0.441521 | `foundry_resource_get` | ❌ | +| 1 | 0.855967 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | +| 2 | 0.807722 | `foundry_agents_connect` | ❌ | +| 3 | 0.789843 | `foundry_agents_evaluate` | ❌ | +| 4 | 0.786866 | `foundry_agents_query-and-evaluate` | ❌ | +| 5 | 0.783063 | `postgres_database_query` | ❌ | --- ## Test 15 -**Expected Tool:** `foundry_knowledge_index_schema` -**Prompt:** Show me the schema for knowledge index in my Microsoft Foundry resource +**Expected Tool:** `search_knowledge_base_retrieve` +**Prompt:** Ask knowledge base in search service to retrieve information about ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.739885 | `foundry_knowledge_index_schema` | ✅ **EXPECTED** | -| 2 | 0.589536 | `foundry_knowledge_index_list` | ❌ | -| 3 | 0.494004 | `foundry_resource_get` | ❌ | -| 4 | 0.491510 | `search_index_get` | ❌ | -| 5 | 0.490410 | `search_knowledge_base_get` | ❌ | +| 1 | 0.834862 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | +| 2 | 0.805012 | `foundry_agents_connect` | ❌ | +| 3 | 0.794203 | `search_knowledge_base_get` | ❌ | +| 4 | 0.783073 | `search_index_query` | ❌ | +| 5 | 0.779336 | `foundry_agents_query-and-evaluate` | ❌ | --- ## Test 16 -**Expected Tool:** `foundry_knowledge_index_schema` -**Prompt:** Get the schema configuration for knowledge index +**Expected Tool:** `search_knowledge_base_retrieve` +**Prompt:** Query knowledge base in search service about ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.650269 | `foundry_knowledge_index_schema` | ✅ **EXPECTED** | -| 2 | 0.432758 | `postgres_table_schema_get` | ❌ | -| 3 | 0.417421 | `kusto_table_schema` | ❌ | -| 4 | 0.398186 | `mysql_table_schema_get` | ❌ | -| 5 | 0.396194 | `foundry_knowledge_index_list` | ❌ | +| 1 | 0.826851 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | +| 2 | 0.808859 | `foundry_agents_connect` | ❌ | +| 3 | 0.795991 | `search_index_query` | ❌ | +| 4 | 0.791363 | `kusto_query` | ❌ | +| 5 | 0.790295 | `postgres_database_query` | ❌ | --- ## Test 17 -**Expected Tool:** `foundry_models_deploy` -**Prompt:** Deploy a GPT4o instance on my resource +**Expected Tool:** `search_knowledge_base_retrieve` +**Prompt:** Search knowledge base in Azure AI Search service for ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.562920 | `foundry_models_deploy` | ✅ **EXPECTED** | -| 2 | 0.299986 | `foundry_openai_models-list` | ❌ | -| 3 | 0.298490 | `loadtesting_testrun_create` | ❌ | -| 4 | 0.293050 | `loadtesting_testresource_create` | ❌ | -| 5 | 0.290381 | `foundry_openai_embeddings-create` | ❌ | +| 1 | 0.855299 | `search_index_query` | ❌ | +| 2 | 0.837748 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | +| 3 | 0.836542 | `search_service_list` | ❌ | +| 4 | 0.807468 | `azureaibestpractices_get` | ❌ | +| 5 | 0.803245 | `postgres_database_query` | ❌ | --- ## Test 18 -**Expected Tool:** `foundry_models_deployments_list` -**Prompt:** List all Microsoft Foundry model deployments +**Expected Tool:** `search_knowledge_base_retrieve` +**Prompt:** What does knowledge base in search service know about ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.681385 | `foundry_models_deployments_list` | ✅ **EXPECTED** | -| 2 | 0.674510 | `foundry_openai_models-list` | ❌ | -| 3 | 0.572625 | `foundry_threads_list` | ❌ | -| 4 | 0.569058 | `foundry_agents_list` | ❌ | -| 5 | 0.566272 | `foundry_resource_get` | ❌ | +| 1 | 0.810631 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | +| 2 | 0.789111 | `foundry_agents_connect` | ❌ | +| 3 | 0.777486 | `search_knowledge_base_get` | ❌ | +| 4 | 0.773482 | `search_index_query` | ❌ | +| 5 | 0.764773 | `foundry_agents_query-and-evaluate` | ❌ | --- ## Test 19 -**Expected Tool:** `foundry_models_deployments_list` -**Prompt:** Show me all Microsoft Foundry model deployments +**Expected Tool:** `search_knowledge_base_retrieve` +**Prompt:** Find information about using knowledge base in search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.620173 | `foundry_models_deployments_list` | ✅ **EXPECTED** | -| 2 | 0.619231 | `foundry_openai_models-list` | ❌ | -| 3 | 0.543352 | `foundry_resource_get` | ❌ | -| 4 | 0.540551 | `foundry_agents_list` | ❌ | -| 5 | 0.527121 | `foundry_threads_list` | ❌ | +| 1 | 0.828871 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | +| 2 | 0.796520 | `foundry_agents_connect` | ❌ | +| 3 | 0.789486 | `search_knowledge_base_get` | ❌ | +| 4 | 0.782309 | `kusto_query` | ❌ | +| 5 | 0.781076 | `search_index_query` | ❌ | --- ## Test 20 -**Expected Tool:** `foundry_models_list` -**Prompt:** List all Microsoft Foundry models +**Expected Tool:** `search_knowledge_source_get` +**Prompt:** List all knowledge sources in the Azure AI Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.603415 | `foundry_openai_models-list` | ❌ | -| 2 | 0.560022 | `foundry_models_list` | ✅ **EXPECTED** | -| 3 | 0.553634 | `foundry_threads_list` | ❌ | -| 4 | 0.537980 | `foundry_models_deployments_list` | ❌ | -| 5 | 0.519472 | `foundry_agents_list` | ❌ | +| 1 | 0.888833 | `search_service_list` | ❌ | +| 2 | 0.870413 | `search_knowledge_source_get` | ✅ **EXPECTED** | +| 3 | 0.842855 | `search_knowledge_base_get` | ❌ | +| 4 | 0.842165 | `search_index_query` | ❌ | +| 5 | 0.829867 | `search_index_get` | ❌ | --- ## Test 21 -**Expected Tool:** `foundry_models_list` -**Prompt:** Show me the available Microsoft Foundry models +**Expected Tool:** `search_knowledge_source_get` +**Prompt:** Show me the knowledge sources in the Azure AI Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.576904 | `foundry_openai_models-list` | ❌ | -| 2 | 0.574818 | `foundry_models_list` | ✅ **EXPECTED** | -| 3 | 0.525312 | `foundry_resource_get` | ❌ | -| 4 | 0.521473 | `foundry_agents_get-sdk-sample` | ❌ | -| 5 | 0.517980 | `foundry_models_deployments_list` | ❌ | +| 1 | 0.868552 | `search_service_list` | ❌ | +| 2 | 0.851623 | `search_knowledge_source_get` | ✅ **EXPECTED** | +| 3 | 0.842769 | `search_index_query` | ❌ | +| 4 | 0.830612 | `search_knowledge_base_get` | ❌ | +| 5 | 0.827743 | `search_knowledge_base_retrieve` | ❌ | --- ## Test 22 -**Expected Tool:** `foundry_openai_chat-completions-create` -**Prompt:** Create a chat completion with the message "Hello, how are you today?" using my Microsoft Foundry resource +**Expected Tool:** `search_knowledge_source_get` +**Prompt:** List all knowledge sources in the search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.641302 | `foundry_openai_chat-completions-create` | ✅ **EXPECTED** | -| 2 | 0.546738 | `foundry_openai_create-completion` | ❌ | -| 3 | 0.420028 | `foundry_threads_create` | ❌ | -| 4 | 0.415490 | `foundry_agents_connect` | ❌ | -| 5 | 0.407194 | `azureaibestpractices_get` | ❌ | +| 1 | 0.836042 | `search_knowledge_source_get` | ✅ **EXPECTED** | +| 2 | 0.817888 | `search_service_list` | ❌ | +| 3 | 0.810200 | `search_knowledge_base_get` | ❌ | +| 4 | 0.804014 | `search_knowledge_base_retrieve` | ❌ | +| 5 | 0.780384 | `aks_cluster_get` | ❌ | --- ## Test 23 -**Expected Tool:** `foundry_openai_create-completion` -**Prompt:** Create a completion with the prompt "What is Azure?" using my Microsoft Foundry resource +**Expected Tool:** `search_knowledge_source_get` +**Prompt:** Show me the knowledge sources in the search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.696936 | `foundry_openai_create-completion` | ✅ **EXPECTED** | -| 2 | 0.579108 | `foundry_openai_chat-completions-create` | ❌ | -| 3 | 0.475862 | `azureaibestpractices_get` | ❌ | -| 4 | 0.463703 | `foundry_models_deploy` | ❌ | -| 5 | 0.459126 | `foundry_resource_get` | ❌ | +| 1 | 0.810764 | `search_knowledge_source_get` | ✅ **EXPECTED** | +| 2 | 0.803303 | `search_service_list` | ❌ | +| 3 | 0.795723 | `search_knowledge_base_retrieve` | ❌ | +| 4 | 0.792056 | `search_knowledge_base_get` | ❌ | +| 5 | 0.774091 | `applens_resource_diagnose` | ❌ | --- ## Test 24 -**Expected Tool:** `foundry_openai_embeddings-create` -**Prompt:** Generate embeddings for the text "Azure OpenAI Service" using my Microsoft Foundry resource +**Expected Tool:** `search_knowledge_source_get` +**Prompt:** Get the details of knowledge source in the Azure AI Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.766338 | `foundry_openai_embeddings-create` | ✅ **EXPECTED** | -| 2 | 0.543339 | `foundry_models_deploy` | ❌ | -| 3 | 0.542214 | `foundry_openai_create-completion` | ❌ | -| 4 | 0.520746 | `foundry_openai_models-list` | ❌ | -| 5 | 0.519335 | `foundry_resource_get` | ❌ | +| 1 | 0.880082 | `search_knowledge_source_get` | ✅ **EXPECTED** | +| 2 | 0.849831 | `search_service_list` | ❌ | +| 3 | 0.842576 | `search_index_get` | ❌ | +| 4 | 0.841171 | `search_knowledge_base_get` | ❌ | +| 5 | 0.834825 | `search_index_query` | ❌ | --- ## Test 25 -**Expected Tool:** `foundry_openai_embeddings-create` -**Prompt:** Create vector embeddings for my text using my Microsoft Foundry resource +**Expected Tool:** `search_knowledge_source_get` +**Prompt:** Show me the knowledge source in search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.724120 | `foundry_openai_embeddings-create` | ✅ **EXPECTED** | -| 2 | 0.494485 | `foundry_resource_get` | ❌ | -| 3 | 0.480296 | `foundry_models_deploy` | ❌ | -| 4 | 0.480218 | `foundry_openai_create-completion` | ❌ | -| 5 | 0.463797 | `foundry_openai_chat-completions-create` | ❌ | +| 1 | 0.801259 | `search_knowledge_source_get` | ✅ **EXPECTED** | +| 2 | 0.785640 | `search_knowledge_base_retrieve` | ❌ | +| 3 | 0.780791 | `search_service_list` | ❌ | +| 4 | 0.769001 | `search_knowledge_base_get` | ❌ | +| 5 | 0.760896 | `postgres_database_query` | ❌ | --- ## Test 26 -**Expected Tool:** `foundry_openai_models-list` -**Prompt:** List all available OpenAI models in my Microsoft Foundry resource +**Expected Tool:** `search_index_get` +**Prompt:** Show me the details of the index in Cognitive Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.799059 | `foundry_openai_models-list` | ✅ **EXPECTED** | -| 2 | 0.668887 | `foundry_resource_get` | ❌ | -| 3 | 0.667041 | `foundry_models_list` | ❌ | -| 4 | 0.666208 | `foundry_models_deployments_list` | ❌ | -| 5 | 0.657545 | `foundry_agents_list` | ❌ | +| 1 | 0.842818 | `search_index_get` | ✅ **EXPECTED** | +| 2 | 0.819586 | `search_service_list` | ❌ | +| 3 | 0.818564 | `search_index_query` | ❌ | +| 4 | 0.801176 | `foundry_resource_get` | ❌ | +| 5 | 0.778269 | `kusto_table_schema` | ❌ | --- ## Test 27 -**Expected Tool:** `foundry_openai_models-list` -**Prompt:** Show me the OpenAI model deployments in my Microsoft Foundry resource +**Expected Tool:** `search_index_get` +**Prompt:** List all indexes in the Cognitive Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.741659 | `foundry_openai_models-list` | ✅ **EXPECTED** | -| 2 | 0.660160 | `foundry_models_deployments_list` | ❌ | -| 3 | 0.648218 | `foundry_resource_get` | ❌ | -| 4 | 0.640650 | `foundry_models_deploy` | ❌ | -| 5 | 0.619878 | `foundry_agents_list` | ❌ | +| 1 | 0.835503 | `search_service_list` | ❌ | +| 2 | 0.820775 | `search_index_get` | ✅ **EXPECTED** | +| 3 | 0.794226 | `search_index_query` | ❌ | +| 4 | 0.783625 | `kusto_cluster_list` | ❌ | +| 5 | 0.772750 | `foundry_resource_get` | ❌ | --- ## Test 28 -**Expected Tool:** `foundry_resource_get` -**Prompt:** List all Microsoft Foundry resources in my subscription +**Expected Tool:** `search_index_get` +**Prompt:** Show me the indexes in the Cognitive Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.594096 | `foundry_resource_get` | ✅ **EXPECTED** | -| 2 | 0.571916 | `foundry_openai_models-list` | ❌ | -| 3 | 0.567019 | `foundry_agents_list` | ❌ | -| 4 | 0.558290 | `search_service_list` | ❌ | -| 5 | 0.558075 | `foundry_threads_list` | ❌ | +| 1 | 0.822042 | `search_service_list` | ❌ | +| 2 | 0.816723 | `search_index_get` | ✅ **EXPECTED** | +| 3 | 0.800532 | `search_index_query` | ❌ | +| 4 | 0.779781 | `foundry_resource_get` | ❌ | +| 5 | 0.776571 | `foundry_models_deployments_list` | ❌ | --- ## Test 29 -**Expected Tool:** `foundry_resource_get` -**Prompt:** Show me the Microsoft Foundry resources in resource group +**Expected Tool:** `search_index_query` +**Prompt:** Search for instances of in the index in Cognitive Search service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.665349 | `foundry_resource_get` | ✅ **EXPECTED** | -| 2 | 0.585338 | `foundry_openai_models-list` | ❌ | -| 3 | 0.554004 | `foundry_agents_list` | ❌ | -| 4 | 0.518757 | `foundry_openai_embeddings-create` | ❌ | -| 5 | 0.492909 | `foundry_models_deploy` | ❌ | +| 1 | 0.811322 | `search_index_query` | ✅ **EXPECTED** | +| 2 | 0.803180 | `search_service_list` | ❌ | +| 3 | 0.782806 | `postgres_database_query` | ❌ | +| 4 | 0.779808 | `search_knowledge_base_retrieve` | ❌ | +| 5 | 0.779115 | `kusto_query` | ❌ | --- ## Test 30 -**Expected Tool:** `foundry_resource_get` -**Prompt:** Get details for Microsoft Foundry resource in resource group +**Expected Tool:** `search_service_list` +**Prompt:** List all Cognitive Search services in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.735316 | `foundry_resource_get` | ✅ **EXPECTED** | -| 2 | 0.571906 | `foundry_openai_models-list` | ❌ | -| 3 | 0.509855 | `monitor_webtests_get` | ❌ | -| 4 | 0.497090 | `foundry_openai_embeddings-create` | ❌ | -| 5 | 0.475722 | `foundry_agents_list` | ❌ | +| 1 | 0.893047 | `search_service_list` | ✅ **EXPECTED** | +| 2 | 0.816139 | `kusto_cluster_list` | ❌ | +| 3 | 0.805669 | `redis_list` | ❌ | +| 4 | 0.805532 | `foundry_resource_get` | ❌ | +| 5 | 0.792097 | `marketplace_product_list` | ❌ | --- ## Test 31 -**Expected Tool:** `foundry_threads_create` -**Prompt:** Create an Microsoft Foundry thread to hold the conversation +**Expected Tool:** `search_service_list` +**Prompt:** Show me the Cognitive Search services in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.606794 | `foundry_threads_create` | ✅ **EXPECTED** | -| 2 | 0.528310 | `foundry_openai_chat-completions-create` | ❌ | -| 3 | 0.519709 | `foundry_threads_get-messages` | ❌ | -| 4 | 0.506089 | `foundry_threads_list` | ❌ | -| 5 | 0.490796 | `foundry_models_deploy` | ❌ | +| 1 | 0.863446 | `search_service_list` | ✅ **EXPECTED** | +| 2 | 0.806040 | `redis_list` | ❌ | +| 3 | 0.804274 | `eventgrid_subscription_list` | ❌ | +| 4 | 0.804187 | `foundry_resource_get` | ❌ | +| 5 | 0.795736 | `marketplace_product_list` | ❌ | --- ## Test 32 -**Expected Tool:** `foundry_threads_get-messages` -**Prompt:** Show me the messages in the Microsoft Foundry thread with id +**Expected Tool:** `search_service_list` +**Prompt:** Show me my Cognitive Search services ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.669937 | `foundry_threads_get-messages` | ✅ **EXPECTED** | -| 2 | 0.584421 | `foundry_threads_create` | ❌ | -| 3 | 0.529381 | `foundry_threads_list` | ❌ | -| 4 | 0.437480 | `foundry_agents_get-sdk-sample` | ❌ | -| 5 | 0.427894 | `foundry_agents_create` | ❌ | +| 1 | 0.813442 | `search_service_list` | ✅ **EXPECTED** | +| 2 | 0.795308 | `foundry_resource_get` | ❌ | +| 3 | 0.789965 | `search_index_query` | ❌ | +| 4 | 0.786346 | `deploy_app_logs_get` | ❌ | +| 5 | 0.780307 | `foundry_agents_get-sdk-sample` | ❌ | --- ## Test 33 -**Expected Tool:** `foundry_threads_list` -**Prompt:** List my Microsoft Foundry threads +**Expected Tool:** `speech_stt_recognize` +**Prompt:** Convert this audio file to text using Azure Speech Services ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.677249 | `foundry_threads_list` | ✅ **EXPECTED** | -| 2 | 0.574068 | `foundry_threads_get-messages` | ❌ | -| 3 | 0.566981 | `foundry_threads_create` | ❌ | -| 4 | 0.471544 | `foundry_agents_get-sdk-sample` | ❌ | -| 5 | 0.448963 | `foundry_agents_list` | ❌ | +| 1 | 0.912779 | `speech_tts_synthesize` | ❌ | +| 2 | 0.907718 | `speech_stt_recognize` | ✅ **EXPECTED** | +| 3 | 0.791230 | `foundry_openai_create-completion` | ❌ | +| 4 | 0.787746 | `foundry_openai_embeddings-create` | ❌ | +| 5 | 0.785739 | `foundry_openai_chat-completions-create` | ❌ | --- ## Test 34 -**Expected Tool:** `search_knowledge_base_get` -**Prompt:** List all knowledge bases in the Azure AI Search service +**Expected Tool:** `speech_stt_recognize` +**Prompt:** Recognize speech from my audio file with language detection ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.785967 | `search_knowledge_base_get` | ✅ **EXPECTED** | -| 2 | 0.700824 | `search_knowledge_source_get` | ❌ | -| 3 | 0.693471 | `search_service_list` | ❌ | -| 4 | 0.635863 | `search_knowledge_base_retrieve` | ❌ | -| 5 | 0.586575 | `search_index_get` | ❌ | +| 1 | 0.871430 | `speech_stt_recognize` | ✅ **EXPECTED** | +| 2 | 0.810741 | `speech_tts_synthesize` | ❌ | +| 3 | 0.755382 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.733954 | `azureaibestpractices_get` | ❌ | +| 5 | 0.733863 | `foundry_openai_create-completion` | ❌ | --- ## Test 35 -**Expected Tool:** `search_knowledge_base_get` -**Prompt:** Show me the knowledge bases in the Azure AI Search service +**Expected Tool:** `speech_stt_recognize` +**Prompt:** Transcribe speech from audio file with profanity filtering ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.748213 | `search_knowledge_base_get` | ✅ **EXPECTED** | -| 2 | 0.668487 | `search_knowledge_source_get` | ❌ | -| 3 | 0.628582 | `search_knowledge_base_retrieve` | ❌ | -| 4 | 0.624480 | `search_service_list` | ❌ | -| 5 | 0.566618 | `search_index_get` | ❌ | +| 1 | 0.800249 | `speech_stt_recognize` | ✅ **EXPECTED** | +| 2 | 0.762297 | `speech_tts_synthesize` | ❌ | +| 3 | 0.679904 | `azureaibestpractices_get` | ❌ | +| 4 | 0.674115 | `foundry_agents_get-sdk-sample` | ❌ | +| 5 | 0.672941 | `mysql_database_query` | ❌ | --- ## Test 36 -**Expected Tool:** `search_knowledge_base_get` -**Prompt:** List all knowledge bases in the search service +**Expected Tool:** `speech_stt_recognize` +**Prompt:** Convert speech to text from audio file using endpoint ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.702942 | `search_knowledge_base_get` | ✅ **EXPECTED** | -| 2 | 0.605964 | `search_knowledge_source_get` | ❌ | -| 3 | 0.583234 | `search_knowledge_base_retrieve` | ❌ | -| 4 | 0.513638 | `search_service_list` | ❌ | -| 5 | 0.476815 | `foundry_knowledge_index_list` | ❌ | +| 1 | 0.845048 | `speech_tts_synthesize` | ❌ | +| 2 | 0.833839 | `speech_stt_recognize` | ✅ **EXPECTED** | +| 3 | 0.712860 | `foundry_openai_create-completion` | ❌ | +| 4 | 0.706974 | `foundry_openai_chat-completions-create` | ❌ | +| 5 | 0.704105 | `foundry_openai_embeddings-create` | ❌ | --- ## Test 37 -**Expected Tool:** `search_knowledge_base_get` -**Prompt:** Show me the knowledge bases in the search service +**Expected Tool:** `speech_stt_recognize` +**Prompt:** Transcribe the audio file in Spanish language ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.688051 | `search_knowledge_base_get` | ✅ **EXPECTED** | -| 2 | 0.599247 | `search_knowledge_source_get` | ❌ | -| 3 | 0.578499 | `search_knowledge_base_retrieve` | ❌ | -| 4 | 0.457619 | `search_service_list` | ❌ | -| 5 | 0.439528 | `foundry_knowledge_index_list` | ❌ | +| 1 | 0.801396 | `speech_stt_recognize` | ✅ **EXPECTED** | +| 2 | 0.781128 | `speech_tts_synthesize` | ❌ | +| 3 | 0.709994 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.704874 | `deploy_pipeline_guidance_get` | ❌ | +| 5 | 0.700876 | `deploy_iac_rules_get` | ❌ | --- ## Test 38 -**Expected Tool:** `search_knowledge_base_get` -**Prompt:** Get the details of knowledge base in the Azure AI Search service +**Expected Tool:** `speech_stt_recognize` +**Prompt:** Convert speech to text with detailed output format from audio file ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.769383 | `search_knowledge_base_get` | ✅ **EXPECTED** | -| 2 | 0.685640 | `search_knowledge_source_get` | ❌ | -| 3 | 0.636958 | `search_knowledge_base_retrieve` | ❌ | -| 4 | 0.585949 | `search_index_get` | ❌ | -| 5 | 0.533701 | `search_service_list` | ❌ | +| 1 | 0.820196 | `speech_tts_synthesize` | ❌ | +| 2 | 0.812918 | `speech_stt_recognize` | ✅ **EXPECTED** | +| 3 | 0.691471 | `foundry_openai_create-completion` | ❌ | +| 4 | 0.686186 | `azureaibestpractices_get` | ❌ | +| 5 | 0.681825 | `extension_cli_generate` | ❌ | --- ## Test 39 -**Expected Tool:** `search_knowledge_base_get` -**Prompt:** Show me the knowledge base in search service +**Expected Tool:** `speech_stt_recognize` +**Prompt:** Recognize speech from with phrase hints for better accuracy ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.595585 | `search_knowledge_base_get` | ✅ **EXPECTED** | -| 2 | 0.551922 | `search_knowledge_base_retrieve` | ❌ | -| 3 | 0.515480 | `search_knowledge_source_get` | ❌ | -| 4 | 0.366893 | `search_service_list` | ❌ | -| 5 | 0.365633 | `search_index_get` | ❌ | +| 1 | 0.820376 | `speech_stt_recognize` | ✅ **EXPECTED** | +| 2 | 0.762572 | `speech_tts_synthesize` | ❌ | +| 3 | 0.720218 | `foundry_openai_create-completion` | ❌ | +| 4 | 0.719858 | `azureaibestpractices_get` | ❌ | +| 5 | 0.713171 | `extension_cli_generate` | ❌ | --- ## Test 40 -**Expected Tool:** `search_knowledge_base_retrieve` -**Prompt:** Run a retrieval with knowledge base in Azure AI Search service for the query +**Expected Tool:** `speech_stt_recognize` +**Prompt:** Transcribe audio using multiple phrase hints: "Azure", "cognitive services", "machine learning" ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.724846 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | -| 2 | 0.650590 | `search_knowledge_base_get` | ❌ | -| 3 | 0.575306 | `search_index_query` | ❌ | -| 4 | 0.567361 | `search_knowledge_source_get` | ❌ | -| 5 | 0.520360 | `foundry_agents_connect` | ❌ | +| 1 | 0.851789 | `speech_stt_recognize` | ✅ **EXPECTED** | +| 2 | 0.836435 | `speech_tts_synthesize` | ❌ | +| 3 | 0.795157 | `cloudarchitect_design` | ❌ | +| 4 | 0.794858 | `azureaibestpractices_get` | ❌ | +| 5 | 0.793442 | `foundry_openai_create-completion` | ❌ | --- ## Test 41 -**Expected Tool:** `search_knowledge_base_retrieve` -**Prompt:** Ask knowledge base in search service to retrieve information about +**Expected Tool:** `speech_stt_recognize` +**Prompt:** Convert speech to text with comma-separated phrase hints: "Azure, cognitive services, API" ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.633766 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | -| 2 | 0.589869 | `search_knowledge_base_get` | ❌ | -| 3 | 0.502085 | `search_knowledge_source_get` | ❌ | -| 4 | 0.422732 | `foundry_agents_query-and-evaluate` | ❌ | -| 5 | 0.399595 | `search_index_query` | ❌ | +| 1 | 0.855506 | `speech_tts_synthesize` | ❌ | +| 2 | 0.849995 | `speech_stt_recognize` | ✅ **EXPECTED** | +| 3 | 0.811326 | `foundry_openai_create-completion` | ❌ | +| 4 | 0.801933 | `foundry_openai_chat-completions-create` | ❌ | +| 5 | 0.796301 | `azureaibestpractices_get` | ❌ | --- ## Test 42 -**Expected Tool:** `search_knowledge_base_retrieve` -**Prompt:** Run a retrieval with knowledge base in search service for the query +**Expected Tool:** `speech_stt_recognize` +**Prompt:** Transcribe audio with raw profanity output from file ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.657916 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | -| 2 | 0.557230 | `search_knowledge_base_get` | ❌ | -| 3 | 0.463587 | `search_knowledge_source_get` | ❌ | -| 4 | 0.436919 | `foundry_agents_query-and-evaluate` | ❌ | -| 5 | 0.422317 | `foundry_agents_connect` | ❌ | +| 1 | 0.782061 | `speech_stt_recognize` | ✅ **EXPECTED** | +| 2 | 0.756244 | `speech_tts_synthesize` | ❌ | +| 3 | 0.675683 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.673177 | `foundry_agents_connect` | ❌ | +| 5 | 0.670173 | `extension_cli_generate` | ❌ | --- ## Test 43 -**Expected Tool:** `search_knowledge_base_retrieve` -**Prompt:** Ask knowledge base in search service to retrieve information about +**Expected Tool:** `speech_tts_synthesize` +**Prompt:** Convert text to speech and save to output.wav ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.633757 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | -| 2 | 0.589846 | `search_knowledge_base_get` | ❌ | -| 3 | 0.502057 | `search_knowledge_source_get` | ❌ | -| 4 | 0.422702 | `foundry_agents_query-and-evaluate` | ❌ | -| 5 | 0.399618 | `search_index_query` | ❌ | +| 1 | 0.852732 | `speech_tts_synthesize` | ✅ **EXPECTED** | +| 2 | 0.829449 | `speech_stt_recognize` | ❌ | +| 3 | 0.737252 | `foundry_openai_create-completion` | ❌ | +| 4 | 0.728115 | `foundry_agents_get-sdk-sample` | ❌ | +| 5 | 0.727432 | `foundry_openai_embeddings-create` | ❌ | --- ## Test 44 -**Expected Tool:** `search_knowledge_base_retrieve` -**Prompt:** Query knowledge base in search service about +**Expected Tool:** `speech_tts_synthesize` +**Prompt:** Synthesize speech from "Hello, welcome to Azure" and save to welcome.wav ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.598868 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | -| 2 | 0.547862 | `search_knowledge_base_get` | ❌ | -| 3 | 0.467982 | `foundry_agents_query-and-evaluate` | ❌ | -| 4 | 0.464904 | `search_knowledge_source_get` | ❌ | -| 5 | 0.412481 | `foundry_agents_connect` | ❌ | +| 1 | 0.847909 | `speech_stt_recognize` | ❌ | +| 2 | 0.832691 | `speech_tts_synthesize` | ✅ **EXPECTED** | +| 3 | 0.781259 | `foundry_openai_create-completion` | ❌ | +| 4 | 0.778955 | `foundry_openai_chat-completions-create` | ❌ | +| 5 | 0.760548 | `foundry_agents_get-sdk-sample` | ❌ | --- ## Test 45 -**Expected Tool:** `search_knowledge_base_retrieve` -**Prompt:** Search knowledge base in Azure AI Search service for +**Expected Tool:** `speech_tts_synthesize` +**Prompt:** Generate speech audio from text "Hello world" using Azure Speech Services ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.649767 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | -| 2 | 0.631435 | `search_knowledge_base_get` | ❌ | -| 3 | 0.581386 | `search_index_query` | ❌ | -| 4 | 0.571156 | `search_knowledge_source_get` | ❌ | -| 5 | 0.544501 | `search_service_list` | ❌ | +| 1 | 0.882853 | `speech_stt_recognize` | ❌ | +| 2 | 0.881965 | `speech_tts_synthesize` | ✅ **EXPECTED** | +| 3 | 0.798395 | `foundry_openai_create-completion` | ❌ | +| 4 | 0.792187 | `foundry_openai_chat-completions-create` | ❌ | +| 5 | 0.787775 | `foundry_openai_embeddings-create` | ❌ | --- ## Test 46 -**Expected Tool:** `search_knowledge_base_retrieve` -**Prompt:** What does knowledge base in search service know about +**Expected Tool:** `speech_tts_synthesize` +**Prompt:** Convert text to speech with Spanish language and save to spanish-audio.wav ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.579716 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | -| 2 | 0.560688 | `search_knowledge_base_get` | ❌ | -| 3 | 0.477941 | `search_knowledge_source_get` | ❌ | -| 4 | 0.402615 | `foundry_agents_query-and-evaluate` | ❌ | -| 5 | 0.361231 | `foundry_knowledge_index_list` | ❌ | +| 1 | 0.839992 | `speech_tts_synthesize` | ✅ **EXPECTED** | +| 2 | 0.825812 | `speech_stt_recognize` | ❌ | +| 3 | 0.741351 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.728882 | `foundry_openai_create-completion` | ❌ | +| 5 | 0.724734 | `foundry_openai_chat-completions-create` | ❌ | --- ## Test 47 -**Expected Tool:** `search_knowledge_base_retrieve` -**Prompt:** Find information about using knowledge base in search service +**Expected Tool:** `speech_tts_synthesize` +**Prompt:** Synthesize speech with voice en-US-JennyNeural from text "Azure AI Services" ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.582662 | `search_knowledge_base_retrieve` | ✅ **EXPECTED** | -| 2 | 0.528610 | `search_knowledge_base_get` | ❌ | -| 3 | 0.449336 | `search_knowledge_source_get` | ❌ | -| 4 | 0.447832 | `foundry_agents_query-and-evaluate` | ❌ | -| 5 | 0.397187 | `foundry_agents_connect` | ❌ | +| 1 | 0.884680 | `speech_tts_synthesize` | ✅ **EXPECTED** | +| 2 | 0.867037 | `speech_stt_recognize` | ❌ | +| 3 | 0.819734 | `foundry_openai_chat-completions-create` | ❌ | +| 4 | 0.818966 | `foundry_openai_create-completion` | ❌ | +| 5 | 0.813921 | `foundry_openai_embeddings-create` | ❌ | --- ## Test 48 -**Expected Tool:** `search_knowledge_source_get` -**Prompt:** List all knowledge sources in the Azure AI Search service +**Expected Tool:** `speech_tts_synthesize` +**Prompt:** Create MP3 audio file from text "Welcome to Azure" with high quality format ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.760422 | `search_knowledge_source_get` | ✅ **EXPECTED** | -| 2 | 0.691960 | `search_service_list` | ❌ | -| 3 | 0.665897 | `search_knowledge_base_get` | ❌ | -| 4 | 0.573031 | `search_index_get` | ❌ | -| 5 | 0.560755 | `search_knowledge_base_retrieve` | ❌ | +| 1 | 0.846572 | `speech_stt_recognize` | ❌ | +| 2 | 0.834861 | `speech_tts_synthesize` | ✅ **EXPECTED** | +| 3 | 0.765231 | `foundry_openai_create-completion` | ❌ | +| 4 | 0.758524 | `azureaibestpractices_get` | ❌ | +| 5 | 0.754444 | `deploy_iac_rules_get` | ❌ | --- ## Test 49 -**Expected Tool:** `search_knowledge_source_get` -**Prompt:** Show me the knowledge sources in the Azure AI Search service +**Expected Tool:** `speech_tts_synthesize` +**Prompt:** Generate speech with custom voice model using endpoint ID ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.737860 | `search_knowledge_source_get` | ✅ **EXPECTED** | -| 2 | 0.660169 | `search_service_list` | ❌ | -| 3 | 0.652969 | `search_knowledge_base_get` | ❌ | -| 4 | 0.578836 | `search_index_get` | ❌ | -| 5 | 0.560564 | `search_index_query` | ❌ | +| 1 | 0.829757 | `speech_tts_synthesize` | ✅ **EXPECTED** | +| 2 | 0.808370 | `speech_stt_recognize` | ❌ | +| 3 | 0.756112 | `foundry_resource_get` | ❌ | +| 4 | 0.754856 | `foundry_openai_create-completion` | ❌ | +| 5 | 0.746172 | `foundry_openai_chat-completions-create` | ❌ | --- ## Test 50 -**Expected Tool:** `search_knowledge_source_get` -**Prompt:** List all knowledge sources in the search service +**Expected Tool:** `speech_tts_synthesize` +**Prompt:** Convert text to OGG/Opus format audio file ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.657936 | `search_knowledge_source_get` | ✅ **EXPECTED** | -| 2 | 0.558516 | `search_knowledge_base_get` | ❌ | -| 3 | 0.511469 | `search_service_list` | ❌ | -| 4 | 0.470560 | `search_knowledge_base_retrieve` | ❌ | -| 5 | 0.433657 | `foundry_knowledge_index_list` | ❌ | +| 1 | 0.796950 | `speech_stt_recognize` | ❌ | +| 2 | 0.784945 | `speech_tts_synthesize` | ✅ **EXPECTED** | +| 3 | 0.712920 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.711991 | `communication_email_send` | ❌ | +| 5 | 0.705540 | `extension_cli_generate` | ❌ | --- ## Test 51 -**Expected Tool:** `search_knowledge_source_get` -**Prompt:** Show me the knowledge sources in the search service +**Expected Tool:** `speech_tts_synthesize` +**Prompt:** Synthesize long text content to audio file with streaming ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.652929 | `search_knowledge_source_get` | ✅ **EXPECTED** | -| 2 | 0.563274 | `search_knowledge_base_get` | ❌ | -| 3 | 0.487037 | `search_service_list` | ❌ | -| 4 | 0.477605 | `search_knowledge_base_retrieve` | ❌ | -| 5 | 0.430499 | `search_index_get` | ❌ | +| 1 | 0.786826 | `speech_tts_synthesize` | ✅ **EXPECTED** | +| 2 | 0.781695 | `speech_stt_recognize` | ❌ | +| 3 | 0.725620 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.722756 | `foundry_openai_create-completion` | ❌ | +| 5 | 0.721370 | `communication_email_send` | ❌ | --- ## Test 52 -**Expected Tool:** `search_knowledge_source_get` -**Prompt:** Get the details of knowledge source in the Azure AI Search service +**Expected Tool:** `speech_tts_synthesize` +**Prompt:** Create audio file from text in French language with appropriate voice ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.825604 | `search_knowledge_source_get` | ✅ **EXPECTED** | -| 2 | 0.693438 | `search_knowledge_base_get` | ❌ | -| 3 | 0.595643 | `search_index_get` | ❌ | -| 4 | 0.540550 | `search_knowledge_base_retrieve` | ❌ | -| 5 | 0.531247 | `search_service_list` | ❌ | +| 1 | 0.818795 | `speech_stt_recognize` | ❌ | +| 2 | 0.816038 | `speech_tts_synthesize` | ✅ **EXPECTED** | +| 3 | 0.739962 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.734955 | `foundry_openai_create-completion` | ❌ | +| 5 | 0.732513 | `communication_email_send` | ❌ | --- ## Test 53 -**Expected Tool:** `search_knowledge_source_get` -**Prompt:** Show me the knowledge source in search service +**Expected Tool:** `appconfig_account_list` +**Prompt:** List all App Configuration stores in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.630840 | `search_knowledge_source_get` | ✅ **EXPECTED** | -| 2 | 0.523643 | `search_knowledge_base_get` | ❌ | -| 3 | 0.459923 | `search_knowledge_base_retrieve` | ❌ | -| 4 | 0.371465 | `search_index_get` | ❌ | -| 5 | 0.370838 | `search_service_list` | ❌ | +| 1 | 0.921068 | `appconfig_account_list` | ✅ **EXPECTED** | +| 2 | 0.811400 | `appconfig_kv_get` | ❌ | +| 3 | 0.809799 | `kusto_cluster_list` | ❌ | +| 4 | 0.804719 | `redis_list` | ❌ | +| 5 | 0.803314 | `subscription_list` | ❌ | --- ## Test 54 -**Expected Tool:** `search_index_get` -**Prompt:** Show me the details of the index in Cognitive Search service +**Expected Tool:** `appconfig_account_list` +**Prompt:** Show me the App Configuration stores in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.681052 | `search_index_get` | ✅ **EXPECTED** | -| 2 | 0.544557 | `foundry_knowledge_index_schema` | ❌ | -| 3 | 0.528153 | `search_knowledge_base_get` | ❌ | -| 4 | 0.521765 | `search_knowledge_source_get` | ❌ | -| 5 | 0.490625 | `search_service_list` | ❌ | +| 1 | 0.881722 | `appconfig_account_list` | ✅ **EXPECTED** | +| 2 | 0.805855 | `redis_list` | ❌ | +| 3 | 0.804700 | `eventgrid_subscription_list` | ❌ | +| 4 | 0.803939 | `appconfig_kv_get` | ❌ | +| 5 | 0.799546 | `eventgrid_topic_list` | ❌ | --- ## Test 55 -**Expected Tool:** `search_index_get` -**Prompt:** List all indexes in the Cognitive Search service +**Expected Tool:** `appconfig_account_list` +**Prompt:** Show me my App Configuration stores ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.640256 | `search_index_get` | ✅ **EXPECTED** | -| 2 | 0.620140 | `search_service_list` | ❌ | -| 3 | 0.538885 | `foundry_knowledge_index_list` | ❌ | -| 4 | 0.511485 | `search_knowledge_base_get` | ❌ | -| 5 | 0.496094 | `search_knowledge_source_get` | ❌ | +| 1 | 0.838378 | `appconfig_account_list` | ✅ **EXPECTED** | +| 2 | 0.806416 | `appconfig_kv_get` | ❌ | +| 3 | 0.771104 | `deploy_app_logs_get` | ❌ | +| 4 | 0.762362 | `appconfig_kv_set` | ❌ | +| 5 | 0.754373 | `postgres_server_config_get` | ❌ | --- ## Test 56 -**Expected Tool:** `search_index_get` -**Prompt:** Show me the indexes in the Cognitive Search service +**Expected Tool:** `appconfig_kv_delete` +**Prompt:** Delete the key in App Configuration store ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.620759 | `search_index_get` | ✅ **EXPECTED** | -| 2 | 0.562775 | `search_service_list` | ❌ | -| 3 | 0.538471 | `foundry_knowledge_index_list` | ❌ | -| 4 | 0.500365 | `search_knowledge_base_get` | ❌ | -| 5 | 0.490025 | `search_knowledge_source_get` | ❌ | +| 1 | 0.828986 | `appconfig_kv_delete` | ✅ **EXPECTED** | +| 2 | 0.786676 | `appconfig_kv_set` | ❌ | +| 3 | 0.786255 | `appconfig_kv_lock_set` | ❌ | +| 4 | 0.782200 | `appconfig_kv_get` | ❌ | +| 5 | 0.738970 | `appconfig_account_list` | ❌ | --- ## Test 57 -**Expected Tool:** `search_index_query` -**Prompt:** Search for instances of in the index in Cognitive Search service +**Expected Tool:** `appconfig_kv_get` +**Prompt:** List all key-value settings in App Configuration store ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.522826 | `search_index_get` | ❌ | -| 2 | 0.515870 | `search_index_query` | ✅ **EXPECTED** | -| 3 | 0.497467 | `search_service_list` | ❌ | -| 4 | 0.447977 | `search_knowledge_base_retrieve` | ❌ | -| 5 | 0.437677 | `postgres_database_query` | ❌ | +| 1 | 0.871372 | `appconfig_kv_get` | ✅ **EXPECTED** | +| 2 | 0.838423 | `appconfig_account_list` | ❌ | +| 3 | 0.822388 | `appconfig_kv_set` | ❌ | +| 4 | 0.792702 | `appconfig_kv_lock_set` | ❌ | +| 5 | 0.788942 | `appconfig_kv_delete` | ❌ | --- ## Test 58 -**Expected Tool:** `search_service_list` -**Prompt:** List all Cognitive Search services in my subscription +**Expected Tool:** `appconfig_kv_get` +**Prompt:** Show me the key-value settings in App Configuration store ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.793651 | `search_service_list` | ✅ **EXPECTED** | -| 2 | 0.553012 | `kusto_cluster_list` | ❌ | -| 3 | 0.509567 | `subscription_list` | ❌ | -| 4 | 0.505971 | `search_index_get` | ❌ | -| 5 | 0.504693 | `marketplace_product_list` | ❌ | +| 1 | 0.860199 | `appconfig_kv_get` | ✅ **EXPECTED** | +| 2 | 0.829010 | `appconfig_kv_set` | ❌ | +| 3 | 0.811705 | `appconfig_account_list` | ❌ | +| 4 | 0.802906 | `appconfig_kv_lock_set` | ❌ | +| 5 | 0.795144 | `appconfig_kv_delete` | ❌ | --- ## Test 59 -**Expected Tool:** `search_service_list` -**Prompt:** Show me the Cognitive Search services in my subscription +**Expected Tool:** `appconfig_kv_get` +**Prompt:** List all key-value settings with key name starting with 'prod-' in App Configuration store ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.686140 | `search_service_list` | ✅ **EXPECTED** | -| 2 | 0.484092 | `marketplace_product_list` | ❌ | -| 3 | 0.479898 | `search_index_get` | ❌ | -| 4 | 0.462337 | `search_knowledge_base_get` | ❌ | -| 5 | 0.461786 | `kusto_cluster_list` | ❌ | +| 1 | 0.822577 | `appconfig_kv_get` | ✅ **EXPECTED** | +| 2 | 0.798993 | `appconfig_account_list` | ❌ | +| 3 | 0.773761 | `appconfig_kv_set` | ❌ | +| 4 | 0.751419 | `appconfig_kv_delete` | ❌ | +| 5 | 0.747206 | `appconfig_kv_lock_set` | ❌ | --- ## Test 60 -**Expected Tool:** `search_service_list` -**Prompt:** Show me my Cognitive Search services +**Expected Tool:** `appconfig_kv_get` +**Prompt:** Show the content for the key in App Configuration store ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.553025 | `search_service_list` | ✅ **EXPECTED** | -| 2 | 0.436230 | `search_index_get` | ❌ | -| 3 | 0.415277 | `search_knowledge_base_get` | ❌ | -| 4 | 0.410461 | `search_knowledge_source_get` | ❌ | -| 5 | 0.404758 | `search_index_query` | ❌ | +| 1 | 0.821011 | `appconfig_kv_get` | ✅ **EXPECTED** | +| 2 | 0.815211 | `appconfig_kv_set` | ❌ | +| 3 | 0.776588 | `appconfig_kv_lock_set` | ❌ | +| 4 | 0.776106 | `appconfig_account_list` | ❌ | +| 5 | 0.762513 | `appconfig_kv_delete` | ❌ | --- ## Test 61 -**Expected Tool:** `speech_stt_recognize` -**Prompt:** Convert this audio file to text using Azure Speech Services +**Expected Tool:** `appconfig_kv_lock_set` +**Prompt:** Lock the key in App Configuration store ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.682065 | `speech_tts_synthesize` | ❌ | -| 2 | 0.666038 | `speech_stt_recognize` | ✅ **EXPECTED** | -| 3 | 0.377022 | `foundry_openai_embeddings-create` | ❌ | -| 4 | 0.351127 | `deploy_plan_get` | ❌ | -| 5 | 0.339269 | `azureaibestpractices_get` | ❌ | +| 1 | 0.838018 | `appconfig_kv_lock_set` | ✅ **EXPECTED** | +| 2 | 0.798669 | `appconfig_kv_set` | ❌ | +| 3 | 0.784101 | `appconfig_kv_get` | ❌ | +| 4 | 0.782583 | `appconfig_kv_delete` | ❌ | +| 5 | 0.738791 | `appconfig_account_list` | ❌ | --- ## Test 62 -**Expected Tool:** `speech_stt_recognize` -**Prompt:** Recognize speech from my audio file with language detection +**Expected Tool:** `appconfig_kv_lock_set` +**Prompt:** Unlock the key in App Configuration store ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.511324 | `speech_stt_recognize` | ✅ **EXPECTED** | -| 2 | 0.344404 | `speech_tts_synthesize` | ❌ | -| 3 | 0.197854 | `foundry_agents_get-sdk-sample` | ❌ | -| 4 | 0.192450 | `foundry_openai_embeddings-create` | ❌ | -| 5 | 0.170157 | `foundry_openai_create-completion` | ❌ | +| 1 | 0.834195 | `appconfig_kv_lock_set` | ✅ **EXPECTED** | +| 2 | 0.801717 | `appconfig_kv_set` | ❌ | +| 3 | 0.798289 | `appconfig_kv_get` | ❌ | +| 4 | 0.783738 | `appconfig_kv_delete` | ❌ | +| 5 | 0.751469 | `appconfig_account_list` | ❌ | --- ## Test 63 -**Expected Tool:** `speech_stt_recognize` -**Prompt:** Transcribe speech from audio file with profanity filtering +**Expected Tool:** `appconfig_kv_set` +**Prompt:** Set the key in App Configuration store to ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.486489 | `speech_stt_recognize` | ✅ **EXPECTED** | -| 2 | 0.335116 | `speech_tts_synthesize` | ❌ | -| 3 | 0.162869 | `foundry_threads_create` | ❌ | -| 4 | 0.160209 | `foundry_agents_connect` | ❌ | -| 5 | 0.156850 | `deploy_pipeline_guidance_get` | ❌ | +| 1 | 0.843448 | `appconfig_kv_set` | ✅ **EXPECTED** | +| 2 | 0.814112 | `appconfig_kv_lock_set` | ❌ | +| 3 | 0.790744 | `appconfig_kv_get` | ❌ | +| 4 | 0.788584 | `appconfig_kv_delete` | ❌ | +| 5 | 0.751340 | `mysql_server_param_set` | ❌ | --- ## Test 64 -**Expected Tool:** `speech_stt_recognize` -**Prompt:** Convert speech to text from audio file using endpoint +**Expected Tool:** `applens_resource_diagnose` +**Prompt:** Please help me diagnose issues with my app using app lens ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.611992 | `speech_stt_recognize` | ✅ **EXPECTED** | -| 2 | 0.573185 | `speech_tts_synthesize` | ❌ | -| 3 | 0.309895 | `foundry_openai_embeddings-create` | ❌ | -| 4 | 0.244218 | `foundry_resource_get` | ❌ | -| 5 | 0.243626 | `foundry_openai_create-completion` | ❌ | +| 1 | 0.860204 | `applens_resource_diagnose` | ✅ **EXPECTED** | +| 2 | 0.808555 | `deploy_app_logs_get` | ❌ | +| 3 | 0.763654 | `deploy_architecture_diagram_generate` | ❌ | +| 4 | 0.755670 | `cloudarchitect_design` | ❌ | +| 5 | 0.743357 | `foundry_agents_get-sdk-sample` | ❌ | --- ## Test 65 -**Expected Tool:** `speech_stt_recognize` -**Prompt:** Transcribe the audio file in Spanish language +**Expected Tool:** `applens_resource_diagnose` +**Prompt:** Use app lens to check why my app is slow? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.410533 | `speech_stt_recognize` | ✅ **EXPECTED** | -| 2 | 0.353783 | `speech_tts_synthesize` | ❌ | -| 3 | 0.152391 | `foundry_openai_embeddings-create` | ❌ | -| 4 | 0.152137 | `foundry_models_deploy` | ❌ | -| 5 | 0.151632 | `deploy_pipeline_guidance_get` | ❌ | +| 1 | 0.834916 | `applens_resource_diagnose` | ✅ **EXPECTED** | +| 2 | 0.794117 | `deploy_app_logs_get` | ❌ | +| 3 | 0.743505 | `cloudarchitect_design` | ❌ | +| 4 | 0.737515 | `deploy_architecture_diagram_generate` | ❌ | +| 5 | 0.723613 | `quota_usage_check` | ❌ | --- ## Test 66 -**Expected Tool:** `speech_stt_recognize` -**Prompt:** Convert speech to text with detailed output format from audio file +**Expected Tool:** `applens_resource_diagnose` +**Prompt:** What does app lens say is wrong with my service? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.546174 | `speech_stt_recognize` | ✅ **EXPECTED** | -| 2 | 0.480140 | `speech_tts_synthesize` | ❌ | -| 3 | 0.218078 | `foundry_resource_get` | ❌ | -| 4 | 0.202978 | `foundry_openai_embeddings-create` | ❌ | -| 5 | 0.183449 | `extension_azqr` | ❌ | +| 1 | 0.827699 | `applens_resource_diagnose` | ✅ **EXPECTED** | +| 2 | 0.773674 | `deploy_app_logs_get` | ❌ | +| 3 | 0.728346 | `deploy_architecture_diagram_generate` | ❌ | +| 4 | 0.727222 | `cloudarchitect_design` | ❌ | +| 5 | 0.721066 | `extension_cli_install` | ❌ | --- ## Test 67 -**Expected Tool:** `speech_stt_recognize` -**Prompt:** Recognize speech from with phrase hints for better accuracy +**Expected Tool:** `appservice_database_add` +**Prompt:** Add database connection to my app service for database in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.539963 | `speech_stt_recognize` | ✅ **EXPECTED** | -| 2 | 0.367401 | `speech_tts_synthesize` | ❌ | -| 3 | 0.228587 | `foundry_openai_create-completion` | ❌ | -| 4 | 0.203413 | `foundry_agents_connect` | ❌ | -| 5 | 0.199585 | `foundry_openai_embeddings-create` | ❌ | +| 1 | 0.897919 | `appservice_database_add` | ✅ **EXPECTED** | +| 2 | 0.754033 | `mysql_server_list` | ❌ | +| 3 | 0.746018 | `kusto_table_list` | ❌ | +| 4 | 0.744550 | `sql_db_create` | ❌ | +| 5 | 0.743978 | `mysql_database_list` | ❌ | --- ## Test 68 -**Expected Tool:** `speech_stt_recognize` -**Prompt:** Transcribe audio using multiple phrase hints: "Azure", "cognitive services", "machine learning" +**Expected Tool:** `appservice_database_add` +**Prompt:** Configure SQL Server database for app service with connection string in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.549151 | `speech_stt_recognize` | ✅ **EXPECTED** | -| 2 | 0.468161 | `speech_tts_synthesize` | ❌ | -| 3 | 0.392802 | `azureaibestpractices_get` | ❌ | -| 4 | 0.342537 | `extension_cli_generate` | ❌ | -| 5 | 0.337387 | `cloudarchitect_design` | ❌ | +| 1 | 0.884527 | `appservice_database_add` | ✅ **EXPECTED** | +| 2 | 0.773900 | `sql_db_create` | ❌ | +| 3 | 0.772206 | `sql_server_delete` | ❌ | +| 4 | 0.769539 | `sql_db_update` | ❌ | +| 5 | 0.768535 | `sql_server_show` | ❌ | --- ## Test 69 -**Expected Tool:** `speech_stt_recognize` -**Prompt:** Convert speech to text with comma-separated phrase hints: "Azure, cognitive services, API" +**Expected Tool:** `appservice_database_add` +**Prompt:** Add MySQL database to app service using connection in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.532536 | `speech_stt_recognize` | ✅ **EXPECTED** | -| 2 | 0.515532 | `speech_tts_synthesize` | ❌ | -| 3 | 0.349892 | `foundry_openai_create-completion` | ❌ | -| 4 | 0.349810 | `azureaibestpractices_get` | ❌ | -| 5 | 0.340893 | `foundry_openai_chat-completions-create` | ❌ | +| 1 | 0.881476 | `appservice_database_add` | ✅ **EXPECTED** | +| 2 | 0.769664 | `mysql_server_list` | ❌ | +| 3 | 0.757291 | `mysql_database_list` | ❌ | +| 4 | 0.756193 | `mysql_table_list` | ❌ | +| 5 | 0.749496 | `kusto_table_list` | ❌ | --- ## Test 70 -**Expected Tool:** `speech_stt_recognize` -**Prompt:** Transcribe audio with raw profanity output from file +**Expected Tool:** `appservice_database_add` +**Prompt:** Add PostgreSQL database to app service using connection in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.453396 | `speech_stt_recognize` | ✅ **EXPECTED** | -| 2 | 0.322710 | `speech_tts_synthesize` | ❌ | -| 3 | 0.173205 | `deploy_pipeline_guidance_get` | ❌ | -| 4 | 0.164990 | `foundry_openai_embeddings-create` | ❌ | -| 5 | 0.160483 | `foundry_agents_connect` | ❌ | +| 1 | 0.872626 | `appservice_database_add` | ✅ **EXPECTED** | +| 2 | 0.766107 | `postgres_database_query` | ❌ | +| 3 | 0.765076 | `postgres_database_list` | ❌ | +| 4 | 0.753598 | `postgres_server_param_set` | ❌ | +| 5 | 0.751675 | `postgres_table_list` | ❌ | --- ## Test 71 -**Expected Tool:** `speech_tts_synthesize` -**Prompt:** Convert text to speech and save to output.wav +**Expected Tool:** `appservice_database_add` +**Prompt:** Connect CosmosDB database using connection string to app service in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.521797 | `speech_tts_synthesize` | ✅ **EXPECTED** | -| 2 | 0.422457 | `speech_stt_recognize` | ❌ | -| 3 | 0.196049 | `foundry_openai_embeddings-create` | ❌ | -| 4 | 0.189438 | `foundry_openai_create-completion` | ❌ | -| 5 | 0.174955 | `foundry_openai_chat-completions-create` | ❌ | +| 1 | 0.869715 | `appservice_database_add` | ✅ **EXPECTED** | +| 2 | 0.793050 | `cosmos_database_container_list` | ❌ | +| 3 | 0.792432 | `kusto_query` | ❌ | +| 4 | 0.791978 | `cosmos_database_list` | ❌ | +| 5 | 0.788846 | `cosmos_database_container_item_query` | ❌ | --- ## Test 72 -**Expected Tool:** `speech_tts_synthesize` -**Prompt:** Synthesize speech from "Hello, welcome to Azure" and save to welcome.wav +**Expected Tool:** `appservice_database_add` +**Prompt:** Add database connection for database on server to app service in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.516973 | `speech_tts_synthesize` | ✅ **EXPECTED** | -| 2 | 0.486019 | `speech_stt_recognize` | ❌ | -| 3 | 0.329765 | `deploy_pipeline_guidance_get` | ❌ | -| 4 | 0.323728 | `extension_cli_generate` | ❌ | -| 5 | 0.317525 | `azureterraformbestpractices_get` | ❌ | +| 1 | 0.904503 | `appservice_database_add` | ✅ **EXPECTED** | +| 2 | 0.762192 | `mysql_server_list` | ❌ | +| 3 | 0.752833 | `sql_server_delete` | ❌ | +| 4 | 0.751123 | `sql_db_create` | ❌ | +| 5 | 0.750808 | `mysql_database_list` | ❌ | --- ## Test 73 -**Expected Tool:** `speech_tts_synthesize` -**Prompt:** Generate speech audio from text "Hello world" using Azure Speech Services +**Expected Tool:** `appservice_database_add` +**Prompt:** Add database connection string for to app service using connection string in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.592140 | `speech_tts_synthesize` | ✅ **EXPECTED** | -| 2 | 0.533997 | `speech_stt_recognize` | ❌ | -| 3 | 0.339572 | `foundry_openai_embeddings-create` | ❌ | -| 4 | 0.336349 | `azureaibestpractices_get` | ❌ | -| 5 | 0.327405 | `foundry_openai_create-completion` | ❌ | +| 1 | 0.906557 | `appservice_database_add` | ✅ **EXPECTED** | +| 2 | 0.746562 | `sql_db_rename` | ❌ | +| 3 | 0.739074 | `kusto_table_list` | ❌ | +| 4 | 0.736701 | `sql_db_create` | ❌ | +| 5 | 0.736203 | `deploy_architecture_diagram_generate` | ❌ | --- ## Test 74 -**Expected Tool:** `speech_tts_synthesize` -**Prompt:** Convert text to speech with Spanish language and save to spanish-audio.wav +**Expected Tool:** `appservice_database_add` +**Prompt:** Connect database to my app service using connection string in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.501096 | `speech_tts_synthesize` | ✅ **EXPECTED** | -| 2 | 0.452648 | `speech_stt_recognize` | ❌ | -| 3 | 0.210841 | `foundry_openai_embeddings-create` | ❌ | -| 4 | 0.196766 | `foundry_models_deploy` | ❌ | -| 5 | 0.191812 | `foundry_openai_chat-completions-create` | ❌ | +| 1 | 0.887945 | `appservice_database_add` | ✅ **EXPECTED** | +| 2 | 0.765721 | `deploy_app_logs_get` | ❌ | +| 3 | 0.764405 | `mysql_server_list` | ❌ | +| 4 | 0.763303 | `kusto_query` | ❌ | +| 5 | 0.763053 | `kusto_table_list` | ❌ | --- ## Test 75 -**Expected Tool:** `speech_tts_synthesize` -**Prompt:** Synthesize speech with voice en-US-JennyNeural from text "Azure AI Services" +**Expected Tool:** `appservice_database_add` +**Prompt:** Set up database for app service with connection string under resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.604878 | `speech_tts_synthesize` | ✅ **EXPECTED** | -| 2 | 0.496715 | `speech_stt_recognize` | ❌ | -| 3 | 0.417045 | `foundry_openai_embeddings-create` | ❌ | -| 4 | 0.384311 | `azureaibestpractices_get` | ❌ | -| 5 | 0.379840 | `foundry_openai_create-completion` | ❌ | +| 1 | 0.889735 | `appservice_database_add` | ✅ **EXPECTED** | +| 2 | 0.778608 | `sql_db_create` | ❌ | +| 3 | 0.771983 | `deploy_architecture_diagram_generate` | ❌ | +| 4 | 0.771033 | `kusto_table_list` | ❌ | +| 5 | 0.767668 | `redis_create` | ❌ | --- ## Test 76 -**Expected Tool:** `speech_tts_synthesize` -**Prompt:** Create MP3 audio file from text "Welcome to Azure" with high quality format +**Expected Tool:** `appservice_database_add` +**Prompt:** Configure database for app service with the connection string in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.561284 | `speech_tts_synthesize` | ✅ **EXPECTED** | -| 2 | 0.510909 | `speech_stt_recognize` | ❌ | -| 3 | 0.350088 | `azureaibestpractices_get` | ❌ | -| 4 | 0.348757 | `foundry_openai_embeddings-create` | ❌ | -| 5 | 0.347597 | `deploy_pipeline_guidance_get` | ❌ | +| 1 | 0.893730 | `appservice_database_add` | ✅ **EXPECTED** | +| 2 | 0.770268 | `sql_db_create` | ❌ | +| 3 | 0.770237 | `deploy_architecture_diagram_generate` | ❌ | +| 4 | 0.770072 | `sql_db_update` | ❌ | +| 5 | 0.765463 | `postgres_server_param_set` | ❌ | --- ## Test 77 -**Expected Tool:** `speech_tts_synthesize` -**Prompt:** Generate speech with custom voice model using endpoint ID +**Expected Tool:** `applicationinsights_recommendation_list` +**Prompt:** List code optimization recommendations across my Application Insights components ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.527294 | `speech_tts_synthesize` | ✅ **EXPECTED** | -| 2 | 0.455734 | `speech_stt_recognize` | ❌ | -| 3 | 0.353108 | `foundry_resource_get` | ❌ | -| 4 | 0.343308 | `foundry_models_deploy` | ❌ | -| 5 | 0.337888 | `foundry_openai_embeddings-create` | ❌ | +| 1 | 0.841405 | `applicationinsights_recommendation_list` | ✅ **EXPECTED** | +| 2 | 0.795617 | `azureaibestpractices_get` | ❌ | +| 3 | 0.791605 | `applens_resource_diagnose` | ❌ | +| 4 | 0.782883 | `deploy_app_logs_get` | ❌ | +| 5 | 0.781903 | `get_bestpractices_get` | ❌ | --- ## Test 78 -**Expected Tool:** `speech_tts_synthesize` -**Prompt:** Convert text to OGG/Opus format audio file +**Expected Tool:** `applicationinsights_recommendation_list` +**Prompt:** Show me code optimization recommendations for all Application Insights resources in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.432836 | `speech_tts_synthesize` | ✅ **EXPECTED** | -| 2 | 0.410086 | `speech_stt_recognize` | ❌ | -| 3 | 0.234237 | `foundry_openai_embeddings-create` | ❌ | -| 4 | 0.196153 | `extension_cli_generate` | ❌ | -| 5 | 0.175963 | `foundry_openai_create-completion` | ❌ | +| 1 | 0.882969 | `applicationinsights_recommendation_list` | ✅ **EXPECTED** | +| 2 | 0.821414 | `applens_resource_diagnose` | ❌ | +| 3 | 0.807270 | `redis_list` | ❌ | +| 4 | 0.804153 | `search_service_list` | ❌ | +| 5 | 0.802166 | `deploy_app_logs_get` | ❌ | --- ## Test 79 -**Expected Tool:** `speech_tts_synthesize` -**Prompt:** Synthesize long text content to audio file with streaming +**Expected Tool:** `applicationinsights_recommendation_list` +**Prompt:** List profiler recommendations for Application Insights in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.428079 | `speech_tts_synthesize` | ✅ **EXPECTED** | -| 2 | 0.369045 | `speech_stt_recognize` | ❌ | -| 3 | 0.230725 | `foundry_openai_embeddings-create` | ❌ | -| 4 | 0.220793 | `foundry_openai_create-completion` | ❌ | -| 5 | 0.216476 | `foundry_openai_chat-completions-create` | ❌ | +| 1 | 0.878587 | `applicationinsights_recommendation_list` | ✅ **EXPECTED** | +| 2 | 0.808693 | `applens_resource_diagnose` | ❌ | +| 3 | 0.786629 | `deploy_app_logs_get` | ❌ | +| 4 | 0.782837 | `azureaibestpractices_get` | ❌ | +| 5 | 0.780562 | `get_bestpractices_get` | ❌ | --- ## Test 80 -**Expected Tool:** `speech_tts_synthesize` -**Prompt:** Create audio file from text in French language with appropriate voice +**Expected Tool:** `applicationinsights_recommendation_list` +**Prompt:** Show me performance improvement recommendations from Application Insights ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.444444 | `speech_tts_synthesize` | ✅ **EXPECTED** | -| 2 | 0.385267 | `speech_stt_recognize` | ❌ | -| 3 | 0.229890 | `foundry_openai_create-completion` | ❌ | -| 4 | 0.228317 | `foundry_openai_embeddings-create` | ❌ | -| 5 | 0.213222 | `foundry_openai_chat-completions-create` | ❌ | +| 1 | 0.815200 | `applens_resource_diagnose` | ❌ | +| 2 | 0.814016 | `applicationinsights_recommendation_list` | ✅ **EXPECTED** | +| 3 | 0.798283 | `deploy_app_logs_get` | ❌ | +| 4 | 0.793547 | `azureaibestpractices_get` | ❌ | +| 5 | 0.788448 | `cloudarchitect_design` | ❌ | --- ## Test 81 -**Expected Tool:** `appconfig_account_list` -**Prompt:** List all App Configuration stores in my subscription +**Expected Tool:** `extension_cli_generate` +**Prompt:** Create a Storage account with name using Azure CLI ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.786360 | `appconfig_account_list` | ✅ **EXPECTED** | -| 2 | 0.530613 | `appconfig_kv_get` | ❌ | -| 3 | 0.491380 | `postgres_server_list` | ❌ | -| 4 | 0.481223 | `kusto_cluster_list` | ❌ | -| 5 | 0.479966 | `subscription_list` | ❌ | +| 1 | 0.845006 | `storage_account_create` | ❌ | +| 2 | 0.840701 | `storage_blob_container_create` | ❌ | +| 3 | 0.832012 | `storage_account_get` | ❌ | +| 4 | 0.799995 | `redis_create` | ❌ | +| 5 | 0.796441 | `storage_blob_container_get` | ❌ | --- ## Test 82 -**Expected Tool:** `appconfig_account_list` -**Prompt:** Show me the App Configuration stores in my subscription +**Expected Tool:** `extension_cli_generate` +**Prompt:** List all virtual machines in my subscription using Azure CLI ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.634978 | `appconfig_account_list` | ✅ **EXPECTED** | -| 2 | 0.464865 | `appconfig_kv_get` | ❌ | -| 3 | 0.398476 | `subscription_list` | ❌ | -| 4 | 0.391291 | `redis_list` | ❌ | -| 5 | 0.372456 | `postgres_server_list` | ❌ | +| 1 | 0.853361 | `search_service_list` | ❌ | +| 2 | 0.842535 | `kusto_cluster_list` | ❌ | +| 3 | 0.829272 | `subscription_list` | ❌ | +| 4 | 0.824854 | `virtualdesktop_hostpool_list` | ❌ | +| 5 | 0.821018 | `redis_list` | ❌ | --- ## Test 83 -**Expected Tool:** `appconfig_account_list` -**Prompt:** Show me my App Configuration stores +**Expected Tool:** `extension_cli_generate` +**Prompt:** Show me the details of the storage account with Azure CLI commands ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.565435 | `appconfig_account_list` | ✅ **EXPECTED** | -| 2 | 0.465344 | `appconfig_kv_get` | ❌ | -| 3 | 0.355916 | `postgres_server_config_get` | ❌ | -| 4 | 0.348661 | `appconfig_kv_delete` | ❌ | -| 5 | 0.327234 | `appconfig_kv_set` | ❌ | +| 1 | 0.887952 | `storage_account_get` | ❌ | +| 2 | 0.855618 | `storage_blob_container_get` | ❌ | +| 3 | 0.838257 | `storage_blob_get` | ❌ | +| 4 | 0.823911 | `storage_account_create` | ❌ | +| 5 | 0.821905 | `storage_blob_container_create` | ❌ | --- ## Test 84 -**Expected Tool:** `appconfig_kv_delete` -**Prompt:** Delete the key in App Configuration store +**Expected Tool:** `extension_cli_install` +**Prompt:** ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.618276 | `appconfig_kv_delete` | ✅ **EXPECTED** | -| 2 | 0.464358 | `appconfig_kv_get` | ❌ | -| 3 | 0.424344 | `appconfig_kv_set` | ❌ | -| 4 | 0.422700 | `appconfig_kv_lock_set` | ❌ | -| 5 | 0.392016 | `appconfig_account_list` | ❌ | +| 1 | 0.783226 | `extension_cli_generate` | ❌ | +| 2 | 0.781459 | `extension_cli_install` | ✅ **EXPECTED** | +| 3 | 0.753667 | `sql_server_delete` | ❌ | +| 4 | 0.750484 | `deploy_iac_rules_get` | ❌ | +| 5 | 0.741619 | `deploy_plan_get` | ❌ | --- ## Test 85 -**Expected Tool:** `appconfig_kv_get` -**Prompt:** List all key-value settings in App Configuration store +**Expected Tool:** `extension_cli_install` +**Prompt:** How to install azd ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.632687 | `appconfig_kv_get` | ✅ **EXPECTED** | -| 2 | 0.557810 | `appconfig_account_list` | ❌ | -| 3 | 0.530884 | `appconfig_kv_set` | ❌ | -| 4 | 0.464635 | `appconfig_kv_delete` | ❌ | -| 5 | 0.439089 | `appconfig_kv_lock_set` | ❌ | +| 1 | 0.790601 | `extension_cli_install` | ✅ **EXPECTED** | +| 2 | 0.728892 | `deploy_iac_rules_get` | ❌ | +| 3 | 0.721054 | `deploy_pipeline_guidance_get` | ❌ | +| 4 | 0.717446 | `extension_cli_generate` | ❌ | +| 5 | 0.714625 | `extension_azqr` | ❌ | --- ## Test 86 -**Expected Tool:** `appconfig_kv_get` -**Prompt:** Show me the key-value settings in App Configuration store +**Expected Tool:** `extension_cli_install` +**Prompt:** What is Azure Functions Core tools and how to install it ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.612555 | `appconfig_kv_get` | ✅ **EXPECTED** | -| 2 | 0.522426 | `appconfig_account_list` | ❌ | -| 3 | 0.512945 | `appconfig_kv_set` | ❌ | -| 4 | 0.468503 | `appconfig_kv_delete` | ❌ | -| 5 | 0.457866 | `appconfig_kv_lock_set` | ❌ | +| 1 | 0.892433 | `extension_cli_install` | ✅ **EXPECTED** | +| 2 | 0.812380 | `deploy_iac_rules_get` | ❌ | +| 3 | 0.811207 | `extension_cli_generate` | ❌ | +| 4 | 0.803697 | `azureaibestpractices_get` | ❌ | +| 5 | 0.803642 | `deploy_pipeline_guidance_get` | ❌ | --- ## Test 87 -**Expected Tool:** `appconfig_kv_get` -**Prompt:** List all key-value settings with key name starting with 'prod-' in App Configuration store +**Expected Tool:** `acr_registry_list` +**Prompt:** List all Azure Container Registries in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.512903 | `appconfig_kv_get` | ✅ **EXPECTED** | -| 2 | 0.449885 | `appconfig_account_list` | ❌ | -| 3 | 0.398671 | `appconfig_kv_set` | ❌ | -| 4 | 0.380619 | `appconfig_kv_delete` | ❌ | -| 5 | 0.346120 | `appconfig_kv_lock_set` | ❌ | +| 1 | 0.893468 | `acr_registry_repository_list` | ❌ | +| 2 | 0.872986 | `acr_registry_list` | ✅ **EXPECTED** | +| 3 | 0.862416 | `kusto_cluster_list` | ❌ | +| 4 | 0.856360 | `search_service_list` | ❌ | +| 5 | 0.837896 | `redis_list` | ❌ | --- ## Test 88 -**Expected Tool:** `appconfig_kv_get` -**Prompt:** Show the content for the key in App Configuration store +**Expected Tool:** `acr_registry_list` +**Prompt:** Show me my Azure Container Registries ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.552300 | `appconfig_kv_get` | ✅ **EXPECTED** | -| 2 | 0.448912 | `appconfig_kv_set` | ❌ | -| 3 | 0.441713 | `appconfig_kv_delete` | ❌ | -| 4 | 0.437432 | `appconfig_account_list` | ❌ | -| 5 | 0.416264 | `appconfig_kv_lock_set` | ❌ | +| 1 | 0.863518 | `acr_registry_repository_list` | ❌ | +| 2 | 0.828738 | `acr_registry_list` | ✅ **EXPECTED** | +| 3 | 0.819051 | `storage_blob_container_get` | ❌ | +| 4 | 0.813630 | `quota_usage_check` | ❌ | +| 5 | 0.803314 | `redis_list` | ❌ | --- ## Test 89 -**Expected Tool:** `appconfig_kv_lock_set` -**Prompt:** Lock the key in App Configuration store +**Expected Tool:** `acr_registry_list` +**Prompt:** Show me the container registries in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.591237 | `appconfig_kv_lock_set` | ✅ **EXPECTED** | -| 2 | 0.487174 | `appconfig_kv_get` | ❌ | -| 3 | 0.445551 | `appconfig_kv_set` | ❌ | -| 4 | 0.431516 | `appconfig_kv_delete` | ❌ | -| 5 | 0.373656 | `appconfig_account_list` | ❌ | +| 1 | 0.845363 | `acr_registry_repository_list` | ❌ | +| 2 | 0.823566 | `acr_registry_list` | ✅ **EXPECTED** | +| 3 | 0.814685 | `redis_list` | ❌ | +| 4 | 0.805737 | `storage_blob_container_get` | ❌ | +| 5 | 0.804288 | `eventgrid_subscription_list` | ❌ | --- ## Test 90 -**Expected Tool:** `appconfig_kv_lock_set` -**Prompt:** Unlock the key in App Configuration store +**Expected Tool:** `acr_registry_list` +**Prompt:** List container registries in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.555699 | `appconfig_kv_lock_set` | ✅ **EXPECTED** | -| 2 | 0.505681 | `appconfig_kv_get` | ❌ | -| 3 | 0.476497 | `appconfig_kv_delete` | ❌ | -| 4 | 0.425488 | `appconfig_kv_set` | ❌ | -| 5 | 0.409406 | `appconfig_account_list` | ❌ | +| 1 | 0.865925 | `acr_registry_repository_list` | ❌ | +| 2 | 0.833004 | `acr_registry_list` | ✅ **EXPECTED** | +| 3 | 0.788833 | `kusto_cluster_list` | ❌ | +| 4 | 0.781741 | `group_list` | ❌ | +| 5 | 0.778240 | `storage_blob_container_get` | ❌ | --- ## Test 91 -**Expected Tool:** `appconfig_kv_set` -**Prompt:** Set the key in App Configuration store to +**Expected Tool:** `acr_registry_list` +**Prompt:** Show me the container registries in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.609543 | `appconfig_kv_set` | ✅ **EXPECTED** | -| 2 | 0.536461 | `appconfig_kv_lock_set` | ❌ | -| 3 | 0.512507 | `appconfig_kv_get` | ❌ | -| 4 | 0.505284 | `appconfig_kv_delete` | ❌ | -| 5 | 0.377794 | `appconfig_account_list` | ❌ | +| 1 | 0.867299 | `acr_registry_repository_list` | ❌ | +| 2 | 0.824300 | `acr_registry_list` | ✅ **EXPECTED** | +| 3 | 0.793830 | `storage_blob_container_get` | ❌ | +| 4 | 0.786488 | `redis_list` | ❌ | +| 5 | 0.776447 | `eventgrid_topic_list` | ❌ | --- ## Test 92 -**Expected Tool:** `applens_resource_diagnose` -**Prompt:** Please help me diagnose issues with my app using app lens +**Expected Tool:** `acr_registry_repository_list` +**Prompt:** List all container registry repositories in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.595632 | `applens_resource_diagnose` | ✅ **EXPECTED** | -| 2 | 0.336090 | `deploy_app_logs_get` | ❌ | -| 3 | 0.300786 | `deploy_architecture_diagram_generate` | ❌ | -| 4 | 0.273083 | `cloudarchitect_design` | ❌ | -| 5 | 0.254473 | `monitor_resource_log_query` | ❌ | +| 1 | 0.886397 | `acr_registry_repository_list` | ✅ **EXPECTED** | +| 2 | 0.837955 | `acr_registry_list` | ❌ | +| 3 | 0.826558 | `kusto_cluster_list` | ❌ | +| 4 | 0.813168 | `search_service_list` | ❌ | +| 5 | 0.808497 | `redis_list` | ❌ | --- ## Test 93 -**Expected Tool:** `applens_resource_diagnose` -**Prompt:** Use app lens to check why my app is slow? +**Expected Tool:** `acr_registry_repository_list` +**Prompt:** Show me my container registry repositories ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.502361 | `applens_resource_diagnose` | ✅ **EXPECTED** | -| 2 | 0.316297 | `deploy_app_logs_get` | ❌ | -| 3 | 0.255570 | `deploy_architecture_diagram_generate` | ❌ | -| 4 | 0.249583 | `monitor_resource_log_query` | ❌ | -| 5 | 0.225972 | `quota_usage_check` | ❌ | +| 1 | 0.846337 | `acr_registry_repository_list` | ✅ **EXPECTED** | +| 2 | 0.787159 | `storage_blob_container_get` | ❌ | +| 3 | 0.771812 | `acr_registry_list` | ❌ | +| 4 | 0.768761 | `deploy_app_logs_get` | ❌ | +| 5 | 0.762557 | `redis_list` | ❌ | --- ## Test 94 -**Expected Tool:** `applens_resource_diagnose` -**Prompt:** What does app lens say is wrong with my service? +**Expected Tool:** `acr_registry_repository_list` +**Prompt:** List repositories in the container registry ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.492820 | `applens_resource_diagnose` | ✅ **EXPECTED** | -| 2 | 0.256325 | `deploy_architecture_diagram_generate` | ❌ | -| 3 | 0.242574 | `cloudarchitect_design` | ❌ | -| 4 | 0.225608 | `resourcehealth_health-events_list` | ❌ | -| 5 | 0.211565 | `deploy_app_logs_get` | ❌ | +| 1 | 0.868138 | `acr_registry_repository_list` | ✅ **EXPECTED** | +| 2 | 0.793332 | `acr_registry_list` | ❌ | +| 3 | 0.773097 | `storage_blob_container_get` | ❌ | +| 4 | 0.759270 | `cosmos_database_container_list` | ❌ | +| 5 | 0.755409 | `kusto_database_list` | ❌ | --- ## Test 95 -**Expected Tool:** `appservice_database_add` -**Prompt:** Add database connection to my app service for database in resource group +**Expected Tool:** `acr_registry_repository_list` +**Prompt:** Show me the repositories in the container registry ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.717767 | `appservice_database_add` | ✅ **EXPECTED** | -| 2 | 0.401084 | `sql_db_rename` | ❌ | -| 3 | 0.399636 | `sql_db_create` | ❌ | -| 4 | 0.362627 | `sql_db_show` | ❌ | -| 5 | 0.357493 | `sql_db_list` | ❌ | +| 1 | 0.853934 | `acr_registry_repository_list` | ✅ **EXPECTED** | +| 2 | 0.783735 | `storage_blob_container_get` | ❌ | +| 3 | 0.779666 | `acr_registry_list` | ❌ | +| 4 | 0.753688 | `redis_list` | ❌ | +| 5 | 0.750924 | `storage_blob_get` | ❌ | --- ## Test 96 -**Expected Tool:** `appservice_database_add` -**Prompt:** Configure SQL Server database for app service with connection string in resource group +**Expected Tool:** `communication_email_send` +**Prompt:** Send an email to with subject ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.688354 | `appservice_database_add` | ✅ **EXPECTED** | -| 2 | 0.498219 | `sql_db_rename` | ❌ | -| 3 | 0.497527 | `sql_db_create` | ❌ | -| 4 | 0.469571 | `sql_db_show` | ❌ | -| 5 | 0.453075 | `sql_db_list` | ❌ | +| 1 | 0.868273 | `communication_email_send` | ✅ **EXPECTED** | +| 2 | 0.742303 | `communication_sms_send` | ❌ | +| 3 | 0.731491 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.718726 | `eventgrid_topic_list` | ❌ | +| 5 | 0.711620 | `kusto_query` | ❌ | --- ## Test 97 -**Expected Tool:** `appservice_database_add` -**Prompt:** Add MySQL database to app service using connection in resource group +**Expected Tool:** `communication_email_send` +**Prompt:** Send an email from my communication service to ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.675440 | `appservice_database_add` | ✅ **EXPECTED** | -| 2 | 0.465196 | `sql_db_create` | ❌ | -| 3 | 0.452770 | `sql_db_rename` | ❌ | -| 4 | 0.433166 | `mysql_server_list` | ❌ | -| 5 | 0.410333 | `sql_db_show` | ❌ | +| 1 | 0.852579 | `communication_email_send` | ✅ **EXPECTED** | +| 2 | 0.744727 | `communication_sms_send` | ❌ | +| 3 | 0.738864 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.723001 | `speech_stt_recognize` | ❌ | +| 5 | 0.721946 | `speech_tts_synthesize` | ❌ | --- ## Test 98 -**Expected Tool:** `appservice_database_add` -**Prompt:** Add PostgreSQL database to app service using connection in resource group +**Expected Tool:** `communication_email_send` +**Prompt:** Send HTML-formatted email to with subject ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.627508 | `appservice_database_add` | ✅ **EXPECTED** | -| 2 | 0.443688 | `sql_db_create` | ❌ | -| 3 | 0.404065 | `postgres_database_query` | ❌ | -| 4 | 0.400782 | `sql_db_rename` | ❌ | -| 5 | 0.400710 | `postgres_database_list` | ❌ | +| 1 | 0.872349 | `communication_email_send` | ✅ **EXPECTED** | +| 2 | 0.741165 | `communication_sms_send` | ❌ | +| 3 | 0.729801 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.709321 | `eventgrid_topic_list` | ❌ | +| 5 | 0.704821 | `extension_cli_generate` | ❌ | --- ## Test 99 -**Expected Tool:** `appservice_database_add` -**Prompt:** Connect CosmosDB database using connection string to app service in resource group +**Expected Tool:** `communication_email_send` +**Prompt:** Send email with CC to and ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.663053 | `appservice_database_add` | ✅ **EXPECTED** | -| 2 | 0.446227 | `cosmos_database_list` | ❌ | -| 3 | 0.441563 | `cosmos_database_container_item_query` | ❌ | -| 4 | 0.426937 | `cosmos_database_container_list` | ❌ | -| 5 | 0.420210 | `sql_db_rename` | ❌ | +| 1 | 0.880319 | `communication_email_send` | ✅ **EXPECTED** | +| 2 | 0.749198 | `communication_sms_send` | ❌ | +| 3 | 0.715848 | `deploy_pipeline_guidance_get` | ❌ | +| 4 | 0.713742 | `cloudarchitect_design` | ❌ | +| 5 | 0.705096 | `foundry_agents_get-sdk-sample` | ❌ | --- ## Test 100 -**Expected Tool:** `appservice_database_add` -**Prompt:** Add database connection for database on server to app service in resource group +**Expected Tool:** `communication_email_send` +**Prompt:** Send email to multiple recipients: , ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.733775 | `appservice_database_add` | ✅ **EXPECTED** | -| 2 | 0.454554 | `sql_db_create` | ❌ | -| 3 | 0.415274 | `sql_db_rename` | ❌ | -| 4 | 0.414045 | `sql_server_create` | ❌ | -| 5 | 0.410260 | `sql_db_list` | ❌ | +| 1 | 0.904716 | `communication_email_send` | ✅ **EXPECTED** | +| 2 | 0.782326 | `communication_sms_send` | ❌ | +| 3 | 0.710614 | `cloudarchitect_design` | ❌ | +| 4 | 0.709656 | `foundry_agents_get-sdk-sample` | ❌ | +| 5 | 0.705415 | `eventgrid_topic_list` | ❌ | --- ## Test 101 -**Expected Tool:** `appservice_database_add` -**Prompt:** Add database connection string for to app service using connection string in resource group +**Expected Tool:** `communication_email_send` +**Prompt:** Send email with reply-to address set to ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.746370 | `appservice_database_add` | ✅ **EXPECTED** | -| 2 | 0.441616 | `sql_db_rename` | ❌ | -| 3 | 0.433929 | `sql_db_create` | ❌ | -| 4 | 0.391303 | `sql_db_list` | ❌ | -| 5 | 0.390203 | `sql_db_show` | ❌ | +| 1 | 0.866825 | `communication_email_send` | ✅ **EXPECTED** | +| 2 | 0.736970 | `communication_sms_send` | ❌ | +| 3 | 0.708399 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.696529 | `foundry_threads_get-messages` | ❌ | +| 5 | 0.695710 | `postgres_server_param_set` | ❌ | --- ## Test 102 -**Expected Tool:** `appservice_database_add` -**Prompt:** Connect database to my app service using connection string in resource group +**Expected Tool:** `communication_email_send` +**Prompt:** Send email with custom sender name ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.680400 | `appservice_database_add` | ✅ **EXPECTED** | -| 2 | 0.429317 | `sql_db_rename` | ❌ | -| 3 | 0.406322 | `sql_db_create` | ❌ | -| 4 | 0.396522 | `sql_db_show` | ❌ | -| 5 | 0.391430 | `sql_db_list` | ❌ | +| 1 | 0.837896 | `communication_email_send` | ✅ **EXPECTED** | +| 2 | 0.721563 | `communication_sms_send` | ❌ | +| 3 | 0.717084 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.696428 | `foundry_openai_create-completion` | ❌ | +| 5 | 0.693472 | `foundry_threads_get-messages` | ❌ | --- ## Test 103 -**Expected Tool:** `appservice_database_add` -**Prompt:** Set up database for app service with connection string under resource group +**Expected Tool:** `communication_email_send` +**Prompt:** Send an email with BCC recipients ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.640547 | `appservice_database_add` | ✅ **EXPECTED** | -| 2 | 0.456884 | `sql_db_create` | ❌ | -| 3 | 0.402743 | `sql_db_rename` | ❌ | -| 4 | 0.402138 | `sql_db_show` | ❌ | -| 5 | 0.394211 | `sql_db_list` | ❌ | +| 1 | 0.879273 | `communication_email_send` | ✅ **EXPECTED** | +| 2 | 0.755289 | `communication_sms_send` | ❌ | +| 3 | 0.707649 | `foundry_threads_get-messages` | ❌ | +| 4 | 0.704448 | `foundry_agents_get-sdk-sample` | ❌ | +| 5 | 0.704119 | `extension_cli_generate` | ❌ | --- ## Test 104 -**Expected Tool:** `appservice_database_add` -**Prompt:** Configure database for app service with the connection string in resource group +**Expected Tool:** `communication_sms_send` +**Prompt:** Send an SMS message to saying "Hello" ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.688390 | `appservice_database_add` | ✅ **EXPECTED** | -| 2 | 0.448095 | `sql_db_rename` | ❌ | -| 3 | 0.447449 | `sql_db_create` | ❌ | -| 4 | 0.413626 | `sql_db_show` | ❌ | -| 5 | 0.411077 | `sql_db_list` | ❌ | +| 1 | 0.820770 | `communication_sms_send` | ✅ **EXPECTED** | +| 2 | 0.769285 | `communication_email_send` | ❌ | +| 3 | 0.725443 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.709364 | `speech_tts_synthesize` | ❌ | +| 5 | 0.708279 | `speech_stt_recognize` | ❌ | --- ## Test 105 -**Expected Tool:** `applicationinsights_recommendation_list` -**Prompt:** List code optimization recommendations across my Application Insights components +**Expected Tool:** `communication_sms_send` +**Prompt:** Send SMS to from with message "Test message" ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.572473 | `applicationinsights_recommendation_list` | ✅ **EXPECTED** | -| 2 | 0.450025 | `azureaibestpractices_get` | ❌ | -| 3 | 0.445157 | `get_bestpractices_get` | ❌ | -| 4 | 0.390478 | `azureterraformbestpractices_get` | ❌ | -| 5 | 0.383948 | `applens_resource_diagnose` | ❌ | +| 1 | 0.821947 | `communication_sms_send` | ✅ **EXPECTED** | +| 2 | 0.775770 | `communication_email_send` | ❌ | +| 3 | 0.712315 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.696385 | `foundry_threads_get-messages` | ❌ | +| 5 | 0.693367 | `foundry_openai_chat-completions-create` | ❌ | --- ## Test 106 -**Expected Tool:** `applicationinsights_recommendation_list` -**Prompt:** Show me code optimization recommendations for all Application Insights resources in my subscription +**Expected Tool:** `communication_sms_send` +**Prompt:** Send SMS to multiple recipients: , ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.696531 | `applicationinsights_recommendation_list` | ✅ **EXPECTED** | -| 2 | 0.505796 | `azureaibestpractices_get` | ❌ | -| 3 | 0.468384 | `get_bestpractices_get` | ❌ | -| 4 | 0.452231 | `applens_resource_diagnose` | ❌ | -| 5 | 0.435241 | `azureterraformbestpractices_get` | ❌ | +| 1 | 0.870041 | `communication_sms_send` | ✅ **EXPECTED** | +| 2 | 0.844795 | `communication_email_send` | ❌ | +| 3 | 0.714595 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.704269 | `eventgrid_topic_list` | ❌ | +| 5 | 0.701691 | `speech_tts_synthesize` | ❌ | --- ## Test 107 -**Expected Tool:** `applicationinsights_recommendation_list` -**Prompt:** List profiler recommendations for Application Insights in resource group +**Expected Tool:** `communication_sms_send` +**Prompt:** Send SMS with delivery reporting enabled ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.626722 | `applicationinsights_recommendation_list` | ✅ **EXPECTED** | -| 2 | 0.488002 | `loadtesting_testresource_list` | ❌ | -| 3 | 0.479392 | `mysql_server_list` | ❌ | -| 4 | 0.477396 | `applens_resource_diagnose` | ❌ | -| 5 | 0.468847 | `resourcehealth_availability-status_list` | ❌ | +| 1 | 0.851135 | `communication_sms_send` | ✅ **EXPECTED** | +| 2 | 0.775737 | `communication_email_send` | ❌ | +| 3 | 0.733053 | `deploy_app_logs_get` | ❌ | +| 4 | 0.726200 | `eventgrid_subscription_list` | ❌ | +| 5 | 0.719715 | `foundry_agents_get-sdk-sample` | ❌ | --- ## Test 108 -**Expected Tool:** `applicationinsights_recommendation_list` -**Prompt:** Show me performance improvement recommendations from Application Insights +**Expected Tool:** `communication_sms_send` +**Prompt:** Send SMS message with custom tracking tag "campaign1" ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.509502 | `applicationinsights_recommendation_list` | ✅ **EXPECTED** | -| 2 | 0.432594 | `azureaibestpractices_get` | ❌ | -| 3 | 0.419670 | `applens_resource_diagnose` | ❌ | -| 4 | 0.383767 | `get_bestpractices_get` | ❌ | -| 5 | 0.367278 | `deploy_architecture_diagram_generate` | ❌ | +| 1 | 0.797352 | `communication_sms_send` | ✅ **EXPECTED** | +| 2 | 0.766617 | `communication_email_send` | ❌ | +| 3 | 0.727646 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.711044 | `foundry_threads_get-messages` | ❌ | +| 5 | 0.704560 | `deploy_app_logs_get` | ❌ | --- ## Test 109 -**Expected Tool:** `extension_cli_generate` -**Prompt:** Create a Storage account with name using Azure CLI +**Expected Tool:** `communication_sms_send` +**Prompt:** Send broadcast SMS to and saying "Urgent notification" ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.593241 | `storage_account_create` | ❌ | -| 2 | 0.564940 | `storage_blob_container_create` | ❌ | -| 3 | 0.493684 | `storage_account_get` | ❌ | -| 4 | 0.474398 | `storage_blob_container_get` | ❌ | -| 5 | 0.454194 | `managedlustre_fs_create` | ❌ | +| 1 | 0.828175 | `communication_sms_send` | ✅ **EXPECTED** | +| 2 | 0.767957 | `communication_email_send` | ❌ | +| 3 | 0.701228 | `foundry_threads_get-messages` | ❌ | +| 4 | 0.697810 | `foundry_agents_get-sdk-sample` | ❌ | +| 5 | 0.696027 | `eventgrid_topic_list` | ❌ | --- ## Test 110 -**Expected Tool:** `extension_cli_generate` -**Prompt:** List all virtual machines in my subscription using Azure CLI +**Expected Tool:** `communication_sms_send` +**Prompt:** Send SMS from my communication service to ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.593467 | `search_service_list` | ❌ | -| 2 | 0.575274 | `kusto_cluster_list` | ❌ | -| 3 | 0.549965 | `virtualdesktop_hostpool_list` | ❌ | -| 4 | 0.544412 | `monitor_workspace_list` | ❌ | -| 5 | 0.536282 | `subscription_list` | ❌ | +| 1 | 0.824706 | `communication_sms_send` | ✅ **EXPECTED** | +| 2 | 0.788074 | `communication_email_send` | ❌ | +| 3 | 0.722517 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.713771 | `speech_tts_synthesize` | ❌ | +| 5 | 0.707065 | `speech_stt_recognize` | ❌ | --- ## Test 111 -**Expected Tool:** `extension_cli_generate` -**Prompt:** Show me the details of the storage account with Azure CLI commands +**Expected Tool:** `communication_sms_send` +**Prompt:** Send an SMS with delivery receipt tracking ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.710307 | `storage_account_get` | ❌ | -| 2 | 0.602173 | `storage_blob_container_get` | ❌ | -| 3 | 0.543268 | `storage_blob_get` | ❌ | -| 4 | 0.519788 | `storage_account_create` | ❌ | -| 5 | 0.493145 | `cosmos_account_list` | ❌ | +| 1 | 0.852271 | `communication_sms_send` | ✅ **EXPECTED** | +| 2 | 0.784496 | `communication_email_send` | ❌ | +| 3 | 0.736170 | `deploy_app_logs_get` | ❌ | +| 4 | 0.734240 | `foundry_agents_get-sdk-sample` | ❌ | +| 5 | 0.730021 | `foundry_threads_get-messages` | ❌ | --- ## Test 112 -**Expected Tool:** `extension_cli_install` -**Prompt:** +**Expected Tool:** `confidentialledger_entries_append` +**Prompt:** Append an entry to my ledger with data {"key": "value"} ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.479590 | `extension_cli_install` | ✅ **EXPECTED** | -| 2 | 0.473250 | `extension_cli_generate` | ❌ | -| 3 | 0.389354 | `azureterraformbestpractices_get` | ❌ | -| 4 | 0.382389 | `deploy_plan_get` | ❌ | -| 5 | 0.366012 | `get_bestpractices_get` | ❌ | +| 1 | 0.777179 | `confidentialledger_entries_append` | ✅ **EXPECTED** | +| 2 | 0.737938 | `appconfig_kv_set` | ❌ | +| 3 | 0.717614 | `keyvault_secret_create` | ❌ | +| 4 | 0.714455 | `appconfig_kv_lock_set` | ❌ | +| 5 | 0.710804 | `keyvault_key_create` | ❌ | --- ## Test 113 -**Expected Tool:** `extension_cli_install` -**Prompt:** How to install azd +**Expected Tool:** `confidentialledger_entries_append` +**Prompt:** Write a tamper-proof entry to ledger containing {"transaction": "data"} ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.460416 | `extension_cli_install` | ✅ **EXPECTED** | -| 2 | 0.429600 | `deploy_app_logs_get` | ❌ | -| 3 | 0.365212 | `deploy_iac_rules_get` | ❌ | -| 4 | 0.335279 | `deploy_plan_get` | ❌ | -| 5 | 0.326135 | `deploy_pipeline_guidance_get` | ❌ | +| 1 | 0.827993 | `confidentialledger_entries_append` | ✅ **EXPECTED** | +| 2 | 0.743736 | `confidentialledger_entries_get` | ❌ | +| 3 | 0.697179 | `eventgrid_events_publish` | ❌ | +| 4 | 0.693981 | `keyvault_secret_create` | ❌ | +| 5 | 0.691925 | `keyvault_certificate_import` | ❌ | --- ## Test 114 -**Expected Tool:** `extension_cli_install` -**Prompt:** What is Azure Functions Core tools and how to install it +**Expected Tool:** `confidentialledger_entries_append` +**Prompt:** Append {"hello": "from mcp"} to my confidential ledger in collection ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.622705 | `extension_cli_install` | ✅ **EXPECTED** | -| 2 | 0.439474 | `get_bestpractices_get` | ❌ | -| 3 | 0.432913 | `deploy_pipeline_guidance_get` | ❌ | -| 4 | 0.430723 | `extension_cli_generate` | ❌ | -| 5 | 0.418161 | `deploy_plan_get` | ❌ | +| 1 | 0.782923 | `confidentialledger_entries_append` | ✅ **EXPECTED** | +| 2 | 0.750702 | `confidentialledger_entries_get` | ❌ | +| 3 | 0.695460 | `redis_create` | ❌ | +| 4 | 0.693021 | `appconfig_kv_set` | ❌ | +| 5 | 0.686606 | `cosmos_database_container_item_query` | ❌ | --- ## Test 115 -**Expected Tool:** `acr_registry_list` -**Prompt:** List all Azure Container Registries in my subscription +**Expected Tool:** `confidentialledger_entries_append` +**Prompt:** Create an immutable ledger entry in with content {"audit": "log"} ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.743568 | `acr_registry_list` | ✅ **EXPECTED** | -| 2 | 0.711580 | `acr_registry_repository_list` | ❌ | -| 3 | 0.585675 | `kusto_cluster_list` | ❌ | -| 4 | 0.541506 | `search_service_list` | ❌ | -| 5 | 0.514293 | `cosmos_account_list` | ❌ | +| 1 | 0.746159 | `confidentialledger_entries_append` | ✅ **EXPECTED** | +| 2 | 0.721142 | `confidentialledger_entries_get` | ❌ | +| 3 | 0.708151 | `keyvault_secret_create` | ❌ | +| 4 | 0.701706 | `redis_create` | ❌ | +| 5 | 0.700351 | `appconfig_kv_set` | ❌ | --- ## Test 116 -**Expected Tool:** `acr_registry_list` -**Prompt:** Show me my Azure Container Registries +**Expected Tool:** `confidentialledger_entries_append` +**Prompt:** Write an entry to confidential ledger ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.586014 | `acr_registry_list` | ✅ **EXPECTED** | -| 2 | 0.563636 | `acr_registry_repository_list` | ❌ | -| 3 | 0.460544 | `storage_blob_container_get` | ❌ | -| 4 | 0.415552 | `cosmos_database_container_list` | ❌ | -| 5 | 0.402247 | `redis_list` | ❌ | +| 1 | 0.835313 | `confidentialledger_entries_append` | ✅ **EXPECTED** | +| 2 | 0.787289 | `confidentialledger_entries_get` | ❌ | +| 3 | 0.732012 | `keyvault_secret_create` | ❌ | +| 4 | 0.718759 | `keyvault_certificate_import` | ❌ | +| 5 | 0.718445 | `appconfig_kv_lock_set` | ❌ | --- ## Test 117 -**Expected Tool:** `acr_registry_list` -**Prompt:** Show me the container registries in my subscription +**Expected Tool:** `confidentialledger_entries_get` +**Prompt:** Get entry from Confidential Ledger for transaction on ledger ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.637130 | `acr_registry_list` | ✅ **EXPECTED** | -| 2 | 0.563476 | `acr_registry_repository_list` | ❌ | -| 3 | 0.516769 | `kusto_cluster_list` | ❌ | -| 4 | 0.515378 | `storage_blob_container_get` | ❌ | -| 5 | 0.480352 | `redis_list` | ❌ | +| 1 | 0.873510 | `confidentialledger_entries_get` | ✅ **EXPECTED** | +| 2 | 0.817697 | `confidentialledger_entries_append` | ❌ | +| 3 | 0.717577 | `keyvault_certificate_get` | ❌ | +| 4 | 0.704764 | `keyvault_key_get` | ❌ | +| 5 | 0.704575 | `keyvault_secret_get` | ❌ | --- ## Test 118 -**Expected Tool:** `acr_registry_list` -**Prompt:** List container registries in resource group +**Expected Tool:** `confidentialledger_entries_get` +**Prompt:** Get transaction from ledger ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.654318 | `acr_registry_repository_list` | ❌ | -| 2 | 0.633938 | `acr_registry_list` | ✅ **EXPECTED** | -| 3 | 0.476015 | `mysql_server_list` | ❌ | -| 4 | 0.454929 | `group_list` | ❌ | -| 5 | 0.454003 | `datadog_monitoredresources_list` | ❌ | +| 1 | 0.794213 | `confidentialledger_entries_get` | ✅ **EXPECTED** | +| 2 | 0.734616 | `confidentialledger_entries_append` | ❌ | +| 3 | 0.732828 | `keyvault_key_get` | ❌ | +| 4 | 0.728510 | `keyvault_certificate_get` | ❌ | +| 5 | 0.717596 | `cosmos_database_container_item_query` | ❌ | --- ## Test 119 -**Expected Tool:** `acr_registry_list` -**Prompt:** Show me the container registries in resource group +**Expected Tool:** `cosmos_account_list` +**Prompt:** List all cosmosdb accounts in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.639391 | `acr_registry_list` | ✅ **EXPECTED** | -| 2 | 0.637972 | `acr_registry_repository_list` | ❌ | -| 3 | 0.468028 | `mysql_server_list` | ❌ | -| 4 | 0.449649 | `datadog_monitoredresources_list` | ❌ | -| 5 | 0.445741 | `group_list` | ❌ | +| 1 | 0.911865 | `cosmos_account_list` | ✅ **EXPECTED** | +| 2 | 0.853519 | `cosmos_database_list` | ❌ | +| 3 | 0.840446 | `kusto_cluster_list` | ❌ | +| 4 | 0.823181 | `subscription_list` | ❌ | +| 5 | 0.817476 | `cosmos_database_container_list` | ❌ | --- ## Test 120 -**Expected Tool:** `acr_registry_repository_list` -**Prompt:** List all container registry repositories in my subscription +**Expected Tool:** `cosmos_account_list` +**Prompt:** Show me my cosmosdb accounts ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.626482 | `acr_registry_repository_list` | ✅ **EXPECTED** | -| 2 | 0.617504 | `acr_registry_list` | ❌ | -| 3 | 0.544172 | `kusto_cluster_list` | ❌ | -| 4 | 0.508483 | `storage_blob_container_get` | ❌ | -| 5 | 0.495567 | `postgres_server_list` | ❌ | +| 1 | 0.839227 | `cosmos_account_list` | ✅ **EXPECTED** | +| 2 | 0.837729 | `cosmos_database_list` | ❌ | +| 3 | 0.809637 | `cosmos_database_container_list` | ❌ | +| 4 | 0.793189 | `cosmos_database_container_item_query` | ❌ | +| 5 | 0.780946 | `storage_account_get` | ❌ | --- ## Test 121 -**Expected Tool:** `acr_registry_repository_list` -**Prompt:** Show me my container registry repositories +**Expected Tool:** `cosmos_account_list` +**Prompt:** Show me the cosmosdb accounts in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.546334 | `acr_registry_repository_list` | ✅ **EXPECTED** | -| 2 | 0.469295 | `acr_registry_list` | ❌ | -| 3 | 0.451083 | `storage_blob_container_get` | ❌ | -| 4 | 0.407973 | `cosmos_database_container_list` | ❌ | -| 5 | 0.373464 | `storage_blob_get` | ❌ | +| 1 | 0.880897 | `cosmos_account_list` | ✅ **EXPECTED** | +| 2 | 0.825658 | `cosmos_database_list` | ❌ | +| 3 | 0.811468 | `kusto_cluster_list` | ❌ | +| 4 | 0.808962 | `redis_list` | ❌ | +| 5 | 0.804846 | `subscription_list` | ❌ | --- ## Test 122 -**Expected Tool:** `acr_registry_repository_list` -**Prompt:** List repositories in the container registry +**Expected Tool:** `cosmos_database_container_item_query` +**Prompt:** Show me the items that contain the word in the container in the database for the cosmosdb account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.674296 | `acr_registry_repository_list` | ✅ **EXPECTED** | -| 2 | 0.541779 | `acr_registry_list` | ❌ | -| 3 | 0.437510 | `storage_blob_container_get` | ❌ | -| 4 | 0.433927 | `cosmos_database_container_list` | ❌ | -| 5 | 0.383160 | `kusto_database_list` | ❌ | +| 1 | 0.874731 | `cosmos_database_container_item_query` | ✅ **EXPECTED** | +| 2 | 0.849399 | `cosmos_database_container_list` | ❌ | +| 3 | 0.795628 | `cosmos_database_list` | ❌ | +| 4 | 0.785297 | `kusto_query` | ❌ | +| 5 | 0.785097 | `storage_blob_container_get` | ❌ | --- ## Test 123 -**Expected Tool:** `acr_registry_repository_list` -**Prompt:** Show me the repositories in the container registry +**Expected Tool:** `cosmos_database_container_list` +**Prompt:** List all the containers in the database for the cosmosdb account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.600780 | `acr_registry_repository_list` | ✅ **EXPECTED** | -| 2 | 0.501842 | `acr_registry_list` | ❌ | -| 3 | 0.430880 | `storage_blob_container_get` | ❌ | -| 4 | 0.418623 | `cosmos_database_container_list` | ❌ | -| 5 | 0.378151 | `redis_list` | ❌ | +| 1 | 0.921300 | `cosmos_database_container_list` | ✅ **EXPECTED** | +| 2 | 0.880697 | `cosmos_database_container_item_query` | ❌ | +| 3 | 0.867517 | `cosmos_database_list` | ❌ | +| 4 | 0.824383 | `cosmos_account_list` | ❌ | +| 5 | 0.809411 | `storage_blob_container_get` | ❌ | --- ## Test 124 -**Expected Tool:** `communication_email_send` -**Prompt:** Send an email to with subject +**Expected Tool:** `cosmos_database_container_list` +**Prompt:** Show me the containers in the database for the cosmosdb account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.498292 | `communication_email_send` | ✅ **EXPECTED** | -| 2 | 0.229080 | `communication_sms_send` | ❌ | -| 3 | 0.188975 | `eventgrid_events_publish` | ❌ | -| 4 | 0.161257 | `foundry_agents_create` | ❌ | -| 5 | 0.145951 | `servicebus_topic_details` | ❌ | +| 1 | 0.900700 | `cosmos_database_container_list` | ✅ **EXPECTED** | +| 2 | 0.863402 | `cosmos_database_container_item_query` | ❌ | +| 3 | 0.853449 | `cosmos_database_list` | ❌ | +| 4 | 0.818800 | `storage_blob_container_get` | ❌ | +| 5 | 0.815366 | `cosmos_account_list` | ❌ | --- ## Test 125 -**Expected Tool:** `communication_email_send` -**Prompt:** Send an email from my communication service to +**Expected Tool:** `cosmos_database_list` +**Prompt:** List all the databases in the cosmosdb account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.498406 | `communication_email_send` | ✅ **EXPECTED** | -| 2 | 0.314462 | `communication_sms_send` | ❌ | -| 3 | 0.235127 | `foundry_openai_chat-completions-create` | ❌ | -| 4 | 0.215392 | `speech_tts_synthesize` | ❌ | -| 5 | 0.211154 | `search_knowledge_base_retrieve` | ❌ | +| 1 | 0.920574 | `cosmos_database_list` | ✅ **EXPECTED** | +| 2 | 0.858775 | `cosmos_database_container_list` | ❌ | +| 3 | 0.847820 | `cosmos_account_list` | ❌ | +| 4 | 0.844652 | `postgres_database_list` | ❌ | +| 5 | 0.831337 | `kusto_database_list` | ❌ | --- ## Test 126 -**Expected Tool:** `communication_email_send` -**Prompt:** Send HTML-formatted email to with subject +**Expected Tool:** `cosmos_database_list` +**Prompt:** Show me the databases in the cosmosdb account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.520967 | `communication_email_send` | ✅ **EXPECTED** | -| 2 | 0.207658 | `communication_sms_send` | ❌ | -| 3 | 0.152418 | `eventgrid_events_publish` | ❌ | -| 4 | 0.152013 | `servicebus_topic_details` | ❌ | -| 5 | 0.143660 | `foundry_agents_evaluate` | ❌ | +| 1 | 0.908829 | `cosmos_database_list` | ✅ **EXPECTED** | +| 2 | 0.855188 | `cosmos_database_container_list` | ❌ | +| 3 | 0.841841 | `cosmos_account_list` | ❌ | +| 4 | 0.837447 | `postgres_database_list` | ❌ | +| 5 | 0.824852 | `kusto_database_list` | ❌ | --- ## Test 127 -**Expected Tool:** `communication_email_send` -**Prompt:** Send email with CC to and +**Expected Tool:** `kusto_cluster_get` +**Prompt:** Show me the details of the Data Explorer cluster ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.533447 | `communication_email_send` | ✅ **EXPECTED** | -| 2 | 0.219584 | `communication_sms_send` | ❌ | -| 3 | 0.106041 | `foundry_agents_query-and-evaluate` | ❌ | -| 4 | 0.103723 | `foundry_openai_chat-completions-create` | ❌ | -| 5 | 0.084905 | `cosmos_account_list` | ❌ | +| 1 | 0.835197 | `kusto_cluster_get` | ✅ **EXPECTED** | +| 2 | 0.819379 | `kusto_cluster_list` | ❌ | +| 3 | 0.812131 | `kusto_database_list` | ❌ | +| 4 | 0.811714 | `kusto_table_schema` | ❌ | +| 5 | 0.800691 | `aks_cluster_get` | ❌ | --- ## Test 128 -**Expected Tool:** `communication_email_send` -**Prompt:** Send email to multiple recipients: , +**Expected Tool:** `kusto_cluster_list` +**Prompt:** List all Data Explorer clusters in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.540793 | `communication_email_send` | ✅ **EXPECTED** | -| 2 | 0.244521 | `communication_sms_send` | ❌ | -| 3 | 0.134975 | `foundry_openai_chat-completions-create` | ❌ | -| 4 | 0.114325 | `foundry_agents_query-and-evaluate` | ❌ | -| 5 | 0.087063 | `postgres_server_param_set` | ❌ | +| 1 | 0.927361 | `kusto_cluster_list` | ✅ **EXPECTED** | +| 2 | 0.862525 | `aks_cluster_get` | ❌ | +| 3 | 0.852044 | `kusto_database_list` | ❌ | +| 4 | 0.846470 | `kusto_cluster_get` | ❌ | +| 5 | 0.840089 | `search_service_list` | ❌ | --- ## Test 129 -**Expected Tool:** `communication_email_send` -**Prompt:** Send email with reply-to address set to +**Expected Tool:** `kusto_cluster_list` +**Prompt:** Show me my Data Explorer clusters ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.512623 | `communication_email_send` | ✅ **EXPECTED** | -| 2 | 0.200177 | `communication_sms_send` | ❌ | -| 3 | 0.164115 | `mysql_server_param_set` | ❌ | -| 4 | 0.158759 | `postgres_server_param_set` | ❌ | -| 5 | 0.143574 | `appconfig_kv_set` | ❌ | +| 1 | 0.840872 | `kusto_cluster_list` | ✅ **EXPECTED** | +| 2 | 0.815379 | `kusto_database_list` | ❌ | +| 3 | 0.804825 | `kusto_table_schema` | ❌ | +| 4 | 0.803327 | `kusto_cluster_get` | ❌ | +| 5 | 0.799965 | `aks_cluster_get` | ❌ | --- ## Test 130 -**Expected Tool:** `communication_email_send` -**Prompt:** Send email with custom sender name +**Expected Tool:** `kusto_cluster_list` +**Prompt:** Show me the Data Explorer clusters in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.473175 | `communication_email_send` | ✅ **EXPECTED** | -| 2 | 0.255169 | `communication_sms_send` | ❌ | -| 3 | 0.164811 | `foundry_openai_chat-completions-create` | ❌ | -| 4 | 0.160393 | `foundry_openai_embeddings-create` | ❌ | -| 5 | 0.156791 | `cosmos_database_container_item_query` | ❌ | +| 1 | 0.886281 | `kusto_cluster_list` | ✅ **EXPECTED** | +| 2 | 0.846745 | `kusto_cluster_get` | ❌ | +| 3 | 0.835073 | `aks_cluster_get` | ❌ | +| 4 | 0.828444 | `kusto_database_list` | ❌ | +| 5 | 0.823721 | `eventgrid_subscription_list` | ❌ | --- ## Test 131 -**Expected Tool:** `communication_email_send` -**Prompt:** Send an email with BCC recipients +**Expected Tool:** `kusto_database_list` +**Prompt:** List all databases in the Data Explorer cluster ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.528789 | `communication_email_send` | ✅ **EXPECTED** | -| 2 | 0.241114 | `communication_sms_send` | ❌ | -| 3 | 0.137538 | `confidentialledger_entries_append` | ❌ | -| 4 | 0.108748 | `confidentialledger_entries_get` | ❌ | -| 5 | 0.105033 | `storage_blob_upload` | ❌ | +| 1 | 0.884062 | `kusto_database_list` | ✅ **EXPECTED** | +| 2 | 0.855148 | `kusto_cluster_list` | ❌ | +| 3 | 0.855056 | `kusto_table_list` | ❌ | +| 4 | 0.851531 | `postgres_database_list` | ❌ | +| 5 | 0.835723 | `cosmos_database_list` | ❌ | --- ## Test 132 -**Expected Tool:** `communication_sms_send` -**Prompt:** Send an SMS message to saying "Hello" +**Expected Tool:** `kusto_database_list` +**Prompt:** Show me the databases in the Data Explorer cluster ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.533868 | `communication_sms_send` | ✅ **EXPECTED** | -| 2 | 0.251429 | `communication_email_send` | ❌ | -| 3 | 0.218656 | `foundry_openai_chat-completions-create` | ❌ | -| 4 | 0.175534 | `foundry_agents_create` | ❌ | -| 5 | 0.166041 | `speech_tts_synthesize` | ❌ | +| 1 | 0.868395 | `kusto_database_list` | ✅ **EXPECTED** | +| 2 | 0.840656 | `kusto_table_list` | ❌ | +| 3 | 0.838742 | `kusto_cluster_list` | ❌ | +| 4 | 0.838492 | `postgres_database_list` | ❌ | +| 5 | 0.827136 | `kusto_table_schema` | ❌ | --- ## Test 133 -**Expected Tool:** `communication_sms_send` -**Prompt:** Send SMS to from with message "Test message" +**Expected Tool:** `kusto_query` +**Prompt:** Show me all items that contain the word in the Data Explorer table in cluster ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.546019 | `communication_sms_send` | ✅ **EXPECTED** | -| 2 | 0.294860 | `communication_email_send` | ❌ | -| 3 | 0.204588 | `loadtesting_testrun_create` | ❌ | -| 4 | 0.200655 | `foundry_openai_chat-completions-create` | ❌ | -| 5 | 0.141113 | `foundry_agents_create` | ❌ | +| 1 | 0.793703 | `kusto_query` | ✅ **EXPECTED** | +| 2 | 0.790473 | `postgres_database_query` | ❌ | +| 3 | 0.783002 | `kusto_table_list` | ❌ | +| 4 | 0.777589 | `kusto_cluster_list` | ❌ | +| 5 | 0.775796 | `kusto_sample` | ❌ | --- ## Test 134 -**Expected Tool:** `communication_sms_send` -**Prompt:** Send SMS to multiple recipients: , +**Expected Tool:** `kusto_sample` +**Prompt:** Show me a data sample from the Data Explorer table in cluster ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.545755 | `communication_sms_send` | ✅ **EXPECTED** | -| 2 | 0.421988 | `communication_email_send` | ❌ | -| 3 | 0.186088 | `foundry_openai_chat-completions-create` | ❌ | -| 4 | 0.142074 | `foundry_agents_query-and-evaluate` | ❌ | -| 5 | 0.113722 | `foundry_threads_get-messages` | ❌ | +| 1 | 0.852149 | `kusto_sample` | ✅ **EXPECTED** | +| 2 | 0.824247 | `kusto_table_schema` | ❌ | +| 3 | 0.799034 | `kusto_table_list` | ❌ | +| 4 | 0.793904 | `kusto_cluster_list` | ❌ | +| 5 | 0.792562 | `kusto_database_list` | ❌ | --- ## Test 135 -**Expected Tool:** `communication_sms_send` -**Prompt:** Send SMS with delivery reporting enabled +**Expected Tool:** `kusto_table_list` +**Prompt:** List all tables in the Data Explorer database in cluster ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.554908 | `communication_sms_send` | ✅ **EXPECTED** | -| 2 | 0.269080 | `communication_email_send` | ❌ | -| 3 | 0.191848 | `extension_azqr` | ❌ | -| 4 | 0.185916 | `foundry_openai_chat-completions-create` | ❌ | -| 5 | 0.170801 | `foundry_agents_query-and-evaluate` | ❌ | +| 1 | 0.881083 | `kusto_table_list` | ✅ **EXPECTED** | +| 2 | 0.847942 | `kusto_database_list` | ❌ | +| 3 | 0.847927 | `postgres_table_list` | ❌ | +| 4 | 0.834679 | `mysql_table_list` | ❌ | +| 5 | 0.834263 | `kusto_table_schema` | ❌ | --- ## Test 136 -**Expected Tool:** `communication_sms_send` -**Prompt:** Send SMS message with custom tracking tag "campaign1" +**Expected Tool:** `kusto_table_list` +**Prompt:** Show me the tables in the Data Explorer database in cluster ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.538827 | `communication_sms_send` | ✅ **EXPECTED** | -| 2 | 0.269794 | `communication_email_send` | ❌ | -| 3 | 0.188153 | `loadtesting_testrun_create` | ❌ | -| 4 | 0.185403 | `foundry_openai_chat-completions-create` | ❌ | -| 5 | 0.175135 | `foundry_agents_create` | ❌ | +| 1 | 0.864747 | `kusto_table_list` | ✅ **EXPECTED** | +| 2 | 0.850075 | `kusto_table_schema` | ❌ | +| 3 | 0.840143 | `postgres_table_list` | ❌ | +| 4 | 0.831394 | `kusto_database_list` | ❌ | +| 5 | 0.829153 | `mysql_table_list` | ❌ | --- ## Test 137 -**Expected Tool:** `communication_sms_send` -**Prompt:** Send broadcast SMS to and saying "Urgent notification" +**Expected Tool:** `kusto_table_schema` +**Prompt:** Show me the schema for table in the Data Explorer database in cluster ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.474786 | `communication_sms_send` | ✅ **EXPECTED** | -| 2 | 0.286338 | `communication_email_send` | ❌ | -| 3 | 0.164373 | `foundry_agents_query-and-evaluate` | ❌ | -| 4 | 0.147338 | `foundry_openai_chat-completions-create` | ❌ | -| 5 | 0.128704 | `cosmos_account_list` | ❌ | +| 1 | 0.889776 | `kusto_table_schema` | ✅ **EXPECTED** | +| 2 | 0.832717 | `kusto_table_list` | ❌ | +| 3 | 0.832316 | `postgres_table_schema_get` | ❌ | +| 4 | 0.831456 | `mysql_table_schema_get` | ❌ | +| 5 | 0.819940 | `postgres_table_list` | ❌ | --- ## Test 138 -**Expected Tool:** `communication_sms_send` -**Prompt:** Send SMS from my communication service to +**Expected Tool:** `mysql_database_list` +**Prompt:** List all MySQL databases in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.564114 | `communication_sms_send` | ✅ **EXPECTED** | -| 2 | 0.302363 | `communication_email_send` | ❌ | -| 3 | 0.238296 | `foundry_openai_chat-completions-create` | ❌ | -| 4 | 0.184265 | `foundry_agents_create` | ❌ | -| 5 | 0.183651 | `search_knowledge_base_retrieve` | ❌ | +| 1 | 0.877674 | `postgres_database_list` | ❌ | +| 2 | 0.850027 | `mysql_database_list` | ✅ **EXPECTED** | +| 3 | 0.835616 | `mysql_table_list` | ❌ | +| 4 | 0.818887 | `postgres_table_list` | ❌ | +| 5 | 0.808188 | `mysql_server_list` | ❌ | --- ## Test 139 -**Expected Tool:** `communication_sms_send` -**Prompt:** Send an SMS with delivery receipt tracking +**Expected Tool:** `mysql_database_list` +**Prompt:** Show me the MySQL databases in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.598211 | `communication_sms_send` | ✅ **EXPECTED** | -| 2 | 0.314134 | `communication_email_send` | ❌ | -| 3 | 0.206967 | `foundry_agents_query-and-evaluate` | ❌ | -| 4 | 0.201142 | `foundry_openai_chat-completions-create` | ❌ | -| 5 | 0.187824 | `confidentialledger_entries_append` | ❌ | +| 1 | 0.856156 | `postgres_database_list` | ❌ | +| 2 | 0.843299 | `mysql_database_list` | ✅ **EXPECTED** | +| 3 | 0.831006 | `mysql_table_list` | ❌ | +| 4 | 0.807422 | `mysql_server_list` | ❌ | +| 5 | 0.806940 | `postgres_table_list` | ❌ | --- ## Test 140 -**Expected Tool:** `confidentialledger_entries_append` -**Prompt:** Append an entry to my ledger with data {"key": "value"} +**Expected Tool:** `mysql_database_query` +**Prompt:** Show me all items that contain the word in the MySQL database in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.510650 | `confidentialledger_entries_append` | ✅ **EXPECTED** | -| 2 | 0.294885 | `confidentialledger_entries_get` | ❌ | -| 3 | 0.292014 | `appconfig_kv_set` | ❌ | -| 4 | 0.258967 | `appconfig_kv_lock_set` | ❌ | -| 5 | 0.249908 | `keyvault_certificate_import` | ❌ | +| 1 | 0.783082 | `postgres_database_query` | ❌ | +| 2 | 0.781748 | `mysql_table_list` | ❌ | +| 3 | 0.755838 | `mysql_database_list` | ❌ | +| 4 | 0.753655 | `mysql_server_list` | ❌ | +| 5 | 0.751427 | `postgres_database_list` | ❌ | --- ## Test 141 -**Expected Tool:** `confidentialledger_entries_append` -**Prompt:** Write a tamper-proof entry to ledger containing {"transaction": "data"} +**Expected Tool:** `mysql_server_config_get` +**Prompt:** Show me the configuration of MySQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.602247 | `confidentialledger_entries_append` | ✅ **EXPECTED** | -| 2 | 0.357646 | `confidentialledger_entries_get` | ❌ | -| 3 | 0.211990 | `appconfig_kv_lock_set` | ❌ | -| 4 | 0.195471 | `keyvault_secret_create` | ❌ | -| 5 | 0.184077 | `keyvault_certificate_import` | ❌ | +| 1 | 0.818327 | `mysql_server_config_get` | ✅ **EXPECTED** | +| 2 | 0.802899 | `postgres_server_config_get` | ❌ | +| 3 | 0.789340 | `mysql_table_schema_get` | ❌ | +| 4 | 0.786809 | `mysql_server_param_set` | ❌ | +| 5 | 0.779135 | `postgres_server_param_set` | ❌ | --- ## Test 142 -**Expected Tool:** `confidentialledger_entries_append` -**Prompt:** Append {"hello": "from mcp"} to my confidential ledger in collection +**Expected Tool:** `mysql_server_list` +**Prompt:** List all MySQL servers in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.546660 | `confidentialledger_entries_append` | ✅ **EXPECTED** | -| 2 | 0.451994 | `confidentialledger_entries_get` | ❌ | -| 3 | 0.225141 | `appconfig_kv_lock_set` | ❌ | -| 4 | 0.215932 | `appconfig_kv_set` | ❌ | -| 5 | 0.203262 | `keyvault_certificate_import` | ❌ | +| 1 | 0.864807 | `postgres_server_list` | ❌ | +| 2 | 0.824894 | `kusto_cluster_list` | ❌ | +| 3 | 0.816137 | `mysql_server_list` | ✅ **EXPECTED** | +| 4 | 0.811819 | `mysql_database_list` | ❌ | +| 5 | 0.809186 | `mysql_table_list` | ❌ | --- ## Test 143 -**Expected Tool:** `confidentialledger_entries_append` -**Prompt:** Create an immutable ledger entry in with content {"audit": "log"} +**Expected Tool:** `mysql_server_list` +**Prompt:** Show me my MySQL servers ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.495873 | `confidentialledger_entries_append` | ✅ **EXPECTED** | -| 2 | 0.340018 | `confidentialledger_entries_get` | ❌ | -| 3 | 0.218351 | `monitor_activitylog_list` | ❌ | -| 4 | 0.215093 | `storage_blob_container_create` | ❌ | -| 5 | 0.204967 | `monitor_resource_log_query` | ❌ | +| 1 | 0.807672 | `mysql_database_list` | ❌ | +| 2 | 0.804885 | `mysql_server_list` | ✅ **EXPECTED** | +| 3 | 0.803458 | `mysql_table_list` | ❌ | +| 4 | 0.793162 | `postgres_database_list` | ❌ | +| 5 | 0.785478 | `mysql_server_config_get` | ❌ | --- ## Test 144 -**Expected Tool:** `confidentialledger_entries_append` -**Prompt:** Write an entry to confidential ledger +**Expected Tool:** `mysql_server_list` +**Prompt:** Show me the MySQL servers in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.622138 | `confidentialledger_entries_append` | ✅ **EXPECTED** | -| 2 | 0.524777 | `confidentialledger_entries_get` | ❌ | -| 3 | 0.252508 | `appconfig_kv_lock_set` | ❌ | -| 4 | 0.240252 | `keyvault_secret_create` | ❌ | -| 5 | 0.186890 | `appconfig_kv_set` | ❌ | +| 1 | 0.839670 | `postgres_server_list` | ❌ | +| 2 | 0.804401 | `redis_list` | ❌ | +| 3 | 0.804145 | `mysql_server_list` | ✅ **EXPECTED** | +| 4 | 0.794522 | `kusto_cluster_list` | ❌ | +| 5 | 0.794384 | `eventgrid_subscription_list` | ❌ | --- ## Test 145 -**Expected Tool:** `confidentialledger_entries_get` -**Prompt:** Get entry from Confidential Ledger for transaction on ledger +**Expected Tool:** `mysql_server_param_get` +**Prompt:** Show me the value of connection timeout in seconds in my MySQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.707267 | `confidentialledger_entries_get` | ✅ **EXPECTED** | -| 2 | 0.551859 | `confidentialledger_entries_append` | ❌ | -| 3 | 0.245626 | `keyvault_secret_get` | ❌ | -| 4 | 0.230031 | `keyvault_key_get` | ❌ | -| 5 | 0.211897 | `loadtesting_testrun_get` | ❌ | +| 1 | 0.796653 | `mysql_server_param_get` | ✅ **EXPECTED** | +| 2 | 0.752874 | `mysql_server_param_set` | ❌ | +| 3 | 0.751912 | `mysql_server_config_get` | ❌ | +| 4 | 0.745646 | `mysql_table_schema_get` | ❌ | +| 5 | 0.744210 | `mysql_database_query` | ❌ | --- ## Test 146 -**Expected Tool:** `confidentialledger_entries_get` -**Prompt:** Get transaction from ledger +**Expected Tool:** `mysql_server_param_set` +**Prompt:** Set connection timeout to 20 seconds for my MySQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.509714 | `confidentialledger_entries_get` | ✅ **EXPECTED** | -| 2 | 0.416580 | `confidentialledger_entries_append` | ❌ | -| 3 | 0.223959 | `loadtesting_testrun_get` | ❌ | -| 4 | 0.218412 | `monitor_resource_log_query` | ❌ | -| 5 | 0.217671 | `loadtesting_testrun_list` | ❌ | +| 1 | 0.768077 | `mysql_server_param_set` | ✅ **EXPECTED** | +| 2 | 0.758285 | `mysql_server_param_get` | ❌ | +| 3 | 0.749990 | `postgres_server_param_set` | ❌ | +| 4 | 0.732204 | `mysql_database_query` | ❌ | +| 5 | 0.722285 | `mysql_server_config_get` | ❌ | --- ## Test 147 -**Expected Tool:** `cosmos_account_list` -**Prompt:** List all cosmosdb accounts in my subscription +**Expected Tool:** `mysql_table_list` +**Prompt:** List all tables in the MySQL database in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.818357 | `cosmos_account_list` | ✅ **EXPECTED** | -| 2 | 0.668480 | `cosmos_database_list` | ❌ | -| 3 | 0.636059 | `subscription_list` | ❌ | -| 4 | 0.615268 | `cosmos_database_container_list` | ❌ | -| 5 | 0.601467 | `kusto_cluster_list` | ❌ | +| 1 | 0.869002 | `mysql_table_list` | ✅ **EXPECTED** | +| 2 | 0.866280 | `postgres_table_list` | ❌ | +| 3 | 0.844293 | `postgres_database_list` | ❌ | +| 4 | 0.829906 | `kusto_table_list` | ❌ | +| 5 | 0.829767 | `mysql_database_list` | ❌ | --- ## Test 148 -**Expected Tool:** `cosmos_account_list` -**Prompt:** Show me my cosmosdb accounts +**Expected Tool:** `mysql_table_list` +**Prompt:** Show me the tables in the MySQL database in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.665447 | `cosmos_account_list` | ✅ **EXPECTED** | -| 2 | 0.605357 | `cosmos_database_list` | ❌ | -| 3 | 0.571613 | `cosmos_database_container_list` | ❌ | -| 4 | 0.549555 | `cosmos_database_container_item_query` | ❌ | -| 5 | 0.503830 | `storage_account_get` | ❌ | +| 1 | 0.855600 | `mysql_table_list` | ✅ **EXPECTED** | +| 2 | 0.847666 | `postgres_table_list` | ❌ | +| 3 | 0.824616 | `postgres_database_list` | ❌ | +| 4 | 0.821093 | `mysql_database_list` | ❌ | +| 5 | 0.819615 | `mysql_table_schema_get` | ❌ | --- ## Test 149 -**Expected Tool:** `cosmos_account_list` -**Prompt:** Show me the cosmosdb accounts in my subscription +**Expected Tool:** `mysql_table_schema_get` +**Prompt:** Show me the schema of table
in the MySQL database in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.752494 | `cosmos_account_list` | ✅ **EXPECTED** | -| 2 | 0.607235 | `subscription_list` | ❌ | -| 3 | 0.605125 | `cosmos_database_list` | ❌ | -| 4 | 0.566249 | `cosmos_database_container_list` | ❌ | -| 5 | 0.563963 | `cosmos_database_container_item_query` | ❌ | +| 1 | 0.851928 | `mysql_table_schema_get` | ✅ **EXPECTED** | +| 2 | 0.830822 | `postgres_table_schema_get` | ❌ | +| 3 | 0.826168 | `kusto_table_schema` | ❌ | +| 4 | 0.826033 | `mysql_table_list` | ❌ | +| 5 | 0.818184 | `postgres_table_list` | ❌ | --- ## Test 150 -**Expected Tool:** `cosmos_database_container_item_query` -**Prompt:** Show me the items that contain the word in the container in the database for the cosmosdb account +**Expected Tool:** `postgres_database_list` +**Prompt:** List all PostgreSQL databases in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.658771 | `cosmos_database_container_item_query` | ✅ **EXPECTED** | -| 2 | 0.605253 | `cosmos_database_container_list` | ❌ | -| 3 | 0.487612 | `storage_blob_container_get` | ❌ | -| 4 | 0.477874 | `cosmos_database_list` | ❌ | -| 5 | 0.447757 | `cosmos_account_list` | ❌ | +| 1 | 0.943988 | `postgres_database_list` | ✅ **EXPECTED** | +| 2 | 0.886663 | `postgres_table_list` | ❌ | +| 3 | 0.852392 | `postgres_server_list` | ❌ | +| 4 | 0.810673 | `postgres_server_config_get` | ❌ | +| 5 | 0.799363 | `kusto_table_list` | ❌ | --- ## Test 151 -**Expected Tool:** `cosmos_database_container_list` -**Prompt:** List all the containers in the database for the cosmosdb account +**Expected Tool:** `postgres_database_list` +**Prompt:** Show me the PostgreSQL databases in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.852832 | `cosmos_database_container_list` | ✅ **EXPECTED** | -| 2 | 0.681044 | `cosmos_database_list` | ❌ | -| 3 | 0.680819 | `cosmos_database_container_item_query` | ❌ | -| 4 | 0.632335 | `storage_blob_container_get` | ❌ | -| 5 | 0.630659 | `cosmos_account_list` | ❌ | +| 1 | 0.926930 | `postgres_database_list` | ✅ **EXPECTED** | +| 2 | 0.877987 | `postgres_table_list` | ❌ | +| 3 | 0.842093 | `postgres_server_list` | ❌ | +| 4 | 0.820788 | `postgres_server_config_get` | ❌ | +| 5 | 0.809743 | `postgres_table_schema_get` | ❌ | --- ## Test 152 -**Expected Tool:** `cosmos_database_container_list` -**Prompt:** Show me the containers in the database for the cosmosdb account +**Expected Tool:** `postgres_database_query` +**Prompt:** Show me all items that contain the word in the PostgreSQL database in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.789395 | `cosmos_database_container_list` | ✅ **EXPECTED** | -| 2 | 0.648199 | `cosmos_database_container_item_query` | ❌ | -| 3 | 0.614220 | `cosmos_database_list` | ❌ | -| 4 | 0.591361 | `storage_blob_container_get` | ❌ | -| 5 | 0.562062 | `cosmos_account_list` | ❌ | +| 1 | 0.829593 | `postgres_database_query` | ✅ **EXPECTED** | +| 2 | 0.808740 | `postgres_database_list` | ❌ | +| 3 | 0.806504 | `postgres_table_list` | ❌ | +| 4 | 0.785447 | `postgres_server_list` | ❌ | +| 5 | 0.756521 | `postgres_server_param_get` | ❌ | --- ## Test 153 -**Expected Tool:** `cosmos_database_list` -**Prompt:** List all the databases in the cosmosdb account +**Expected Tool:** `postgres_server_config_get` +**Prompt:** Show me the configuration of PostgreSQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.815683 | `cosmos_database_list` | ✅ **EXPECTED** | -| 2 | 0.668515 | `cosmos_account_list` | ❌ | -| 3 | 0.665298 | `cosmos_database_container_list` | ❌ | -| 4 | 0.606462 | `cosmos_database_container_item_query` | ❌ | -| 5 | 0.583308 | `kusto_database_list` | ❌ | +| 1 | 0.895026 | `postgres_server_config_get` | ✅ **EXPECTED** | +| 2 | 0.859290 | `postgres_server_param_set` | ❌ | +| 3 | 0.833239 | `postgres_database_list` | ❌ | +| 4 | 0.820952 | `postgres_table_list` | ❌ | +| 5 | 0.804190 | `postgres_server_param_get` | ❌ | --- ## Test 154 -**Expected Tool:** `cosmos_database_list` -**Prompt:** Show me the databases in the cosmosdb account +**Expected Tool:** `postgres_server_list` +**Prompt:** List all PostgreSQL servers in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.749370 | `cosmos_database_list` | ✅ **EXPECTED** | -| 2 | 0.624759 | `cosmos_database_container_list` | ❌ | -| 3 | 0.614572 | `cosmos_account_list` | ❌ | -| 4 | 0.579996 | `cosmos_database_container_item_query` | ❌ | -| 5 | 0.538479 | `mysql_database_list` | ❌ | +| 1 | 0.946732 | `postgres_server_list` | ✅ **EXPECTED** | +| 2 | 0.861266 | `postgres_database_list` | ❌ | +| 3 | 0.841747 | `postgres_table_list` | ❌ | +| 4 | 0.831264 | `kusto_cluster_list` | ❌ | +| 5 | 0.803690 | `search_service_list` | ❌ | --- ## Test 155 -**Expected Tool:** `kusto_cluster_get` -**Prompt:** Show me the details of the Data Explorer cluster +**Expected Tool:** `postgres_server_list` +**Prompt:** Show me my PostgreSQL servers ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.592860 | `kusto_cluster_get` | ✅ **EXPECTED** | -| 2 | 0.463832 | `kusto_cluster_list` | ❌ | -| 3 | 0.428159 | `kusto_query` | ❌ | -| 4 | 0.425743 | `kusto_database_list` | ❌ | -| 5 | 0.422577 | `kusto_table_schema` | ❌ | +| 1 | 0.863293 | `postgres_database_list` | ❌ | +| 2 | 0.854387 | `postgres_server_list` | ✅ **EXPECTED** | +| 3 | 0.846394 | `postgres_table_list` | ❌ | +| 4 | 0.825378 | `postgres_server_config_get` | ❌ | +| 5 | 0.812839 | `postgres_server_param_set` | ❌ | --- ## Test 156 -**Expected Tool:** `kusto_cluster_list` -**Prompt:** List all Data Explorer clusters in my subscription +**Expected Tool:** `postgres_server_list` +**Prompt:** Show me the PostgreSQL servers in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.793744 | `kusto_cluster_list` | ✅ **EXPECTED** | -| 2 | 0.630596 | `kusto_database_list` | ❌ | -| 3 | 0.571596 | `kusto_cluster_get` | ❌ | -| 4 | 0.525025 | `aks_cluster_get` | ❌ | -| 5 | 0.509397 | `grafana_list` | ❌ | +| 1 | 0.920229 | `postgres_server_list` | ✅ **EXPECTED** | +| 2 | 0.836549 | `postgres_database_list` | ❌ | +| 3 | 0.823609 | `postgres_table_list` | ❌ | +| 4 | 0.804581 | `kusto_cluster_list` | ❌ | +| 5 | 0.801470 | `postgres_server_config_get` | ❌ | --- ## Test 157 -**Expected Tool:** `kusto_cluster_list` -**Prompt:** Show me my Data Explorer clusters +**Expected Tool:** `postgres_server_param_get` +**Prompt:** Show me if the parameter my PostgreSQL server has replication enabled ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.531307 | `kusto_cluster_list` | ✅ **EXPECTED** | -| 2 | 0.467899 | `kusto_cluster_get` | ❌ | -| 3 | 0.432349 | `kusto_database_list` | ❌ | -| 4 | 0.369596 | `aks_cluster_get` | ❌ | -| 5 | 0.363119 | `kusto_table_schema` | ❌ | +| 1 | 0.827655 | `postgres_server_param_set` | ❌ | +| 2 | 0.806040 | `postgres_server_param_get` | ✅ **EXPECTED** | +| 3 | 0.800510 | `postgres_server_config_get` | ❌ | +| 4 | 0.791725 | `postgres_server_list` | ❌ | +| 5 | 0.788745 | `postgres_database_list` | ❌ | --- ## Test 158 -**Expected Tool:** `kusto_cluster_list` -**Prompt:** Show me the Data Explorer clusters in my subscription +**Expected Tool:** `postgres_server_param_set` +**Prompt:** Enable replication for my PostgreSQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.701484 | `kusto_cluster_list` | ✅ **EXPECTED** | -| 2 | 0.572169 | `kusto_cluster_get` | ❌ | -| 3 | 0.548796 | `kusto_database_list` | ❌ | -| 4 | 0.498909 | `aks_cluster_get` | ❌ | -| 5 | 0.474201 | `redis_list` | ❌ | +| 1 | 0.846240 | `postgres_server_param_set` | ✅ **EXPECTED** | +| 2 | 0.797091 | `postgres_server_config_get` | ❌ | +| 3 | 0.780646 | `postgres_database_list` | ❌ | +| 4 | 0.768556 | `postgres_server_list` | ❌ | +| 5 | 0.765124 | `postgres_table_list` | ❌ | --- ## Test 159 -**Expected Tool:** `kusto_database_list` -**Prompt:** List all databases in the Data Explorer cluster +**Expected Tool:** `postgres_table_list` +**Prompt:** List all tables in the PostgreSQL database in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.676910 | `kusto_database_list` | ✅ **EXPECTED** | -| 2 | 0.560592 | `kusto_cluster_list` | ❌ | -| 3 | 0.556795 | `kusto_table_list` | ❌ | -| 4 | 0.553218 | `postgres_database_list` | ❌ | -| 5 | 0.549673 | `cosmos_database_list` | ❌ | +| 1 | 0.935256 | `postgres_table_list` | ✅ **EXPECTED** | +| 2 | 0.909184 | `postgres_database_list` | ❌ | +| 3 | 0.844600 | `postgres_server_list` | ❌ | +| 4 | 0.836233 | `postgres_table_schema_get` | ❌ | +| 5 | 0.824605 | `kusto_table_list` | ❌ | --- ## Test 160 -**Expected Tool:** `kusto_database_list` -**Prompt:** Show me the databases in the Data Explorer cluster +**Expected Tool:** `postgres_table_list` +**Prompt:** Show me the tables in the PostgreSQL database in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.623353 | `kusto_database_list` | ✅ **EXPECTED** | -| 2 | 0.509952 | `kusto_cluster_list` | ❌ | -| 3 | 0.507073 | `kusto_table_list` | ❌ | -| 4 | 0.497144 | `cosmos_database_list` | ❌ | -| 5 | 0.491400 | `mysql_database_list` | ❌ | +| 1 | 0.915594 | `postgres_table_list` | ✅ **EXPECTED** | +| 2 | 0.889063 | `postgres_database_list` | ❌ | +| 3 | 0.842322 | `postgres_table_schema_get` | ❌ | +| 4 | 0.828271 | `postgres_server_list` | ❌ | +| 5 | 0.811715 | `postgres_server_config_get` | ❌ | --- ## Test 161 -**Expected Tool:** `kusto_query` -**Prompt:** Show me all items that contain the word in the Data Explorer table in cluster +**Expected Tool:** `postgres_table_schema_get` +**Prompt:** Show me the schema of table
in the PostgreSQL database in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.423805 | `kusto_query` | ✅ **EXPECTED** | -| 2 | 0.409254 | `postgres_database_query` | ❌ | -| 3 | 0.408273 | `kusto_table_schema` | ❌ | -| 4 | 0.407966 | `kusto_sample` | ❌ | -| 5 | 0.404296 | `kusto_cluster_list` | ❌ | +| 1 | 0.898503 | `postgres_table_schema_get` | ✅ **EXPECTED** | +| 2 | 0.882606 | `postgres_table_list` | ❌ | +| 3 | 0.861496 | `postgres_database_list` | ❌ | +| 4 | 0.828578 | `kusto_table_schema` | ❌ | +| 5 | 0.817162 | `postgres_server_config_get` | ❌ | --- ## Test 162 -**Expected Tool:** `kusto_sample` -**Prompt:** Show me a data sample from the Data Explorer table in cluster +**Expected Tool:** `deploy_app_logs_get` +**Prompt:** Show me the log of the application in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.595554 | `kusto_sample` | ✅ **EXPECTED** | -| 2 | 0.510233 | `kusto_table_schema` | ❌ | -| 3 | 0.424212 | `kusto_table_list` | ❌ | -| 4 | 0.401751 | `kusto_cluster_get` | ❌ | -| 5 | 0.400924 | `kusto_cluster_list` | ❌ | +| 1 | 0.854430 | `deploy_app_logs_get` | ✅ **EXPECTED** | +| 2 | 0.804046 | `monitor_activitylog_list` | ❌ | +| 3 | 0.801034 | `monitor_workspace_log_query` | ❌ | +| 4 | 0.799285 | `monitor_resource_log_query` | ❌ | +| 5 | 0.796333 | `applens_resource_diagnose` | ❌ | --- ## Test 163 -**Expected Tool:** `kusto_table_list` -**Prompt:** List all tables in the Data Explorer database in cluster +**Expected Tool:** `deploy_architecture_diagram_generate` +**Prompt:** Generate the azure architecture diagram for this application ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.679620 | `kusto_table_list` | ✅ **EXPECTED** | -| 2 | 0.585270 | `postgres_table_list` | ❌ | -| 3 | 0.581109 | `kusto_database_list` | ❌ | -| 4 | 0.556665 | `mysql_table_list` | ❌ | -| 5 | 0.549938 | `monitor_table_list` | ❌ | +| 1 | 0.913195 | `deploy_architecture_diagram_generate` | ✅ **EXPECTED** | +| 2 | 0.837844 | `deploy_plan_get` | ❌ | +| 3 | 0.829720 | `deploy_iac_rules_get` | ❌ | +| 4 | 0.823312 | `cloudarchitect_design` | ❌ | +| 5 | 0.819095 | `deploy_pipeline_guidance_get` | ❌ | --- ## Test 164 -**Expected Tool:** `kusto_table_list` -**Prompt:** Show me the tables in the Data Explorer database in cluster +**Expected Tool:** `deploy_iac_rules_get` +**Prompt:** Show me the rules to generate scripts ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.619252 | `kusto_table_list` | ✅ **EXPECTED** | -| 2 | 0.554332 | `kusto_table_schema` | ❌ | -| 3 | 0.527507 | `kusto_database_list` | ❌ | -| 4 | 0.524691 | `mysql_table_list` | ❌ | -| 5 | 0.523432 | `postgres_table_list` | ❌ | +| 1 | 0.786372 | `foundry_agents_get-sdk-sample` | ❌ | +| 2 | 0.766716 | `extension_cli_generate` | ❌ | +| 3 | 0.752556 | `deploy_iac_rules_get` | ✅ **EXPECTED** | +| 4 | 0.747679 | `azureaibestpractices_get` | ❌ | +| 5 | 0.741796 | `deploy_pipeline_guidance_get` | ❌ | --- ## Test 165 -**Expected Tool:** `kusto_table_schema` -**Prompt:** Show me the schema for table in the Data Explorer database in cluster +**Expected Tool:** `deploy_pipeline_guidance_get` +**Prompt:** How can I create a CI/CD pipeline to deploy this app to Azure? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.667052 | `kusto_table_schema` | ✅ **EXPECTED** | -| 2 | 0.564311 | `postgres_table_schema_get` | ❌ | -| 3 | 0.527917 | `mysql_table_schema_get` | ❌ | -| 4 | 0.490904 | `kusto_sample` | ❌ | -| 5 | 0.489680 | `kusto_table_list` | ❌ | +| 1 | 0.904455 | `deploy_pipeline_guidance_get` | ✅ **EXPECTED** | +| 2 | 0.847580 | `deploy_iac_rules_get` | ❌ | +| 3 | 0.836244 | `deploy_plan_get` | ❌ | +| 4 | 0.804504 | `cloudarchitect_design` | ❌ | +| 5 | 0.802719 | `deploy_architecture_diagram_generate` | ❌ | --- ## Test 166 -**Expected Tool:** `mysql_database_list` -**Prompt:** List all MySQL databases in server +**Expected Tool:** `deploy_plan_get` +**Prompt:** Create a plan to deploy this application to azure ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.634056 | `postgres_database_list` | ❌ | -| 2 | 0.623421 | `mysql_database_list` | ✅ **EXPECTED** | -| 3 | 0.534457 | `mysql_table_list` | ❌ | -| 4 | 0.498918 | `mysql_server_list` | ❌ | -| 5 | 0.490148 | `sql_db_list` | ❌ | +| 1 | 0.888872 | `deploy_plan_get` | ✅ **EXPECTED** | +| 2 | 0.866662 | `deploy_pipeline_guidance_get` | ❌ | +| 3 | 0.850645 | `deploy_iac_rules_get` | ❌ | +| 4 | 0.844504 | `deploy_architecture_diagram_generate` | ❌ | +| 5 | 0.820138 | `foundry_models_deploy` | ❌ | --- ## Test 167 -**Expected Tool:** `mysql_database_list` -**Prompt:** Show me the MySQL databases in server +**Expected Tool:** `eventgrid_events_publish` +**Prompt:** Publish an event to Event Grid topic using with the following data ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.588121 | `mysql_database_list` | ✅ **EXPECTED** | -| 2 | 0.574089 | `postgres_database_list` | ❌ | -| 3 | 0.483855 | `mysql_table_list` | ❌ | -| 4 | 0.463244 | `mysql_server_list` | ❌ | -| 5 | 0.444547 | `sql_db_list` | ❌ | +| 1 | 0.889654 | `eventgrid_events_publish` | ✅ **EXPECTED** | +| 2 | 0.798699 | `eventgrid_topic_list` | ❌ | +| 3 | 0.782893 | `eventgrid_subscription_list` | ❌ | +| 4 | 0.749106 | `eventhubs_eventhub_update` | ❌ | +| 5 | 0.736760 | `kusto_table_schema` | ❌ | --- ## Test 168 -**Expected Tool:** `mysql_database_query` -**Prompt:** Show me all items that contain the word in the MySQL database in server +**Expected Tool:** `eventgrid_events_publish` +**Prompt:** Publish event to my Event Grid topic with the following events ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.476423 | `mysql_table_list` | ❌ | -| 2 | 0.455770 | `mysql_database_list` | ❌ | -| 3 | 0.433392 | `mysql_database_query` | ✅ **EXPECTED** | -| 4 | 0.419859 | `mysql_server_list` | ❌ | -| 5 | 0.409445 | `mysql_table_schema_get` | ❌ | +| 1 | 0.859537 | `eventgrid_events_publish` | ✅ **EXPECTED** | +| 2 | 0.820994 | `eventgrid_topic_list` | ❌ | +| 3 | 0.804214 | `eventgrid_subscription_list` | ❌ | +| 4 | 0.757091 | `eventhubs_eventhub_update` | ❌ | +| 5 | 0.736109 | `eventhubs_eventhub_get` | ❌ | --- ## Test 169 -**Expected Tool:** `mysql_server_config_get` -**Prompt:** Show me the configuration of MySQL server +**Expected Tool:** `eventgrid_events_publish` +**Prompt:** Send an event to Event Grid topic in resource group with ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.531887 | `postgres_server_config_get` | ❌ | -| 2 | 0.516893 | `mysql_server_param_set` | ❌ | -| 3 | 0.489816 | `mysql_server_config_get` | ✅ **EXPECTED** | -| 4 | 0.476863 | `mysql_server_param_get` | ❌ | -| 5 | 0.426507 | `mysql_table_schema_get` | ❌ | +| 1 | 0.829645 | `eventgrid_topic_list` | ❌ | +| 2 | 0.823490 | `eventgrid_events_publish` | ✅ **EXPECTED** | +| 3 | 0.800179 | `eventgrid_subscription_list` | ❌ | +| 4 | 0.761489 | `eventhubs_eventhub_update` | ❌ | +| 5 | 0.761001 | `eventhubs_eventhub_consumergroup_update` | ❌ | --- ## Test 170 -**Expected Tool:** `mysql_server_list` -**Prompt:** List all MySQL servers in my subscription +**Expected Tool:** `eventgrid_topic_list` +**Prompt:** List all Event Grid topics in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.678473 | `postgres_server_list` | ❌ | -| 2 | 0.558177 | `mysql_database_list` | ❌ | -| 3 | 0.554818 | `mysql_server_list` | ✅ **EXPECTED** | -| 4 | 0.513706 | `kusto_cluster_list` | ❌ | -| 5 | 0.501199 | `mysql_table_list` | ❌ | +| 1 | 0.905660 | `eventgrid_topic_list` | ✅ **EXPECTED** | +| 2 | 0.892916 | `eventgrid_subscription_list` | ❌ | +| 3 | 0.804302 | `kusto_cluster_list` | ❌ | +| 4 | 0.791512 | `redis_list` | ❌ | +| 5 | 0.790915 | `search_service_list` | ❌ | --- ## Test 171 -**Expected Tool:** `mysql_server_list` -**Prompt:** Show me my MySQL servers +**Expected Tool:** `eventgrid_topic_list` +**Prompt:** Show me the Event Grid topics in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.478518 | `mysql_database_list` | ❌ | -| 2 | 0.474586 | `mysql_server_list` | ✅ **EXPECTED** | -| 3 | 0.435642 | `postgres_server_list` | ❌ | -| 4 | 0.412380 | `mysql_table_list` | ❌ | -| 5 | 0.389993 | `postgres_database_list` | ❌ | +| 1 | 0.898506 | `eventgrid_topic_list` | ✅ **EXPECTED** | +| 2 | 0.892006 | `eventgrid_subscription_list` | ❌ | +| 3 | 0.793690 | `redis_list` | ❌ | +| 4 | 0.788307 | `kusto_cluster_list` | ❌ | +| 5 | 0.785561 | `marketplace_product_list` | ❌ | --- ## Test 172 -**Expected Tool:** `mysql_server_list` -**Prompt:** Show me the MySQL servers in my subscription +**Expected Tool:** `eventgrid_topic_list` +**Prompt:** List all Event Grid topics in subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.636435 | `postgres_server_list` | ❌ | -| 2 | 0.534266 | `mysql_server_list` | ✅ **EXPECTED** | -| 3 | 0.530210 | `mysql_database_list` | ❌ | -| 4 | 0.475052 | `kusto_cluster_list` | ❌ | -| 5 | 0.470468 | `redis_list` | ❌ | +| 1 | 0.913184 | `eventgrid_topic_list` | ✅ **EXPECTED** | +| 2 | 0.894709 | `eventgrid_subscription_list` | ❌ | +| 3 | 0.814853 | `postgres_server_list` | ❌ | +| 4 | 0.810667 | `kusto_cluster_list` | ❌ | +| 5 | 0.802949 | `redis_list` | ❌ | --- ## Test 173 -**Expected Tool:** `mysql_server_param_get` -**Prompt:** Show me the value of connection timeout in seconds in my MySQL server +**Expected Tool:** `eventgrid_topic_list` +**Prompt:** List all Event Grid topics in resource group in subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.495071 | `mysql_server_param_get` | ✅ **EXPECTED** | -| 2 | 0.438075 | `mysql_server_param_set` | ❌ | -| 3 | 0.333841 | `mysql_database_query` | ❌ | -| 4 | 0.313150 | `mysql_table_schema_get` | ❌ | -| 5 | 0.310834 | `postgres_server_param_get` | ❌ | +| 1 | 0.920089 | `eventgrid_topic_list` | ✅ **EXPECTED** | +| 2 | 0.882339 | `eventgrid_subscription_list` | ❌ | +| 3 | 0.837201 | `group_list` | ❌ | +| 4 | 0.820569 | `kusto_cluster_list` | ❌ | +| 5 | 0.816757 | `loadtesting_testresource_list` | ❌ | --- ## Test 174 -**Expected Tool:** `mysql_server_param_set` -**Prompt:** Set connection timeout to 20 seconds for my MySQL server +**Expected Tool:** `eventgrid_subscription_list` +**Prompt:** Show me all Event Grid subscriptions for topic ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.449419 | `mysql_server_param_set` | ✅ **EXPECTED** | -| 2 | 0.381144 | `mysql_server_param_get` | ❌ | -| 3 | 0.303499 | `postgres_server_param_set` | ❌ | -| 4 | 0.298911 | `mysql_database_query` | ❌ | -| 5 | 0.254180 | `mysql_server_list` | ❌ | +| 1 | 0.888829 | `eventgrid_subscription_list` | ✅ **EXPECTED** | +| 2 | 0.883944 | `eventgrid_topic_list` | ❌ | +| 3 | 0.800907 | `kusto_cluster_list` | ❌ | +| 4 | 0.792277 | `postgres_server_list` | ❌ | +| 5 | 0.790827 | `redis_list` | ❌ | --- ## Test 175 -**Expected Tool:** `mysql_table_list` -**Prompt:** List all tables in the MySQL database in server +**Expected Tool:** `eventgrid_subscription_list` +**Prompt:** List Event Grid subscriptions for topic in subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.633448 | `mysql_table_list` | ✅ **EXPECTED** | -| 2 | 0.573844 | `postgres_table_list` | ❌ | -| 3 | 0.550898 | `postgres_database_list` | ❌ | -| 4 | 0.546963 | `mysql_database_list` | ❌ | -| 5 | 0.511847 | `kusto_table_list` | ❌ | +| 1 | 0.864350 | `eventgrid_topic_list` | ❌ | +| 2 | 0.860571 | `eventgrid_subscription_list` | ✅ **EXPECTED** | +| 3 | 0.786464 | `postgres_server_list` | ❌ | +| 4 | 0.776853 | `kusto_cluster_list` | ❌ | +| 5 | 0.767170 | `grafana_list` | ❌ | --- ## Test 176 -**Expected Tool:** `mysql_table_list` -**Prompt:** Show me the tables in the MySQL database in server +**Expected Tool:** `eventgrid_subscription_list` +**Prompt:** List Event Grid subscriptions for topic in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.609131 | `mysql_table_list` | ✅ **EXPECTED** | -| 2 | 0.526236 | `postgres_table_list` | ❌ | -| 3 | 0.525709 | `mysql_database_list` | ❌ | -| 4 | 0.507258 | `mysql_table_schema_get` | ❌ | -| 5 | 0.498050 | `postgres_database_list` | ❌ | +| 1 | 0.886646 | `eventgrid_topic_list` | ❌ | +| 2 | 0.870517 | `eventgrid_subscription_list` | ✅ **EXPECTED** | +| 3 | 0.804615 | `group_list` | ❌ | +| 4 | 0.800191 | `kusto_cluster_list` | ❌ | +| 5 | 0.792609 | `monitor_webtests_list` | ❌ | --- ## Test 177 -**Expected Tool:** `mysql_table_schema_get` -**Prompt:** Show me the schema of table
in the MySQL database in server +**Expected Tool:** `eventgrid_subscription_list` +**Prompt:** Show all Event Grid subscriptions in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.630623 | `mysql_table_schema_get` | ✅ **EXPECTED** | -| 2 | 0.558306 | `postgres_table_schema_get` | ❌ | -| 3 | 0.545025 | `mysql_table_list` | ❌ | -| 4 | 0.517419 | `kusto_table_schema` | ❌ | -| 5 | 0.457739 | `mysql_database_list` | ❌ | +| 1 | 0.895381 | `eventgrid_subscription_list` | ✅ **EXPECTED** | +| 2 | 0.877596 | `eventgrid_topic_list` | ❌ | +| 3 | 0.811393 | `kusto_cluster_list` | ❌ | +| 4 | 0.806815 | `redis_list` | ❌ | +| 5 | 0.803717 | `subscription_list` | ❌ | --- ## Test 178 -**Expected Tool:** `postgres_database_list` -**Prompt:** List all PostgreSQL databases in server +**Expected Tool:** `eventgrid_subscription_list` +**Prompt:** List all Event Grid subscriptions in subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.815617 | `postgres_database_list` | ✅ **EXPECTED** | -| 2 | 0.644014 | `postgres_table_list` | ❌ | -| 3 | 0.622790 | `postgres_server_list` | ❌ | -| 4 | 0.542685 | `postgres_server_config_get` | ❌ | -| 5 | 0.490904 | `postgres_server_param_get` | ❌ | +| 1 | 0.888654 | `eventgrid_subscription_list` | ✅ **EXPECTED** | +| 2 | 0.884649 | `eventgrid_topic_list` | ❌ | +| 3 | 0.824591 | `postgres_server_list` | ❌ | +| 4 | 0.822312 | `kusto_cluster_list` | ❌ | +| 5 | 0.813453 | `group_list` | ❌ | --- ## Test 179 -**Expected Tool:** `postgres_database_list` -**Prompt:** Show me the PostgreSQL databases in server +**Expected Tool:** `eventgrid_subscription_list` +**Prompt:** Show Event Grid subscriptions in resource group in subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.760033 | `postgres_database_list` | ✅ **EXPECTED** | -| 2 | 0.589784 | `postgres_server_list` | ❌ | -| 3 | 0.585891 | `postgres_table_list` | ❌ | -| 4 | 0.552660 | `postgres_server_config_get` | ❌ | -| 5 | 0.495629 | `postgres_server_param_get` | ❌ | +| 1 | 0.891397 | `eventgrid_topic_list` | ❌ | +| 2 | 0.884803 | `eventgrid_subscription_list` | ✅ **EXPECTED** | +| 3 | 0.841621 | `group_list` | ❌ | +| 4 | 0.820360 | `redis_list` | ❌ | +| 5 | 0.815762 | `kusto_cluster_list` | ❌ | --- ## Test 180 -**Expected Tool:** `postgres_database_query` -**Prompt:** Show me all items that contain the word in the PostgreSQL database in server +**Expected Tool:** `eventgrid_subscription_list` +**Prompt:** List Event Grid subscriptions for subscription in location ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.546211 | `postgres_database_list` | ❌ | -| 2 | 0.523015 | `postgres_database_query` | ✅ **EXPECTED** | -| 3 | 0.503267 | `postgres_table_list` | ❌ | -| 4 | 0.466599 | `postgres_server_list` | ❌ | -| 5 | 0.403969 | `postgres_server_param_get` | ❌ | +| 1 | 0.856701 | `eventgrid_subscription_list` | ✅ **EXPECTED** | +| 2 | 0.849284 | `eventgrid_topic_list` | ❌ | +| 3 | 0.789738 | `postgres_server_list` | ❌ | +| 4 | 0.776123 | `kusto_cluster_list` | ❌ | +| 5 | 0.773041 | `appconfig_account_list` | ❌ | --- ## Test 181 -**Expected Tool:** `postgres_server_config_get` -**Prompt:** Show me the configuration of PostgreSQL server +**Expected Tool:** `eventhubs_eventhub_consumergroup_delete` +**Prompt:** Delete my consumer group in my event hub , namespace , and resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.756593 | `postgres_server_config_get` | ✅ **EXPECTED** | -| 2 | 0.615429 | `postgres_server_param_set` | ❌ | -| 3 | 0.599471 | `postgres_server_param_get` | ❌ | -| 4 | 0.535050 | `postgres_database_list` | ❌ | -| 5 | 0.518574 | `postgres_server_list` | ❌ | +| 1 | 0.903198 | `eventhubs_eventhub_consumergroup_delete` | ✅ **EXPECTED** | +| 2 | 0.859003 | `eventhubs_eventhub_consumergroup_update` | ❌ | +| 3 | 0.857360 | `eventhubs_eventhub_consumergroup_get` | ❌ | +| 4 | 0.849651 | `eventhubs_namespace_delete` | ❌ | +| 5 | 0.822938 | `eventhubs_eventhub_delete` | ❌ | --- ## Test 182 -**Expected Tool:** `postgres_server_list` -**Prompt:** List all PostgreSQL servers in my subscription +**Expected Tool:** `eventhubs_eventhub_consumergroup_get` +**Prompt:** List all consumer groups in my event hub in namespace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.900023 | `postgres_server_list` | ✅ **EXPECTED** | -| 2 | 0.640733 | `postgres_database_list` | ❌ | -| 3 | 0.565914 | `postgres_table_list` | ❌ | -| 4 | 0.538997 | `postgres_server_config_get` | ❌ | -| 5 | 0.534239 | `kusto_cluster_list` | ❌ | +| 1 | 0.875460 | `eventhubs_eventhub_consumergroup_get` | ✅ **EXPECTED** | +| 2 | 0.837769 | `eventhubs_eventhub_get` | ❌ | +| 3 | 0.833309 | `eventhubs_namespace_get` | ❌ | +| 4 | 0.823753 | `eventhubs_eventhub_consumergroup_delete` | ❌ | +| 5 | 0.823385 | `eventhubs_eventhub_consumergroup_update` | ❌ | --- ## Test 183 -**Expected Tool:** `postgres_server_list` -**Prompt:** Show me my PostgreSQL servers +**Expected Tool:** `eventhubs_eventhub_consumergroup_get` +**Prompt:** Get the details of my consumer group in my event hub , namespace , and resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.674327 | `postgres_server_list` | ✅ **EXPECTED** | -| 2 | 0.607062 | `postgres_database_list` | ❌ | -| 3 | 0.576348 | `postgres_server_config_get` | ❌ | -| 4 | 0.522995 | `postgres_table_list` | ❌ | -| 5 | 0.506171 | `postgres_server_param_get` | ❌ | +| 1 | 0.886891 | `eventhubs_eventhub_consumergroup_get` | ✅ **EXPECTED** | +| 2 | 0.848877 | `eventhubs_eventhub_consumergroup_update` | ❌ | +| 3 | 0.839500 | `eventhubs_eventhub_consumergroup_delete` | ❌ | +| 4 | 0.835102 | `eventhubs_namespace_get` | ❌ | +| 5 | 0.825449 | `eventhubs_eventhub_get` | ❌ | --- ## Test 184 -**Expected Tool:** `postgres_server_list` -**Prompt:** Show me the PostgreSQL servers in my subscription +**Expected Tool:** `eventhubs_eventhub_consumergroup_update` +**Prompt:** Create a new consumer group in my event hub , namespace , and resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.832155 | `postgres_server_list` | ✅ **EXPECTED** | -| 2 | 0.579232 | `postgres_database_list` | ❌ | -| 3 | 0.531804 | `postgres_server_config_get` | ❌ | -| 4 | 0.514445 | `postgres_table_list` | ❌ | -| 5 | 0.505869 | `postgres_server_param_get` | ❌ | +| 1 | 0.884826 | `eventhubs_eventhub_consumergroup_update` | ✅ **EXPECTED** | +| 2 | 0.869484 | `eventhubs_eventhub_consumergroup_get` | ❌ | +| 3 | 0.858082 | `eventhubs_eventhub_consumergroup_delete` | ❌ | +| 4 | 0.822265 | `eventhubs_namespace_delete` | ❌ | +| 5 | 0.821795 | `eventhubs_eventhub_update` | ❌ | --- ## Test 185 -**Expected Tool:** `postgres_server_param_get` -**Prompt:** Show me if the parameter my PostgreSQL server has replication enabled +**Expected Tool:** `eventhubs_eventhub_consumergroup_update` +**Prompt:** Update my consumer group in my event hub , namespace , and resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.594753 | `postgres_server_param_get` | ✅ **EXPECTED** | -| 2 | 0.552678 | `postgres_server_param_set` | ❌ | -| 3 | 0.539671 | `postgres_server_config_get` | ❌ | -| 4 | 0.489693 | `postgres_server_list` | ❌ | -| 5 | 0.451871 | `postgres_database_list` | ❌ | +| 1 | 0.881362 | `eventhubs_eventhub_consumergroup_update` | ✅ **EXPECTED** | +| 2 | 0.862158 | `eventhubs_eventhub_consumergroup_get` | ❌ | +| 3 | 0.853036 | `eventhubs_eventhub_consumergroup_delete` | ❌ | +| 4 | 0.828428 | `eventhubs_eventhub_update` | ❌ | +| 5 | 0.810440 | `eventhubs_namespace_get` | ❌ | --- ## Test 186 -**Expected Tool:** `postgres_server_param_set` -**Prompt:** Enable replication for my PostgreSQL server +**Expected Tool:** `eventhubs_eventhub_delete` +**Prompt:** Delete my event hub in my namespace and resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.579873 | `postgres_server_param_set` | ✅ **EXPECTED** | -| 2 | 0.488474 | `postgres_server_config_get` | ❌ | -| 3 | 0.469794 | `postgres_server_list` | ❌ | -| 4 | 0.447011 | `postgres_server_param_get` | ❌ | -| 5 | 0.440760 | `postgres_database_list` | ❌ | +| 1 | 0.873644 | `eventhubs_namespace_delete` | ❌ | +| 2 | 0.854152 | `eventhubs_eventhub_delete` | ✅ **EXPECTED** | +| 3 | 0.840655 | `eventhubs_eventhub_consumergroup_delete` | ❌ | +| 4 | 0.824830 | `eventhubs_namespace_get` | ❌ | +| 5 | 0.821433 | `eventhubs_eventhub_get` | ❌ | --- ## Test 187 -**Expected Tool:** `postgres_table_list` -**Prompt:** List all tables in the PostgreSQL database in server +**Expected Tool:** `eventhubs_eventhub_get` +**Prompt:** List all Event Hubs in my namespace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.789883 | `postgres_table_list` | ✅ **EXPECTED** | -| 2 | 0.750580 | `postgres_database_list` | ❌ | -| 3 | 0.574931 | `postgres_server_list` | ❌ | -| 4 | 0.519820 | `postgres_table_schema_get` | ❌ | -| 5 | 0.501400 | `postgres_server_config_get` | ❌ | +| 1 | 0.884791 | `eventhubs_eventhub_get` | ✅ **EXPECTED** | +| 2 | 0.850468 | `eventhubs_namespace_get` | ❌ | +| 3 | 0.820604 | `eventhubs_eventhub_update` | ❌ | +| 4 | 0.816794 | `eventhubs_namespace_delete` | ❌ | +| 5 | 0.815068 | `kusto_cluster_list` | ❌ | --- ## Test 188 -**Expected Tool:** `postgres_table_list` -**Prompt:** Show me the tables in the PostgreSQL database in server +**Expected Tool:** `eventhubs_eventhub_get` +**Prompt:** Get the details of my event hub in my namespace and resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.736083 | `postgres_table_list` | ✅ **EXPECTED** | -| 2 | 0.690112 | `postgres_database_list` | ❌ | -| 3 | 0.558357 | `postgres_table_schema_get` | ❌ | -| 4 | 0.543331 | `postgres_server_list` | ❌ | -| 5 | 0.521570 | `postgres_server_config_get` | ❌ | +| 1 | 0.872481 | `eventhubs_eventhub_get` | ✅ **EXPECTED** | +| 2 | 0.863325 | `eventhubs_namespace_get` | ❌ | +| 3 | 0.828888 | `eventhubs_eventhub_consumergroup_get` | ❌ | +| 4 | 0.825528 | `eventgrid_topic_list` | ❌ | +| 5 | 0.823248 | `eventhubs_eventhub_update` | ❌ | --- ## Test 189 -**Expected Tool:** `postgres_table_schema_get` -**Prompt:** Show me the schema of table
in the PostgreSQL database in server +**Expected Tool:** `eventhubs_eventhub_update` +**Prompt:** Create a new event hub in my namespace and resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.714877 | `postgres_table_schema_get` | ✅ **EXPECTED** | -| 2 | 0.597846 | `postgres_table_list` | ❌ | -| 3 | 0.574230 | `postgres_database_list` | ❌ | -| 4 | 0.508082 | `postgres_server_config_get` | ❌ | -| 5 | 0.502626 | `kusto_table_schema` | ❌ | +| 1 | 0.858259 | `eventhubs_eventhub_update` | ✅ **EXPECTED** | +| 2 | 0.848236 | `eventhubs_namespace_delete` | ❌ | +| 3 | 0.847070 | `eventhubs_namespace_get` | ❌ | +| 4 | 0.844468 | `eventhubs_eventhub_get` | ❌ | +| 5 | 0.819099 | `eventhubs_eventhub_consumergroup_update` | ❌ | --- ## Test 190 -**Expected Tool:** `deploy_app_logs_get` -**Prompt:** Show me the log of the application deployed by azd +**Expected Tool:** `eventhubs_eventhub_update` +**Prompt:** Update my event hub in my namespace and resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.711770 | `deploy_app_logs_get` | ✅ **EXPECTED** | -| 2 | 0.471692 | `deploy_plan_get` | ❌ | -| 3 | 0.451639 | `monitor_activitylog_list` | ❌ | -| 4 | 0.404891 | `deploy_pipeline_guidance_get` | ❌ | -| 5 | 0.401388 | `monitor_resource_log_query` | ❌ | +| 1 | 0.861448 | `eventhubs_eventhub_update` | ✅ **EXPECTED** | +| 2 | 0.837861 | `eventhubs_namespace_get` | ❌ | +| 3 | 0.831253 | `eventhubs_namespace_delete` | ❌ | +| 4 | 0.828681 | `eventhubs_eventhub_get` | ❌ | +| 5 | 0.821437 | `eventhubs_eventhub_consumergroup_update` | ❌ | --- ## Test 191 -**Expected Tool:** `deploy_architecture_diagram_generate` -**Prompt:** Generate the azure architecture diagram for this application +**Expected Tool:** `eventhubs_namespace_delete` +**Prompt:** Delete my namespace in my resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.680640 | `deploy_architecture_diagram_generate` | ✅ **EXPECTED** | -| 2 | 0.562521 | `deploy_plan_get` | ❌ | -| 3 | 0.497193 | `deploy_pipeline_guidance_get` | ❌ | -| 4 | 0.489344 | `cloudarchitect_design` | ❌ | -| 5 | 0.435921 | `deploy_iac_rules_get` | ❌ | +| 1 | 0.847128 | `eventhubs_namespace_delete` | ✅ **EXPECTED** | +| 2 | 0.809806 | `eventhubs_namespace_update` | ❌ | +| 3 | 0.796933 | `eventhubs_eventhub_consumergroup_delete` | ❌ | +| 4 | 0.786281 | `eventhubs_namespace_get` | ❌ | +| 5 | 0.774290 | `sql_server_delete` | ❌ | --- ## Test 192 -**Expected Tool:** `deploy_iac_rules_get` -**Prompt:** Show me the rules to generate bicep scripts +**Expected Tool:** `eventhubs_namespace_get` +**Prompt:** List all Event Hubs namespaces in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.529092 | `deploy_iac_rules_get` | ✅ **EXPECTED** | -| 2 | 0.479903 | `bicepschema_get` | ❌ | -| 3 | 0.391965 | `get_bestpractices_get` | ❌ | -| 4 | 0.383210 | `azureterraformbestpractices_get` | ❌ | -| 5 | 0.375561 | `extension_cli_generate` | ❌ | +| 1 | 0.872513 | `eventhubs_namespace_get` | ✅ **EXPECTED** | +| 2 | 0.864321 | `eventhubs_eventhub_get` | ❌ | +| 3 | 0.848809 | `kusto_cluster_list` | ❌ | +| 4 | 0.826985 | `eventgrid_topic_list` | ❌ | +| 5 | 0.818780 | `eventgrid_subscription_list` | ❌ | --- ## Test 193 -**Expected Tool:** `deploy_pipeline_guidance_get` -**Prompt:** How can I create a CI/CD pipeline to deploy this app to Azure? +**Expected Tool:** `eventhubs_namespace_get` +**Prompt:** Get the details of my namespace in my resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.638841 | `deploy_pipeline_guidance_get` | ✅ **EXPECTED** | -| 2 | 0.499242 | `deploy_plan_get` | ❌ | -| 3 | 0.448917 | `deploy_iac_rules_get` | ❌ | -| 4 | 0.385920 | `deploy_app_logs_get` | ❌ | -| 5 | 0.382240 | `get_bestpractices_get` | ❌ | +| 1 | 0.816780 | `eventhubs_namespace_get` | ✅ **EXPECTED** | +| 2 | 0.808318 | `eventhubs_namespace_update` | ❌ | +| 3 | 0.798274 | `monitor_webtests_get` | ❌ | +| 4 | 0.797139 | `storage_account_get` | ❌ | +| 5 | 0.794771 | `functionapp_get` | ❌ | --- ## Test 194 -**Expected Tool:** `deploy_plan_get` -**Prompt:** Create a plan to deploy this application to azure +**Expected Tool:** `eventhubs_namespace_update` +**Prompt:** Create an new namespace in my resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.688055 | `deploy_plan_get` | ✅ **EXPECTED** | -| 2 | 0.587903 | `deploy_pipeline_guidance_get` | ❌ | -| 3 | 0.499385 | `deploy_iac_rules_get` | ❌ | -| 4 | 0.498575 | `deploy_architecture_diagram_generate` | ❌ | -| 5 | 0.448692 | `loadtesting_test_create` | ❌ | +| 1 | 0.842150 | `eventhubs_namespace_update` | ✅ **EXPECTED** | +| 2 | 0.812118 | `eventhubs_namespace_delete` | ❌ | +| 3 | 0.802209 | `eventhubs_namespace_get` | ❌ | +| 4 | 0.789301 | `redis_create` | ❌ | +| 5 | 0.762777 | `foundry_models_deploy` | ❌ | --- ## Test 195 -**Expected Tool:** `eventgrid_events_publish` -**Prompt:** Publish an event to Event Grid topic using with the following data +**Expected Tool:** `eventhubs_namespace_update` +**Prompt:** Update my namespace in my resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.755365 | `eventgrid_events_publish` | ✅ **EXPECTED** | -| 2 | 0.483021 | `eventgrid_subscription_list` | ❌ | -| 3 | 0.466031 | `eventgrid_topic_list` | ❌ | -| 4 | 0.360676 | `eventhubs_eventhub_update` | ❌ | -| 5 | 0.355598 | `servicebus_topic_details` | ❌ | +| 1 | 0.861629 | `eventhubs_namespace_update` | ✅ **EXPECTED** | +| 2 | 0.804084 | `eventhubs_namespace_delete` | ❌ | +| 3 | 0.796062 | `eventhubs_namespace_get` | ❌ | +| 4 | 0.768136 | `eventhubs_eventhub_update` | ❌ | +| 5 | 0.765585 | `eventhubs_eventhub_consumergroup_update` | ❌ | --- ## Test 196 -**Expected Tool:** `eventgrid_events_publish` -**Prompt:** Publish event to my Event Grid topic with the following events +**Expected Tool:** `functionapp_get` +**Prompt:** Describe the function app in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.654648 | `eventgrid_events_publish` | ✅ **EXPECTED** | -| 2 | 0.524503 | `eventgrid_subscription_list` | ❌ | -| 3 | 0.510038 | `eventgrid_topic_list` | ❌ | -| 4 | 0.373718 | `servicebus_topic_details` | ❌ | -| 5 | 0.359908 | `eventhubs_eventhub_update` | ❌ | +| 1 | 0.864639 | `functionapp_get` | ✅ **EXPECTED** | +| 2 | 0.802003 | `deploy_app_logs_get` | ❌ | +| 3 | 0.779803 | `applens_resource_diagnose` | ❌ | +| 4 | 0.761400 | `deploy_architecture_diagram_generate` | ❌ | +| 5 | 0.759013 | `appservice_database_add` | ❌ | --- ## Test 197 -**Expected Tool:** `eventgrid_events_publish` -**Prompt:** Send an event to Event Grid topic in resource group with +**Expected Tool:** `functionapp_get` +**Prompt:** Get configuration for function app ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.600274 | `eventgrid_events_publish` | ✅ **EXPECTED** | -| 2 | 0.521240 | `eventgrid_topic_list` | ❌ | -| 3 | 0.504808 | `eventgrid_subscription_list` | ❌ | -| 4 | 0.411129 | `eventhubs_eventhub_consumergroup_update` | ❌ | -| 5 | 0.389439 | `eventhubs_eventhub_consumergroup_get` | ❌ | +| 1 | 0.823429 | `functionapp_get` | ✅ **EXPECTED** | +| 2 | 0.772407 | `deploy_app_logs_get` | ❌ | +| 3 | 0.763699 | `postgres_server_config_get` | ❌ | +| 4 | 0.761617 | `appconfig_kv_get` | ❌ | +| 5 | 0.747188 | `appconfig_account_list` | ❌ | --- ## Test 198 -**Expected Tool:** `eventgrid_topic_list` -**Prompt:** List all Event Grid topics in my subscription +**Expected Tool:** `functionapp_get` +**Prompt:** Get function app status for ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.770140 | `eventgrid_topic_list` | ✅ **EXPECTED** | -| 2 | 0.745471 | `eventgrid_subscription_list` | ❌ | -| 3 | 0.561862 | `kusto_cluster_list` | ❌ | -| 4 | 0.545540 | `search_service_list` | ❌ | -| 5 | 0.526225 | `subscription_list` | ❌ | +| 1 | 0.812102 | `functionapp_get` | ✅ **EXPECTED** | +| 2 | 0.760593 | `deploy_app_logs_get` | ❌ | +| 3 | 0.726109 | `applens_resource_diagnose` | ❌ | +| 4 | 0.712137 | `appconfig_kv_get` | ❌ | +| 5 | 0.711637 | `resourcehealth_availability-status_get` | ❌ | --- ## Test 199 -**Expected Tool:** `eventgrid_topic_list` -**Prompt:** Show me the Event Grid topics in my subscription +**Expected Tool:** `functionapp_get` +**Prompt:** Get information about my function app in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.738258 | `eventgrid_topic_list` | ✅ **EXPECTED** | -| 2 | 0.737486 | `eventgrid_subscription_list` | ❌ | -| 3 | 0.492592 | `kusto_cluster_list` | ❌ | -| 4 | 0.480361 | `subscription_list` | ❌ | -| 5 | 0.475119 | `search_service_list` | ❌ | +| 1 | 0.888340 | `functionapp_get` | ✅ **EXPECTED** | +| 2 | 0.801509 | `deploy_app_logs_get` | ❌ | +| 3 | 0.796455 | `applens_resource_diagnose` | ❌ | +| 4 | 0.775717 | `foundry_resource_get` | ❌ | +| 5 | 0.766532 | `monitor_webtests_get` | ❌ | --- ## Test 200 -**Expected Tool:** `eventgrid_topic_list` -**Prompt:** List all Event Grid topics in subscription +**Expected Tool:** `functionapp_get` +**Prompt:** Retrieve host name and status of function app ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.770140 | `eventgrid_topic_list` | ✅ **EXPECTED** | -| 2 | 0.721362 | `eventgrid_subscription_list` | ❌ | -| 3 | 0.535326 | `kusto_cluster_list` | ❌ | -| 4 | 0.514248 | `search_service_list` | ❌ | -| 5 | 0.496087 | `subscription_list` | ❌ | +| 1 | 0.800410 | `functionapp_get` | ✅ **EXPECTED** | +| 2 | 0.782737 | `deploy_app_logs_get` | ❌ | +| 3 | 0.746350 | `applens_resource_diagnose` | ❌ | +| 4 | 0.735675 | `sql_server_show` | ❌ | +| 5 | 0.728265 | `virtualdesktop_hostpool_host_user-list` | ❌ | --- ## Test 201 -**Expected Tool:** `eventgrid_topic_list` -**Prompt:** List all Event Grid topics in resource group in subscription +**Expected Tool:** `functionapp_get` +**Prompt:** Show function app details for in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.758816 | `eventgrid_topic_list` | ✅ **EXPECTED** | -| 2 | 0.704462 | `eventgrid_subscription_list` | ❌ | -| 3 | 0.609175 | `group_list` | ❌ | -| 4 | 0.544809 | `monitor_webtests_list` | ❌ | -| 5 | 0.524209 | `eventhubs_namespace_get` | ❌ | +| 1 | 0.884793 | `functionapp_get` | ✅ **EXPECTED** | +| 2 | 0.791441 | `deploy_app_logs_get` | ❌ | +| 3 | 0.767977 | `applens_resource_diagnose` | ❌ | +| 4 | 0.765038 | `monitor_webtests_get` | ❌ | +| 5 | 0.757883 | `foundry_resource_get` | ❌ | --- ## Test 202 -**Expected Tool:** `eventgrid_subscription_list` -**Prompt:** Show me all Event Grid subscriptions for topic +**Expected Tool:** `functionapp_get` +**Prompt:** Show me the details for the function app ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.769097 | `eventgrid_subscription_list` | ✅ **EXPECTED** | -| 2 | 0.720606 | `eventgrid_topic_list` | ❌ | -| 3 | 0.498615 | `servicebus_topic_details` | ❌ | -| 4 | 0.486216 | `servicebus_topic_subscription_details` | ❌ | -| 5 | 0.486162 | `eventgrid_events_publish` | ❌ | +| 1 | 0.854621 | `functionapp_get` | ✅ **EXPECTED** | +| 2 | 0.806043 | `deploy_app_logs_get` | ❌ | +| 3 | 0.763992 | `applens_resource_diagnose` | ❌ | +| 4 | 0.751034 | `sql_server_show` | ❌ | +| 5 | 0.750748 | `mysql_table_schema_get` | ❌ | --- ## Test 203 -**Expected Tool:** `eventgrid_subscription_list` -**Prompt:** List Event Grid subscriptions for topic in subscription +**Expected Tool:** `functionapp_get` +**Prompt:** Show plan and region for function app ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.718109 | `eventgrid_subscription_list` | ✅ **EXPECTED** | -| 2 | 0.709806 | `eventgrid_topic_list` | ❌ | -| 3 | 0.539977 | `servicebus_topic_subscription_details` | ❌ | -| 4 | 0.529286 | `servicebus_topic_details` | ❌ | -| 5 | 0.477876 | `eventgrid_events_publish` | ❌ | +| 1 | 0.809572 | `functionapp_get` | ✅ **EXPECTED** | +| 2 | 0.793238 | `deploy_app_logs_get` | ❌ | +| 3 | 0.767555 | `quota_usage_check` | ❌ | +| 4 | 0.763218 | `deploy_plan_get` | ❌ | +| 5 | 0.762313 | `deploy_architecture_diagram_generate` | ❌ | --- ## Test 204 -**Expected Tool:** `eventgrid_subscription_list` -**Prompt:** List Event Grid subscriptions for topic in resource group +**Expected Tool:** `functionapp_get` +**Prompt:** What is the status of function app ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.746815 | `eventgrid_subscription_list` | ✅ **EXPECTED** | -| 2 | 0.746174 | `eventgrid_topic_list` | ❌ | -| 3 | 0.535569 | `monitor_webtests_list` | ❌ | -| 4 | 0.524919 | `group_list` | ❌ | -| 5 | 0.503158 | `servicebus_topic_details` | ❌ | +| 1 | 0.822908 | `functionapp_get` | ✅ **EXPECTED** | +| 2 | 0.794801 | `deploy_app_logs_get` | ❌ | +| 3 | 0.758340 | `applens_resource_diagnose` | ❌ | +| 4 | 0.736890 | `extension_cli_install` | ❌ | +| 5 | 0.730025 | `cloudarchitect_design` | ❌ | --- ## Test 205 -**Expected Tool:** `eventgrid_subscription_list` -**Prompt:** Show all Event Grid subscriptions in my subscription +**Expected Tool:** `functionapp_get` +**Prompt:** List all function apps in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.736436 | `eventgrid_subscription_list` | ✅ **EXPECTED** | -| 2 | 0.659728 | `eventgrid_topic_list` | ❌ | -| 3 | 0.569308 | `subscription_list` | ❌ | -| 4 | 0.537922 | `kusto_cluster_list` | ❌ | -| 5 | 0.518858 | `search_service_list` | ❌ | +| 1 | 0.855510 | `functionapp_get` | ✅ **EXPECTED** | +| 2 | 0.801000 | `search_service_list` | ❌ | +| 3 | 0.798885 | `deploy_app_logs_get` | ❌ | +| 4 | 0.797428 | `appconfig_account_list` | ❌ | +| 5 | 0.784778 | `eventgrid_subscription_list` | ❌ | --- ## Test 206 -**Expected Tool:** `eventgrid_subscription_list` -**Prompt:** List all Event Grid subscriptions in subscription +**Expected Tool:** `functionapp_get` +**Prompt:** Show me my Azure function apps ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.684543 | `eventgrid_subscription_list` | ✅ **EXPECTED** | -| 2 | 0.656277 | `eventgrid_topic_list` | ❌ | -| 3 | 0.542427 | `subscription_list` | ❌ | -| 4 | 0.521053 | `kusto_cluster_list` | ❌ | -| 5 | 0.510115 | `group_list` | ❌ | +| 1 | 0.846634 | `functionapp_get` | ✅ **EXPECTED** | +| 2 | 0.834459 | `deploy_app_logs_get` | ❌ | +| 3 | 0.813025 | `applens_resource_diagnose` | ❌ | +| 4 | 0.802079 | `extension_cli_install` | ❌ | +| 5 | 0.801066 | `search_service_list` | ❌ | --- ## Test 207 -**Expected Tool:** `eventgrid_subscription_list` -**Prompt:** Show Event Grid subscriptions in resource group in subscription +**Expected Tool:** `functionapp_get` +**Prompt:** What function apps do I have? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.696099 | `eventgrid_subscription_list` | ✅ **EXPECTED** | -| 2 | 0.691740 | `eventgrid_topic_list` | ❌ | -| 3 | 0.557575 | `group_list` | ❌ | -| 4 | 0.510613 | `monitor_webtests_list` | ❌ | -| 5 | 0.504997 | `resourcehealth_availability-status_list` | ❌ | +| 1 | 0.794721 | `deploy_app_logs_get` | ❌ | +| 2 | 0.786712 | `functionapp_get` | ✅ **EXPECTED** | +| 3 | 0.763268 | `applens_resource_diagnose` | ❌ | +| 4 | 0.758227 | `extension_cli_install` | ❌ | +| 5 | 0.756720 | `cloudarchitect_design` | ❌ | --- ## Test 208 -**Expected Tool:** `eventgrid_subscription_list` -**Prompt:** List Event Grid subscriptions for subscription in location +**Expected Tool:** `keyvault_admin_settings_get` +**Prompt:** Get the account settings for my key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.709801 | `eventgrid_subscription_list` | ✅ **EXPECTED** | -| 2 | 0.642095 | `eventgrid_topic_list` | ❌ | -| 3 | 0.506725 | `subscription_list` | ❌ | -| 4 | 0.476763 | `search_service_list` | ❌ | -| 5 | 0.475782 | `kusto_cluster_list` | ❌ | +| 1 | 0.845654 | `keyvault_admin_settings_get` | ✅ **EXPECTED** | +| 2 | 0.812440 | `storage_account_get` | ❌ | +| 3 | 0.803953 | `keyvault_key_get` | ❌ | +| 4 | 0.788578 | `keyvault_secret_get` | ❌ | +| 5 | 0.781050 | `keyvault_certificate_get` | ❌ | --- ## Test 209 -**Expected Tool:** `eventhubs_eventhub_consumergroup_delete` -**Prompt:** Delete my consumer group in my event hub , namespace , and resource group +**Expected Tool:** `keyvault_admin_settings_get` +**Prompt:** Show me the account settings for managed HSM keyvault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.766922 | `eventhubs_eventhub_consumergroup_delete` | ✅ **EXPECTED** | -| 2 | 0.675846 | `eventhubs_eventhub_consumergroup_update` | ❌ | -| 3 | 0.641111 | `eventhubs_eventhub_consumergroup_get` | ❌ | -| 4 | 0.633787 | `eventhubs_namespace_delete` | ❌ | -| 5 | 0.605477 | `eventhubs_eventhub_delete` | ❌ | +| 1 | 0.879992 | `keyvault_admin_settings_get` | ✅ **EXPECTED** | +| 2 | 0.817886 | `storage_account_get` | ❌ | +| 3 | 0.793266 | `keyvault_key_get` | ❌ | +| 4 | 0.792142 | `keyvault_key_create` | ❌ | +| 5 | 0.789252 | `storage_blob_container_get` | ❌ | --- ## Test 210 -**Expected Tool:** `eventhubs_eventhub_consumergroup_get` -**Prompt:** List all consumer groups in my event hub in namespace +**Expected Tool:** `keyvault_admin_settings_get` +**Prompt:** What's the value of the setting in my key vault with name ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.738475 | `eventhubs_eventhub_consumergroup_get` | ✅ **EXPECTED** | -| 2 | 0.634517 | `eventhubs_eventhub_consumergroup_update` | ❌ | -| 3 | 0.626486 | `eventhubs_eventhub_consumergroup_delete` | ❌ | -| 4 | 0.606619 | `eventhubs_namespace_get` | ❌ | -| 5 | 0.593098 | `eventhubs_eventhub_get` | ❌ | +| 1 | 0.793890 | `keyvault_admin_settings_get` | ✅ **EXPECTED** | +| 2 | 0.778320 | `appconfig_kv_set` | ❌ | +| 3 | 0.777060 | `keyvault_secret_create` | ❌ | +| 4 | 0.773529 | `keyvault_secret_get` | ❌ | +| 5 | 0.771355 | `storage_account_get` | ❌ | --- ## Test 211 -**Expected Tool:** `eventhubs_eventhub_consumergroup_get` -**Prompt:** Get the details of my consumer group in my event hub , namespace , and resource group +**Expected Tool:** `keyvault_certificate_create` +**Prompt:** Create a new certificate called in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.712861 | `eventhubs_eventhub_consumergroup_get` | ✅ **EXPECTED** | -| 2 | 0.637170 | `eventhubs_eventhub_consumergroup_update` | ❌ | -| 3 | 0.625913 | `eventhubs_eventhub_consumergroup_delete` | ❌ | -| 4 | 0.576800 | `eventhubs_namespace_get` | ❌ | -| 5 | 0.529940 | `eventhubs_eventhub_get` | ❌ | +| 1 | 0.858241 | `keyvault_certificate_create` | ✅ **EXPECTED** | +| 2 | 0.840150 | `keyvault_certificate_import` | ❌ | +| 3 | 0.838058 | `keyvault_key_create` | ❌ | +| 4 | 0.829966 | `keyvault_certificate_get` | ❌ | +| 5 | 0.817735 | `keyvault_secret_create` | ❌ | --- ## Test 212 -**Expected Tool:** `eventhubs_eventhub_consumergroup_update` -**Prompt:** Create a new consumer group in my event hub , namespace , and resource group +**Expected Tool:** `keyvault_certificate_create` +**Prompt:** Generate a certificate named in key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.757614 | `eventhubs_eventhub_consumergroup_update` | ✅ **EXPECTED** | -| 2 | 0.688923 | `eventhubs_eventhub_consumergroup_get` | ❌ | -| 3 | 0.670026 | `eventhubs_eventhub_consumergroup_delete` | ❌ | -| 4 | 0.554315 | `eventhubs_eventhub_update` | ❌ | -| 5 | 0.545003 | `eventhubs_namespace_get` | ❌ | +| 1 | 0.847433 | `keyvault_certificate_create` | ✅ **EXPECTED** | +| 2 | 0.826699 | `keyvault_certificate_import` | ❌ | +| 3 | 0.823322 | `keyvault_certificate_get` | ❌ | +| 4 | 0.817519 | `keyvault_key_create` | ❌ | +| 5 | 0.806741 | `keyvault_certificate_list` | ❌ | --- ## Test 213 -**Expected Tool:** `eventhubs_eventhub_consumergroup_update` -**Prompt:** Update my consumer group in my event hub , namespace , and resource group +**Expected Tool:** `keyvault_certificate_create` +**Prompt:** Request creation of certificate in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.738818 | `eventhubs_eventhub_consumergroup_update` | ✅ **EXPECTED** | -| 2 | 0.655614 | `eventhubs_eventhub_consumergroup_delete` | ❌ | -| 3 | 0.642219 | `eventhubs_eventhub_consumergroup_get` | ❌ | -| 4 | 0.552234 | `eventhubs_eventhub_update` | ❌ | -| 5 | 0.524019 | `eventhubs_namespace_delete` | ❌ | +| 1 | 0.845848 | `keyvault_certificate_create` | ✅ **EXPECTED** | +| 2 | 0.828698 | `keyvault_certificate_import` | ❌ | +| 3 | 0.826442 | `keyvault_certificate_get` | ❌ | +| 4 | 0.821290 | `keyvault_key_create` | ❌ | +| 5 | 0.811771 | `keyvault_secret_create` | ❌ | --- ## Test 214 -**Expected Tool:** `eventhubs_eventhub_delete` -**Prompt:** Delete my event hub in my namespace and resource group +**Expected Tool:** `keyvault_certificate_create` +**Prompt:** Provision a new key vault certificate in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.699213 | `eventhubs_namespace_delete` | ❌ | -| 2 | 0.688502 | `eventhubs_eventhub_delete` | ✅ **EXPECTED** | -| 3 | 0.627718 | `eventhubs_eventhub_consumergroup_delete` | ❌ | -| 4 | 0.578687 | `eventhubs_namespace_get` | ❌ | -| 5 | 0.552954 | `eventhubs_eventhub_get` | ❌ | +| 1 | 0.840745 | `keyvault_certificate_create` | ✅ **EXPECTED** | +| 2 | 0.827065 | `keyvault_certificate_import` | ❌ | +| 3 | 0.825962 | `keyvault_certificate_get` | ❌ | +| 4 | 0.814983 | `keyvault_key_create` | ❌ | +| 5 | 0.805347 | `keyvault_secret_create` | ❌ | --- ## Test 215 -**Expected Tool:** `eventhubs_eventhub_get` -**Prompt:** List all Event Hubs in my namespace +**Expected Tool:** `keyvault_certificate_create` +**Prompt:** Issue a certificate in key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.773242 | `eventhubs_eventhub_get` | ✅ **EXPECTED** | -| 2 | 0.687582 | `eventhubs_namespace_get` | ❌ | -| 3 | 0.578689 | `eventhubs_eventhub_update` | ❌ | -| 4 | 0.561544 | `eventhubs_namespace_delete` | ❌ | -| 5 | 0.545474 | `eventhubs_eventhub_consumergroup_get` | ❌ | +| 1 | 0.851949 | `keyvault_certificate_create` | ✅ **EXPECTED** | +| 2 | 0.834756 | `keyvault_certificate_get` | ❌ | +| 3 | 0.831743 | `keyvault_certificate_import` | ❌ | +| 4 | 0.817855 | `keyvault_key_create` | ❌ | +| 5 | 0.817641 | `keyvault_certificate_list` | ❌ | --- ## Test 216 -**Expected Tool:** `eventhubs_eventhub_get` -**Prompt:** Get the details of my event hub in my namespace and resource group +**Expected Tool:** `keyvault_certificate_get` +**Prompt:** Show me the certificate in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.637894 | `eventhubs_namespace_get` | ❌ | -| 2 | 0.627458 | `eventhubs_eventhub_get` | ✅ **EXPECTED** | -| 3 | 0.570716 | `eventhubs_eventhub_consumergroup_get` | ❌ | -| 4 | 0.527525 | `eventhubs_eventhub_update` | ❌ | -| 5 | 0.521748 | `eventhubs_namespace_delete` | ❌ | +| 1 | 0.849686 | `keyvault_certificate_get` | ✅ **EXPECTED** | +| 2 | 0.829160 | `keyvault_certificate_list` | ❌ | +| 3 | 0.822051 | `keyvault_certificate_create` | ❌ | +| 4 | 0.816073 | `keyvault_certificate_import` | ❌ | +| 5 | 0.799569 | `keyvault_key_create` | ❌ | --- ## Test 217 -**Expected Tool:** `eventhubs_eventhub_update` -**Prompt:** Create a new event hub in my namespace and resource group +**Expected Tool:** `keyvault_certificate_get` +**Prompt:** Show me the details of the certificate in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.645976 | `eventhubs_eventhub_update` | ✅ **EXPECTED** | -| 2 | 0.605856 | `eventhubs_namespace_get` | ❌ | -| 3 | 0.574389 | `eventhubs_eventhub_get` | ❌ | -| 4 | 0.571676 | `eventhubs_eventhub_consumergroup_update` | ❌ | -| 5 | 0.557550 | `eventhubs_namespace_delete` | ❌ | +| 1 | 0.875089 | `keyvault_certificate_get` | ✅ **EXPECTED** | +| 2 | 0.834060 | `keyvault_certificate_list` | ❌ | +| 3 | 0.832918 | `keyvault_key_get` | ❌ | +| 4 | 0.824555 | `keyvault_certificate_create` | ❌ | +| 5 | 0.821314 | `keyvault_secret_get` | ❌ | --- ## Test 218 -**Expected Tool:** `eventhubs_eventhub_update` -**Prompt:** Update my event hub in my namespace and resource group +**Expected Tool:** `keyvault_certificate_get` +**Prompt:** Get the certificate from vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.655283 | `eventhubs_eventhub_update` | ✅ **EXPECTED** | -| 2 | 0.571661 | `eventhubs_eventhub_delete` | ❌ | -| 3 | 0.568605 | `eventhubs_eventhub_consumergroup_update` | ❌ | -| 4 | 0.568396 | `eventhubs_namespace_get` | ❌ | -| 5 | 0.565977 | `eventhubs_namespace_delete` | ❌ | +| 1 | 0.836168 | `keyvault_certificate_get` | ✅ **EXPECTED** | +| 2 | 0.814736 | `keyvault_certificate_create` | ❌ | +| 3 | 0.807860 | `keyvault_certificate_import` | ❌ | +| 4 | 0.803158 | `keyvault_certificate_list` | ❌ | +| 5 | 0.786042 | `keyvault_key_create` | ❌ | --- ## Test 219 -**Expected Tool:** `eventhubs_namespace_delete` -**Prompt:** Delete my namespace in my resource group +**Expected Tool:** `keyvault_certificate_get` +**Prompt:** Display the certificate details for in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.623995 | `eventhubs_namespace_delete` | ✅ **EXPECTED** | -| 2 | 0.525446 | `eventhubs_namespace_update` | ❌ | -| 3 | 0.505082 | `eventhubs_eventhub_consumergroup_delete` | ❌ | -| 4 | 0.449841 | `eventhubs_namespace_get` | ❌ | -| 5 | 0.435037 | `workbooks_delete` | ❌ | +| 1 | 0.859098 | `keyvault_certificate_get` | ✅ **EXPECTED** | +| 2 | 0.813786 | `keyvault_certificate_create` | ❌ | +| 3 | 0.813564 | `keyvault_key_get` | ❌ | +| 4 | 0.807689 | `keyvault_certificate_list` | ❌ | +| 5 | 0.806574 | `keyvault_secret_get` | ❌ | --- ## Test 220 -**Expected Tool:** `eventhubs_namespace_get` -**Prompt:** List all Event Hubs namespaces in my subscription +**Expected Tool:** `keyvault_certificate_get` +**Prompt:** Retrieve certificate metadata for in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.659838 | `eventhubs_eventhub_get` | ❌ | -| 2 | 0.658827 | `eventhubs_namespace_get` | ✅ **EXPECTED** | -| 3 | 0.607372 | `kusto_cluster_list` | ❌ | -| 4 | 0.557200 | `eventgrid_topic_list` | ❌ | -| 5 | 0.556126 | `eventgrid_subscription_list` | ❌ | +| 1 | 0.822429 | `keyvault_certificate_get` | ✅ **EXPECTED** | +| 2 | 0.794140 | `keyvault_certificate_list` | ❌ | +| 3 | 0.790227 | `keyvault_certificate_create` | ❌ | +| 4 | 0.787678 | `keyvault_certificate_import` | ❌ | +| 5 | 0.778190 | `keyvault_key_get` | ❌ | --- ## Test 221 -**Expected Tool:** `eventhubs_namespace_get` -**Prompt:** Get the details of my namespace in my resource group +**Expected Tool:** `keyvault_certificate_import` +**Prompt:** Import the certificate in file into the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.509806 | `monitor_webtests_get` | ❌ | -| 2 | 0.509763 | `eventhubs_namespace_get` | ✅ **EXPECTED** | -| 3 | 0.497438 | `servicebus_queue_details` | ❌ | -| 4 | 0.490100 | `eventhubs_namespace_update` | ❌ | -| 5 | 0.470456 | `functionapp_get` | ❌ | +| 1 | 0.820998 | `keyvault_certificate_import` | ✅ **EXPECTED** | +| 2 | 0.775551 | `keyvault_certificate_get` | ❌ | +| 3 | 0.773043 | `keyvault_certificate_create` | ❌ | +| 4 | 0.764307 | `keyvault_certificate_list` | ❌ | +| 5 | 0.762211 | `keyvault_key_create` | ❌ | --- ## Test 222 -**Expected Tool:** `eventhubs_namespace_update` -**Prompt:** Create an new namespace in my resource group +**Expected Tool:** `keyvault_certificate_import` +**Prompt:** Import a certificate into the key vault using the name ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.610589 | `eventhubs_namespace_update` | ✅ **EXPECTED** | -| 2 | 0.466958 | `eventhubs_namespace_get` | ❌ | -| 3 | 0.458694 | `eventhubs_namespace_delete` | ❌ | -| 4 | 0.449756 | `workbooks_create` | ❌ | -| 5 | 0.438709 | `eventhubs_eventhub_consumergroup_update` | ❌ | +| 1 | 0.848227 | `keyvault_certificate_import` | ✅ **EXPECTED** | +| 2 | 0.829118 | `keyvault_certificate_get` | ❌ | +| 3 | 0.817167 | `keyvault_certificate_create` | ❌ | +| 4 | 0.803441 | `keyvault_certificate_list` | ❌ | +| 5 | 0.799048 | `keyvault_key_create` | ❌ | --- ## Test 223 -**Expected Tool:** `eventhubs_namespace_update` -**Prompt:** Update my namespace in my resource group +**Expected Tool:** `keyvault_certificate_import` +**Prompt:** Upload certificate file to key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.622338 | `eventhubs_namespace_update` | ✅ **EXPECTED** | -| 2 | 0.474098 | `eventhubs_namespace_delete` | ❌ | -| 3 | 0.448723 | `eventhubs_namespace_get` | ❌ | -| 4 | 0.436549 | `eventhubs_eventhub_consumergroup_update` | ❌ | -| 5 | 0.372632 | `sql_db_rename` | ❌ | +| 1 | 0.828397 | `keyvault_certificate_import` | ✅ **EXPECTED** | +| 2 | 0.793637 | `keyvault_certificate_create` | ❌ | +| 3 | 0.783872 | `keyvault_certificate_get` | ❌ | +| 4 | 0.781328 | `keyvault_key_create` | ❌ | +| 5 | 0.777882 | `keyvault_secret_create` | ❌ | --- ## Test 224 -**Expected Tool:** `functionapp_get` -**Prompt:** Describe the function app in resource group +**Expected Tool:** `keyvault_certificate_import` +**Prompt:** Load certificate from file into vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.660116 | `functionapp_get` | ✅ **EXPECTED** | -| 2 | 0.451613 | `deploy_app_logs_get` | ❌ | -| 3 | 0.450457 | `applens_resource_diagnose` | ❌ | -| 4 | 0.390048 | `mysql_server_list` | ❌ | -| 5 | 0.380314 | `get_bestpractices_get` | ❌ | +| 1 | 0.817359 | `keyvault_certificate_import` | ✅ **EXPECTED** | +| 2 | 0.779378 | `keyvault_certificate_get` | ❌ | +| 3 | 0.775460 | `keyvault_certificate_create` | ❌ | +| 4 | 0.760767 | `keyvault_certificate_list` | ❌ | +| 5 | 0.753999 | `keyvault_secret_create` | ❌ | --- ## Test 225 -**Expected Tool:** `functionapp_get` -**Prompt:** Get configuration for function app +**Expected Tool:** `keyvault_certificate_import` +**Prompt:** Add existing certificate file to the key vault with name ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.607276 | `functionapp_get` | ✅ **EXPECTED** | -| 2 | 0.447400 | `mysql_server_config_get` | ❌ | -| 3 | 0.424693 | `appconfig_account_list` | ❌ | -| 4 | 0.411267 | `appconfig_kv_get` | ❌ | -| 5 | 0.400402 | `deploy_app_logs_get` | ❌ | +| 1 | 0.825556 | `keyvault_certificate_import` | ✅ **EXPECTED** | +| 2 | 0.785854 | `keyvault_certificate_create` | ❌ | +| 3 | 0.777793 | `keyvault_certificate_get` | ❌ | +| 4 | 0.764990 | `keyvault_secret_create` | ❌ | +| 5 | 0.762896 | `keyvault_key_create` | ❌ | --- ## Test 226 -**Expected Tool:** `functionapp_get` -**Prompt:** Get function app status for +**Expected Tool:** `keyvault_certificate_list` +**Prompt:** List all certificates in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.622384 | `functionapp_get` | ✅ **EXPECTED** | -| 2 | 0.413481 | `resourcehealth_availability-status_get` | ❌ | -| 3 | 0.390708 | `resourcehealth_availability-status_list` | ❌ | -| 4 | 0.383533 | `deploy_app_logs_get` | ❌ | -| 5 | 0.360665 | `storage_account_get` | ❌ | +| 1 | 0.886390 | `keyvault_certificate_list` | ✅ **EXPECTED** | +| 2 | 0.841516 | `keyvault_certificate_get` | ❌ | +| 3 | 0.822561 | `keyvault_secret_list` | ❌ | +| 4 | 0.821789 | `keyvault_key_list` | ❌ | +| 5 | 0.815023 | `keyvault_certificate_create` | ❌ | --- ## Test 227 -**Expected Tool:** `functionapp_get` -**Prompt:** Get information about my function app in +**Expected Tool:** `keyvault_certificate_list` +**Prompt:** Show me the certificates in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.690933 | `functionapp_get` | ✅ **EXPECTED** | -| 2 | 0.441937 | `foundry_resource_get` | ❌ | -| 3 | 0.432317 | `resourcehealth_availability-status_list` | ❌ | -| 4 | 0.431821 | `applens_resource_diagnose` | ❌ | -| 5 | 0.429077 | `storage_account_get` | ❌ | +| 1 | 0.851648 | `keyvault_certificate_list` | ✅ **EXPECTED** | +| 2 | 0.840536 | `keyvault_certificate_get` | ❌ | +| 3 | 0.810598 | `keyvault_certificate_create` | ❌ | +| 4 | 0.802496 | `keyvault_certificate_import` | ❌ | +| 5 | 0.798452 | `keyvault_key_get` | ❌ | --- ## Test 228 -**Expected Tool:** `functionapp_get` -**Prompt:** Retrieve host name and status of function app +**Expected Tool:** `keyvault_certificate_list` +**Prompt:** What certificates are in the key vault ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.592791 | `functionapp_get` | ✅ **EXPECTED** | -| 2 | 0.417817 | `resourcehealth_availability-status_get` | ❌ | -| 3 | 0.409712 | `deploy_app_logs_get` | ❌ | -| 4 | 0.399953 | `storage_account_get` | ❌ | -| 5 | 0.392237 | `applens_resource_diagnose` | ❌ | +| 1 | 0.838086 | `keyvault_certificate_list` | ✅ **EXPECTED** | +| 2 | 0.827396 | `keyvault_certificate_get` | ❌ | +| 3 | 0.809956 | `keyvault_certificate_create` | ❌ | +| 4 | 0.799348 | `keyvault_certificate_import` | ❌ | +| 5 | 0.795610 | `keyvault_key_create` | ❌ | --- ## Test 229 -**Expected Tool:** `functionapp_get` -**Prompt:** Show function app details for in +**Expected Tool:** `keyvault_certificate_list` +**Prompt:** List certificate names in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.687363 | `functionapp_get` | ✅ **EXPECTED** | -| 2 | 0.449314 | `deploy_app_logs_get` | ❌ | -| 3 | 0.428552 | `applens_resource_diagnose` | ❌ | -| 4 | 0.424554 | `foundry_resource_get` | ❌ | -| 5 | 0.391852 | `monitor_webtests_get` | ❌ | +| 1 | 0.850566 | `keyvault_certificate_list` | ✅ **EXPECTED** | +| 2 | 0.823931 | `keyvault_certificate_get` | ❌ | +| 3 | 0.801728 | `keyvault_key_list` | ❌ | +| 4 | 0.797232 | `keyvault_certificate_create` | ❌ | +| 5 | 0.796135 | `keyvault_secret_list` | ❌ | --- ## Test 230 -**Expected Tool:** `functionapp_get` -**Prompt:** Show me the details for the function app +**Expected Tool:** `keyvault_certificate_list` +**Prompt:** Enumerate certificates in key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.644882 | `functionapp_get` | ✅ **EXPECTED** | -| 2 | 0.430189 | `deploy_app_logs_get` | ❌ | -| 3 | 0.421082 | `storage_account_get` | ❌ | -| 4 | 0.403311 | `signalr_runtime_get` | ❌ | -| 5 | 0.391615 | `foundry_resource_get` | ❌ | +| 1 | 0.905894 | `keyvault_certificate_list` | ✅ **EXPECTED** | +| 2 | 0.850109 | `keyvault_key_list` | ❌ | +| 3 | 0.844790 | `keyvault_secret_list` | ❌ | +| 4 | 0.825989 | `keyvault_certificate_get` | ❌ | +| 5 | 0.807584 | `keyvault_certificate_create` | ❌ | --- ## Test 231 -**Expected Tool:** `functionapp_get` -**Prompt:** Show plan and region for function app +**Expected Tool:** `keyvault_certificate_list` +**Prompt:** Show certificate names in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.554980 | `functionapp_get` | ✅ **EXPECTED** | -| 2 | 0.426703 | `quota_usage_check` | ❌ | -| 3 | 0.424610 | `deploy_app_logs_get` | ❌ | -| 4 | 0.408011 | `deploy_plan_get` | ❌ | -| 5 | 0.381629 | `deploy_architecture_diagram_generate` | ❌ | +| 1 | 0.848376 | `keyvault_certificate_list` | ✅ **EXPECTED** | +| 2 | 0.836977 | `keyvault_certificate_get` | ❌ | +| 3 | 0.806944 | `keyvault_certificate_create` | ❌ | +| 4 | 0.805745 | `keyvault_key_list` | ❌ | +| 5 | 0.801511 | `keyvault_secret_list` | ❌ | --- ## Test 232 -**Expected Tool:** `functionapp_get` -**Prompt:** What is the status of function app ? +**Expected Tool:** `keyvault_key_create` +**Prompt:** Create a new key called with the RSA type in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.565797 | `functionapp_get` | ✅ **EXPECTED** | -| 2 | 0.403665 | `deploy_app_logs_get` | ❌ | -| 3 | 0.384159 | `resourcehealth_availability-status_list` | ❌ | -| 4 | 0.369868 | `applens_resource_diagnose` | ❌ | -| 5 | 0.355044 | `resourcehealth_availability-status_get` | ❌ | +| 1 | 0.865691 | `keyvault_key_create` | ✅ **EXPECTED** | +| 2 | 0.779557 | `keyvault_secret_create` | ❌ | +| 3 | 0.773761 | `keyvault_certificate_create` | ❌ | +| 4 | 0.770666 | `keyvault_key_get` | ❌ | +| 5 | 0.761625 | `keyvault_certificate_import` | ❌ | --- ## Test 233 -**Expected Tool:** `functionapp_get` -**Prompt:** List all function apps in my subscription +**Expected Tool:** `keyvault_key_create` +**Prompt:** Generate a key with type in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.646561 | `functionapp_get` | ✅ **EXPECTED** | -| 2 | 0.559382 | `search_service_list` | ❌ | -| 3 | 0.534978 | `subscription_list` | ❌ | -| 4 | 0.529031 | `kusto_cluster_list` | ❌ | -| 5 | 0.516618 | `cosmos_account_list` | ❌ | +| 1 | 0.821569 | `keyvault_key_create` | ✅ **EXPECTED** | +| 2 | 0.769618 | `keyvault_secret_create` | ❌ | +| 3 | 0.765658 | `keyvault_key_get` | ❌ | +| 4 | 0.753743 | `keyvault_certificate_create` | ❌ | +| 5 | 0.748896 | `keyvault_key_list` | ❌ | --- ## Test 234 -**Expected Tool:** `functionapp_get` -**Prompt:** Show me my Azure function apps +**Expected Tool:** `keyvault_key_create` +**Prompt:** Create an oct key in the vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.560249 | `functionapp_get` | ✅ **EXPECTED** | -| 2 | 0.464985 | `deploy_app_logs_get` | ❌ | -| 3 | 0.412646 | `search_service_list` | ❌ | -| 4 | 0.411323 | `get_bestpractices_get` | ❌ | -| 5 | 0.398503 | `extension_cli_install` | ❌ | +| 1 | 0.836425 | `keyvault_key_create` | ✅ **EXPECTED** | +| 2 | 0.795505 | `keyvault_secret_create` | ❌ | +| 3 | 0.786292 | `keyvault_key_get` | ❌ | +| 4 | 0.781614 | `keyvault_certificate_create` | ❌ | +| 5 | 0.772805 | `keyvault_secret_get` | ❌ | --- ## Test 235 -**Expected Tool:** `functionapp_get` -**Prompt:** What function apps do I have? +**Expected Tool:** `keyvault_key_create` +**Prompt:** Create an RSA key in the vault with name ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.433675 | `functionapp_get` | ✅ **EXPECTED** | -| 2 | 0.346619 | `deploy_app_logs_get` | ❌ | -| 3 | 0.337966 | `applens_resource_diagnose` | ❌ | -| 4 | 0.316594 | `extension_cli_install` | ❌ | -| 5 | 0.284362 | `get_bestpractices_get` | ❌ | +| 1 | 0.854742 | `keyvault_key_create` | ✅ **EXPECTED** | +| 2 | 0.793508 | `keyvault_secret_create` | ❌ | +| 3 | 0.786903 | `keyvault_certificate_create` | ❌ | +| 4 | 0.784100 | `keyvault_key_get` | ❌ | +| 5 | 0.777868 | `keyvault_certificate_import` | ❌ | --- ## Test 236 -**Expected Tool:** `keyvault_admin_settings_get` -**Prompt:** Get the account settings for my key vault +**Expected Tool:** `keyvault_key_create` +**Prompt:** Create an EC key with name in the vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.604780 | `keyvault_admin_settings_get` | ✅ **EXPECTED** | -| 2 | 0.532196 | `storage_account_get` | ❌ | -| 3 | 0.496629 | `keyvault_key_get` | ❌ | -| 4 | 0.452367 | `appconfig_kv_set` | ❌ | -| 5 | 0.448039 | `keyvault_secret_get` | ❌ | +| 1 | 0.827686 | `keyvault_key_create` | ✅ **EXPECTED** | +| 2 | 0.775423 | `keyvault_secret_create` | ❌ | +| 3 | 0.769017 | `keyvault_key_get` | ❌ | +| 4 | 0.766467 | `keyvault_certificate_create` | ❌ | +| 5 | 0.762525 | `keyvault_certificate_import` | ❌ | --- ## Test 237 -**Expected Tool:** `keyvault_admin_settings_get` -**Prompt:** Show me the account settings for managed HSM keyvault +**Expected Tool:** `keyvault_key_get` +**Prompt:** Show me the key in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.671370 | `keyvault_admin_settings_get` | ✅ **EXPECTED** | -| 2 | 0.455561 | `storage_account_get` | ❌ | -| 3 | 0.441224 | `keyvault_key_get` | ❌ | -| 4 | 0.404666 | `appconfig_kv_set` | ❌ | -| 5 | 0.395274 | `keyvault_secret_get` | ❌ | +| 1 | 0.831379 | `keyvault_key_get` | ✅ **EXPECTED** | +| 2 | 0.817128 | `keyvault_key_create` | ❌ | +| 3 | 0.815133 | `keyvault_secret_get` | ❌ | +| 4 | 0.813706 | `keyvault_certificate_get` | ❌ | +| 5 | 0.808534 | `keyvault_key_list` | ❌ | --- ## Test 238 -**Expected Tool:** `keyvault_admin_settings_get` -**Prompt:** What's the value of the setting in my key vault with name +**Expected Tool:** `keyvault_key_get` +**Prompt:** Show me the details of the key in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.505750 | `keyvault_admin_settings_get` | ✅ **EXPECTED** | -| 2 | 0.496540 | `appconfig_kv_set` | ❌ | -| 3 | 0.420145 | `appconfig_kv_lock_set` | ❌ | -| 4 | 0.419126 | `keyvault_key_get` | ❌ | -| 5 | 0.410215 | `keyvault_secret_get` | ❌ | +| 1 | 0.858223 | `keyvault_key_get` | ✅ **EXPECTED** | +| 2 | 0.834786 | `keyvault_secret_get` | ❌ | +| 3 | 0.832105 | `keyvault_certificate_get` | ❌ | +| 4 | 0.814015 | `storage_account_get` | ❌ | +| 5 | 0.813369 | `keyvault_key_create` | ❌ | --- ## Test 239 -**Expected Tool:** `keyvault_certificate_create` -**Prompt:** Create a new certificate called in the key vault +**Expected Tool:** `keyvault_key_get` +**Prompt:** Get the key from vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.627727 | `keyvault_certificate_create` | ✅ **EXPECTED** | -| 2 | 0.570319 | `keyvault_certificate_import` | ❌ | -| 3 | 0.540199 | `keyvault_key_create` | ❌ | -| 4 | 0.519218 | `keyvault_certificate_get` | ❌ | -| 5 | 0.500027 | `keyvault_certificate_list` | ❌ | +| 1 | 0.791389 | `keyvault_key_get` | ✅ **EXPECTED** | +| 2 | 0.778753 | `keyvault_key_create` | ❌ | +| 3 | 0.778135 | `keyvault_secret_get` | ❌ | +| 4 | 0.769580 | `keyvault_certificate_get` | ❌ | +| 5 | 0.763332 | `keyvault_key_list` | ❌ | --- ## Test 240 -**Expected Tool:** `keyvault_certificate_create` -**Prompt:** Generate a certificate named in key vault +**Expected Tool:** `keyvault_key_get` +**Prompt:** Display the key details for in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.599990 | `keyvault_certificate_create` | ✅ **EXPECTED** | -| 2 | 0.561445 | `keyvault_certificate_import` | ❌ | -| 3 | 0.522706 | `keyvault_certificate_get` | ❌ | -| 4 | 0.502128 | `keyvault_key_create` | ❌ | -| 5 | 0.497145 | `keyvault_certificate_list` | ❌ | +| 1 | 0.847889 | `keyvault_key_get` | ✅ **EXPECTED** | +| 2 | 0.821518 | `keyvault_secret_get` | ❌ | +| 3 | 0.815933 | `keyvault_certificate_get` | ❌ | +| 4 | 0.795505 | `storage_account_get` | ❌ | +| 5 | 0.789318 | `keyvault_key_create` | ❌ | --- ## Test 241 -**Expected Tool:** `keyvault_certificate_create` -**Prompt:** Request creation of certificate in the key vault +**Expected Tool:** `keyvault_key_get` +**Prompt:** Retrieve key metadata for in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.573998 | `keyvault_certificate_create` | ✅ **EXPECTED** | -| 2 | 0.527759 | `keyvault_certificate_import` | ❌ | -| 3 | 0.498278 | `keyvault_certificate_get` | ❌ | -| 4 | 0.481548 | `keyvault_key_create` | ❌ | -| 5 | 0.469601 | `keyvault_certificate_list` | ❌ | +| 1 | 0.794616 | `keyvault_key_get` | ✅ **EXPECTED** | +| 2 | 0.780498 | `storage_account_get` | ❌ | +| 3 | 0.778820 | `keyvault_secret_get` | ❌ | +| 4 | 0.769088 | `keyvault_certificate_get` | ❌ | +| 5 | 0.768384 | `keyvault_admin_settings_get` | ❌ | --- ## Test 242 -**Expected Tool:** `keyvault_certificate_create` -**Prompt:** Provision a new key vault certificate in vault +**Expected Tool:** `keyvault_key_list` +**Prompt:** List all keys in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.591697 | `keyvault_certificate_create` | ✅ **EXPECTED** | -| 2 | 0.562265 | `keyvault_certificate_import` | ❌ | -| 3 | 0.522147 | `keyvault_certificate_get` | ❌ | -| 4 | 0.502529 | `keyvault_key_create` | ❌ | -| 5 | 0.479992 | `keyvault_certificate_list` | ❌ | +| 1 | 0.872277 | `keyvault_key_list` | ✅ **EXPECTED** | +| 2 | 0.848718 | `keyvault_secret_list` | ❌ | +| 3 | 0.834942 | `keyvault_certificate_list` | ❌ | +| 4 | 0.828677 | `keyvault_key_get` | ❌ | +| 5 | 0.814524 | `keyvault_key_create` | ❌ | --- ## Test 243 -**Expected Tool:** `keyvault_certificate_create` -**Prompt:** Issue a certificate in key vault +**Expected Tool:** `keyvault_key_list` +**Prompt:** Show me the keys in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.622788 | `keyvault_certificate_create` | ✅ **EXPECTED** | -| 2 | 0.558532 | `keyvault_certificate_import` | ❌ | -| 3 | 0.534503 | `keyvault_certificate_get` | ❌ | -| 4 | 0.521316 | `keyvault_certificate_list` | ❌ | -| 5 | 0.465056 | `keyvault_key_create` | ❌ | +| 1 | 0.834771 | `keyvault_key_list` | ✅ **EXPECTED** | +| 2 | 0.827778 | `keyvault_key_get` | ❌ | +| 3 | 0.817688 | `keyvault_secret_list` | ❌ | +| 4 | 0.813443 | `keyvault_key_create` | ❌ | +| 5 | 0.809918 | `keyvault_certificate_get` | ❌ | --- ## Test 244 -**Expected Tool:** `keyvault_certificate_get` -**Prompt:** Show me the certificate in the key vault +**Expected Tool:** `keyvault_key_list` +**Prompt:** What keys are in the key vault ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.600625 | `keyvault_certificate_get` | ✅ **EXPECTED** | -| 2 | 0.528405 | `keyvault_certificate_list` | ❌ | -| 3 | 0.519037 | `keyvault_certificate_import` | ❌ | -| 4 | 0.499293 | `keyvault_certificate_create` | ❌ | -| 5 | 0.486608 | `keyvault_key_get` | ❌ | +| 1 | 0.822309 | `keyvault_key_list` | ✅ **EXPECTED** | +| 2 | 0.812245 | `keyvault_key_create` | ❌ | +| 3 | 0.812200 | `keyvault_key_get` | ❌ | +| 4 | 0.802969 | `keyvault_secret_list` | ❌ | +| 5 | 0.797343 | `keyvault_admin_settings_get` | ❌ | --- ## Test 245 -**Expected Tool:** `keyvault_certificate_get` -**Prompt:** Show me the details of the certificate in the key vault +**Expected Tool:** `keyvault_key_list` +**Prompt:** List key names in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.646098 | `keyvault_certificate_get` | ✅ **EXPECTED** | -| 2 | 0.562988 | `keyvault_key_get` | ❌ | -| 3 | 0.514170 | `keyvault_secret_get` | ❌ | -| 4 | 0.509446 | `keyvault_certificate_list` | ❌ | -| 5 | 0.507738 | `keyvault_certificate_import` | ❌ | +| 1 | 0.842759 | `keyvault_key_list` | ✅ **EXPECTED** | +| 2 | 0.818982 | `keyvault_secret_list` | ❌ | +| 3 | 0.816682 | `keyvault_key_get` | ❌ | +| 4 | 0.808507 | `keyvault_certificate_list` | ❌ | +| 5 | 0.797995 | `keyvault_key_create` | ❌ | --- ## Test 246 -**Expected Tool:** `keyvault_certificate_get` -**Prompt:** Get the certificate from vault +**Expected Tool:** `keyvault_key_list` +**Prompt:** Enumerate keys in key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.609523 | `keyvault_certificate_get` | ✅ **EXPECTED** | -| 2 | 0.515570 | `keyvault_certificate_list` | ❌ | -| 3 | 0.511197 | `keyvault_certificate_create` | ❌ | -| 4 | 0.507768 | `keyvault_certificate_import` | ❌ | -| 5 | 0.474394 | `keyvault_key_get` | ❌ | +| 1 | 0.895928 | `keyvault_key_list` | ✅ **EXPECTED** | +| 2 | 0.864689 | `keyvault_secret_list` | ❌ | +| 3 | 0.854825 | `keyvault_certificate_list` | ❌ | +| 4 | 0.812529 | `keyvault_key_get` | ❌ | +| 5 | 0.810374 | `keyvault_key_create` | ❌ | --- ## Test 247 -**Expected Tool:** `keyvault_certificate_get` -**Prompt:** Display the certificate details for in vault +**Expected Tool:** `keyvault_key_list` +**Prompt:** Show key names in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.647669 | `keyvault_certificate_get` | ✅ **EXPECTED** | -| 2 | 0.527400 | `keyvault_key_get` | ❌ | -| 3 | 0.521556 | `keyvault_certificate_list` | ❌ | -| 4 | 0.509796 | `keyvault_certificate_import` | ❌ | -| 5 | 0.501988 | `keyvault_secret_get` | ❌ | +| 1 | 0.842460 | `keyvault_key_list` | ✅ **EXPECTED** | +| 2 | 0.826003 | `keyvault_key_get` | ❌ | +| 3 | 0.823494 | `keyvault_secret_list` | ❌ | +| 4 | 0.810392 | `keyvault_certificate_list` | ❌ | +| 5 | 0.809585 | `keyvault_key_create` | ❌ | --- ## Test 248 -**Expected Tool:** `keyvault_certificate_get` -**Prompt:** Retrieve certificate metadata for in vault +**Expected Tool:** `keyvault_secret_create` +**Prompt:** Create a new secret called with value in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.595959 | `keyvault_certificate_get` | ✅ **EXPECTED** | -| 2 | 0.527404 | `keyvault_certificate_list` | ❌ | -| 3 | 0.519059 | `keyvault_certificate_import` | ❌ | -| 4 | 0.501138 | `keyvault_certificate_create` | ❌ | -| 5 | 0.465174 | `keyvault_key_get` | ❌ | +| 1 | 0.862855 | `keyvault_secret_create` | ✅ **EXPECTED** | +| 2 | 0.815682 | `keyvault_secret_get` | ❌ | +| 3 | 0.805459 | `keyvault_key_create` | ❌ | +| 4 | 0.785645 | `keyvault_certificate_create` | ❌ | +| 5 | 0.778660 | `keyvault_certificate_import` | ❌ | --- ## Test 249 -**Expected Tool:** `keyvault_certificate_import` -**Prompt:** Import the certificate in file into the key vault +**Expected Tool:** `keyvault_secret_create` +**Prompt:** Set a secret named with value in key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.585481 | `keyvault_certificate_import` | ✅ **EXPECTED** | -| 2 | 0.420747 | `keyvault_certificate_get` | ❌ | -| 3 | 0.402595 | `keyvault_certificate_create` | ❌ | -| 4 | 0.399342 | `keyvault_certificate_list` | ❌ | -| 5 | 0.352905 | `keyvault_key_create` | ❌ | +| 1 | 0.860701 | `keyvault_secret_create` | ✅ **EXPECTED** | +| 2 | 0.816343 | `keyvault_secret_get` | ❌ | +| 3 | 0.784856 | `keyvault_key_create` | ❌ | +| 4 | 0.783952 | `keyvault_secret_list` | ❌ | +| 5 | 0.777104 | `appconfig_kv_set` | ❌ | --- ## Test 250 -**Expected Tool:** `keyvault_certificate_import` -**Prompt:** Import a certificate into the key vault using the name +**Expected Tool:** `keyvault_secret_create` +**Prompt:** Store secret value in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.622268 | `keyvault_certificate_import` | ✅ **EXPECTED** | -| 2 | 0.504363 | `keyvault_certificate_get` | ❌ | -| 3 | 0.498895 | `keyvault_certificate_create` | ❌ | -| 4 | 0.448125 | `keyvault_certificate_list` | ❌ | -| 5 | 0.419737 | `keyvault_key_create` | ❌ | +| 1 | 0.849189 | `keyvault_secret_create` | ✅ **EXPECTED** | +| 2 | 0.816193 | `keyvault_secret_get` | ❌ | +| 3 | 0.787393 | `keyvault_key_create` | ❌ | +| 4 | 0.784295 | `keyvault_secret_list` | ❌ | +| 5 | 0.778744 | `appconfig_kv_set` | ❌ | --- ## Test 251 -**Expected Tool:** `keyvault_certificate_import` -**Prompt:** Upload certificate file to key vault +**Expected Tool:** `keyvault_secret_create` +**Prompt:** Add a new version of secret with value in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.595707 | `keyvault_certificate_import` | ✅ **EXPECTED** | -| 2 | 0.453929 | `keyvault_certificate_create` | ❌ | -| 3 | 0.452551 | `keyvault_certificate_get` | ❌ | -| 4 | 0.418203 | `keyvault_certificate_list` | ❌ | -| 5 | 0.413377 | `keyvault_key_create` | ❌ | +| 1 | 0.847957 | `keyvault_secret_create` | ✅ **EXPECTED** | +| 2 | 0.801169 | `keyvault_secret_get` | ❌ | +| 3 | 0.777110 | `keyvault_key_create` | ❌ | +| 4 | 0.774292 | `keyvault_certificate_import` | ❌ | +| 5 | 0.767067 | `keyvault_certificate_create` | ❌ | --- ## Test 252 -**Expected Tool:** `keyvault_certificate_import` -**Prompt:** Load certificate from file into vault +**Expected Tool:** `keyvault_secret_create` +**Prompt:** Update secret to value in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.619480 | `keyvault_certificate_import` | ✅ **EXPECTED** | -| 2 | 0.517804 | `keyvault_certificate_get` | ❌ | -| 3 | 0.480815 | `keyvault_certificate_create` | ❌ | -| 4 | 0.444386 | `keyvault_certificate_list` | ❌ | -| 5 | 0.381873 | `keyvault_key_create` | ❌ | +| 1 | 0.839098 | `keyvault_secret_create` | ✅ **EXPECTED** | +| 2 | 0.814716 | `keyvault_secret_get` | ❌ | +| 3 | 0.773281 | `keyvault_secret_list` | ❌ | +| 4 | 0.766877 | `keyvault_key_create` | ❌ | +| 5 | 0.766012 | `keyvault_certificate_get` | ❌ | --- ## Test 253 -**Expected Tool:** `keyvault_certificate_import` -**Prompt:** Add existing certificate file to the key vault with name +**Expected Tool:** `keyvault_secret_get` +**Prompt:** Show me the secret in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.595488 | `keyvault_certificate_import` | ✅ **EXPECTED** | -| 2 | 0.452556 | `keyvault_certificate_create` | ❌ | -| 3 | 0.441587 | `keyvault_certificate_get` | ❌ | -| 4 | 0.408031 | `keyvault_key_create` | ❌ | -| 5 | 0.392303 | `keyvault_secret_create` | ❌ | +| 1 | 0.847856 | `keyvault_secret_get` | ✅ **EXPECTED** | +| 2 | 0.832925 | `keyvault_secret_create` | ❌ | +| 3 | 0.822559 | `keyvault_secret_list` | ❌ | +| 4 | 0.812702 | `keyvault_key_get` | ❌ | +| 5 | 0.809324 | `keyvault_certificate_get` | ❌ | --- ## Test 254 -**Expected Tool:** `keyvault_certificate_list` -**Prompt:** List all certificates in the key vault +**Expected Tool:** `keyvault_secret_get` +**Prompt:** Show me the details of the secret in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.726124 | `keyvault_certificate_list` | ✅ **EXPECTED** | -| 2 | 0.583110 | `keyvault_key_list` | ❌ | -| 3 | 0.531988 | `keyvault_secret_list` | ❌ | -| 4 | 0.515236 | `keyvault_certificate_get` | ❌ | -| 5 | 0.485792 | `keyvault_certificate_create` | ❌ | +| 1 | 0.868759 | `keyvault_secret_get` | ✅ **EXPECTED** | +| 2 | 0.844743 | `keyvault_key_get` | ❌ | +| 3 | 0.834545 | `keyvault_certificate_get` | ❌ | +| 4 | 0.825237 | `keyvault_secret_list` | ❌ | +| 5 | 0.819269 | `keyvault_secret_create` | ❌ | --- ## Test 255 -**Expected Tool:** `keyvault_certificate_list` -**Prompt:** Show me the certificates in the key vault +**Expected Tool:** `keyvault_secret_get` +**Prompt:** Get the secret from vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.615541 | `keyvault_certificate_list` | ✅ **EXPECTED** | -| 2 | 0.522453 | `keyvault_certificate_get` | ❌ | -| 3 | 0.475156 | `keyvault_key_list` | ❌ | -| 4 | 0.460973 | `keyvault_certificate_create` | ❌ | -| 5 | 0.448139 | `keyvault_key_get` | ❌ | +| 1 | 0.821182 | `keyvault_secret_get` | ✅ **EXPECTED** | +| 2 | 0.814429 | `keyvault_secret_create` | ❌ | +| 3 | 0.782409 | `keyvault_secret_list` | ❌ | +| 4 | 0.781840 | `keyvault_key_get` | ❌ | +| 5 | 0.776044 | `keyvault_certificate_get` | ❌ | --- ## Test 256 -**Expected Tool:** `keyvault_certificate_list` -**Prompt:** What certificates are in the key vault ? +**Expected Tool:** `keyvault_secret_get` +**Prompt:** Display the secret details for in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.624710 | `keyvault_certificate_list` | ✅ **EXPECTED** | -| 2 | 0.519739 | `keyvault_certificate_get` | ❌ | -| 3 | 0.510048 | `keyvault_certificate_create` | ❌ | -| 4 | 0.505534 | `keyvault_certificate_import` | ❌ | -| 5 | 0.497356 | `keyvault_key_list` | ❌ | +| 1 | 0.849990 | `keyvault_secret_get` | ✅ **EXPECTED** | +| 2 | 0.822671 | `keyvault_key_get` | ❌ | +| 3 | 0.812987 | `keyvault_secret_create` | ❌ | +| 4 | 0.810970 | `keyvault_certificate_get` | ❌ | +| 5 | 0.805665 | `keyvault_secret_list` | ❌ | --- ## Test 257 -**Expected Tool:** `keyvault_certificate_list` -**Prompt:** List certificate names in vault +**Expected Tool:** `keyvault_secret_get` +**Prompt:** Retrieve secret metadata for in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.672622 | `keyvault_certificate_list` | ✅ **EXPECTED** | -| 2 | 0.553990 | `keyvault_key_list` | ❌ | -| 3 | 0.511905 | `keyvault_secret_list` | ❌ | -| 4 | 0.507062 | `keyvault_certificate_get` | ❌ | -| 5 | 0.492357 | `keyvault_certificate_create` | ❌ | +| 1 | 0.819671 | `keyvault_secret_get` | ✅ **EXPECTED** | +| 2 | 0.794790 | `keyvault_secret_create` | ❌ | +| 3 | 0.784690 | `keyvault_secret_list` | ❌ | +| 4 | 0.778922 | `keyvault_key_get` | ❌ | +| 5 | 0.776819 | `storage_account_get` | ❌ | --- ## Test 258 -**Expected Tool:** `keyvault_certificate_list` -**Prompt:** Enumerate certificates in key vault +**Expected Tool:** `keyvault_secret_list` +**Prompt:** List all secrets in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.747408 | `keyvault_certificate_list` | ✅ **EXPECTED** | -| 2 | 0.594216 | `keyvault_key_list` | ❌ | -| 3 | 0.558771 | `keyvault_secret_list` | ❌ | -| 4 | 0.515568 | `keyvault_certificate_get` | ❌ | -| 5 | 0.490876 | `keyvault_certificate_create` | ❌ | +| 1 | 0.878939 | `keyvault_secret_list` | ✅ **EXPECTED** | +| 2 | 0.843347 | `keyvault_secret_get` | ❌ | +| 3 | 0.828204 | `keyvault_key_list` | ❌ | +| 4 | 0.823340 | `keyvault_certificate_list` | ❌ | +| 5 | 0.813526 | `keyvault_key_get` | ❌ | --- ## Test 259 -**Expected Tool:** `keyvault_certificate_list` -**Prompt:** Show certificate names in the key vault +**Expected Tool:** `keyvault_secret_list` +**Prompt:** Show me the secrets in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.639711 | `keyvault_certificate_list` | ✅ **EXPECTED** | -| 2 | 0.512475 | `keyvault_certificate_get` | ❌ | -| 3 | 0.507572 | `keyvault_key_list` | ❌ | -| 4 | 0.482583 | `keyvault_certificate_create` | ❌ | -| 5 | 0.464725 | `keyvault_secret_list` | ❌ | +| 1 | 0.830772 | `keyvault_secret_list` | ✅ **EXPECTED** | +| 2 | 0.830679 | `keyvault_secret_get` | ❌ | +| 3 | 0.821267 | `keyvault_key_get` | ❌ | +| 4 | 0.810178 | `keyvault_certificate_get` | ❌ | +| 5 | 0.810099 | `keyvault_secret_create` | ❌ | --- ## Test 260 -**Expected Tool:** `keyvault_key_create` -**Prompt:** Create a new key called with the RSA type in the key vault +**Expected Tool:** `keyvault_secret_list` +**Prompt:** What secrets are in the key vault ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.661466 | `keyvault_key_create` | ✅ **EXPECTED** | -| 2 | 0.456580 | `keyvault_secret_create` | ❌ | -| 3 | 0.451790 | `keyvault_certificate_create` | ❌ | -| 4 | 0.429614 | `keyvault_certificate_import` | ❌ | -| 5 | 0.399326 | `keyvault_key_get` | ❌ | +| 1 | 0.826301 | `keyvault_secret_list` | ✅ **EXPECTED** | +| 2 | 0.814112 | `keyvault_secret_get` | ❌ | +| 3 | 0.800208 | `keyvault_secret_create` | ❌ | +| 4 | 0.798763 | `keyvault_key_get` | ❌ | +| 5 | 0.795196 | `keyvault_admin_settings_get` | ❌ | --- ## Test 261 -**Expected Tool:** `keyvault_key_create` -**Prompt:** Generate a key with type in vault +**Expected Tool:** `keyvault_secret_list` +**Prompt:** List secrets names in vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.641070 | `keyvault_key_create` | ✅ **EXPECTED** | -| 2 | 0.428502 | `keyvault_key_get` | ❌ | -| 3 | 0.422763 | `keyvault_certificate_create` | ❌ | -| 4 | 0.420045 | `keyvault_secret_create` | ❌ | -| 5 | 0.405644 | `appconfig_kv_set` | ❌ | +| 1 | 0.840566 | `keyvault_secret_list` | ✅ **EXPECTED** | +| 2 | 0.821317 | `keyvault_secret_get` | ❌ | +| 3 | 0.800144 | `keyvault_secret_create` | ❌ | +| 4 | 0.797779 | `keyvault_key_list` | ❌ | +| 5 | 0.792826 | `keyvault_key_get` | ❌ | --- ## Test 262 -**Expected Tool:** `keyvault_key_create` -**Prompt:** Create an oct key in the vault +**Expected Tool:** `keyvault_secret_list` +**Prompt:** Enumerate secrets in key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.547493 | `keyvault_key_create` | ✅ **EXPECTED** | -| 2 | 0.463557 | `keyvault_secret_create` | ❌ | -| 3 | 0.447410 | `keyvault_certificate_create` | ❌ | -| 4 | 0.420366 | `keyvault_key_get` | ❌ | -| 5 | 0.404350 | `keyvault_certificate_import` | ❌ | +| 1 | 0.895927 | `keyvault_secret_list` | ✅ **EXPECTED** | +| 2 | 0.852096 | `keyvault_key_list` | ❌ | +| 3 | 0.837096 | `keyvault_certificate_list` | ❌ | +| 4 | 0.827009 | `keyvault_secret_get` | ❌ | +| 5 | 0.815837 | `keyvault_secret_create` | ❌ | --- ## Test 263 -**Expected Tool:** `keyvault_key_create` -**Prompt:** Create an RSA key in the vault with name +**Expected Tool:** `keyvault_secret_list` +**Prompt:** Show secrets names in the key vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.641369 | `keyvault_key_create` | ✅ **EXPECTED** | -| 2 | 0.501636 | `keyvault_secret_create` | ❌ | -| 3 | 0.491735 | `keyvault_certificate_create` | ❌ | -| 4 | 0.464557 | `keyvault_certificate_import` | ❌ | -| 5 | 0.451016 | `keyvault_key_get` | ❌ | +| 1 | 0.844935 | `keyvault_secret_list` | ✅ **EXPECTED** | +| 2 | 0.839355 | `keyvault_secret_get` | ❌ | +| 3 | 0.824265 | `keyvault_key_get` | ❌ | +| 4 | 0.816774 | `keyvault_certificate_get` | ❌ | +| 5 | 0.815124 | `keyvault_secret_create` | ❌ | --- ## Test 264 -**Expected Tool:** `keyvault_key_create` -**Prompt:** Create an EC key with name in the vault +**Expected Tool:** `aks_cluster_get` +**Prompt:** Get the configuration of AKS cluster ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.571567 | `keyvault_key_create` | ✅ **EXPECTED** | -| 2 | 0.443271 | `keyvault_certificate_create` | ❌ | -| 3 | 0.434589 | `keyvault_secret_create` | ❌ | -| 4 | 0.421533 | `keyvault_key_get` | ❌ | -| 5 | 0.400473 | `keyvault_certificate_import` | ❌ | +| 1 | 0.854567 | `aks_cluster_get` | ✅ **EXPECTED** | +| 2 | 0.831267 | `aks_nodepool_get` | ❌ | +| 3 | 0.804847 | `kusto_cluster_get` | ❌ | +| 4 | 0.800764 | `kusto_cluster_list` | ❌ | +| 5 | 0.789156 | `mysql_server_config_get` | ❌ | --- ## Test 265 -**Expected Tool:** `keyvault_key_get` -**Prompt:** Show me the key in the key vault +**Expected Tool:** `aks_cluster_get` +**Prompt:** Show me the details of AKS cluster in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.549488 | `keyvault_key_get` | ✅ **EXPECTED** | -| 2 | 0.468165 | `keyvault_secret_get` | ❌ | -| 3 | 0.452816 | `keyvault_key_create` | ❌ | -| 4 | 0.439969 | `keyvault_key_list` | ❌ | -| 5 | 0.426545 | `keyvault_certificate_get` | ❌ | +| 1 | 0.877317 | `aks_cluster_get` | ✅ **EXPECTED** | +| 2 | 0.854822 | `aks_nodepool_get` | ❌ | +| 3 | 0.835183 | `kusto_cluster_get` | ❌ | +| 4 | 0.817776 | `kusto_cluster_list` | ❌ | +| 5 | 0.804872 | `redis_list` | ❌ | --- ## Test 266 -**Expected Tool:** `keyvault_key_get` -**Prompt:** Show me the details of the key in the key vault +**Expected Tool:** `aks_cluster_get` +**Prompt:** Show me the network configuration for AKS cluster ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.629588 | `keyvault_key_get` | ✅ **EXPECTED** | -| 2 | 0.532648 | `keyvault_secret_get` | ❌ | -| 3 | 0.512421 | `storage_account_get` | ❌ | -| 4 | 0.496007 | `keyvault_certificate_get` | ❌ | -| 5 | 0.456934 | `keyvault_key_create` | ❌ | +| 1 | 0.831186 | `aks_cluster_get` | ✅ **EXPECTED** | +| 2 | 0.824247 | `aks_nodepool_get` | ❌ | +| 3 | 0.787148 | `kusto_cluster_get` | ❌ | +| 4 | 0.782086 | `kusto_cluster_list` | ❌ | +| 5 | 0.777667 | `sql_server_show` | ❌ | --- ## Test 267 -**Expected Tool:** `keyvault_key_get` -**Prompt:** Get the key from vault +**Expected Tool:** `aks_cluster_get` +**Prompt:** What are the details of my AKS cluster in ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.484645 | `keyvault_key_get` | ✅ **EXPECTED** | -| 2 | 0.443182 | `keyvault_key_create` | ❌ | -| 3 | 0.409388 | `keyvault_secret_get` | ❌ | -| 4 | 0.395491 | `keyvault_admin_settings_get` | ❌ | -| 5 | 0.383519 | `appconfig_kv_lock_set` | ❌ | +| 1 | 0.857609 | `aks_cluster_get` | ✅ **EXPECTED** | +| 2 | 0.841157 | `aks_nodepool_get` | ❌ | +| 3 | 0.811485 | `kusto_cluster_get` | ❌ | +| 4 | 0.795363 | `foundry_resource_get` | ❌ | +| 5 | 0.791661 | `storage_account_get` | ❌ | --- ## Test 268 -**Expected Tool:** `keyvault_key_get` -**Prompt:** Display the key details for in vault +**Expected Tool:** `aks_cluster_get` +**Prompt:** List all AKS clusters in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.590303 | `keyvault_key_get` | ✅ **EXPECTED** | -| 2 | 0.488213 | `keyvault_secret_get` | ❌ | -| 3 | 0.476498 | `storage_account_get` | ❌ | -| 4 | 0.460796 | `keyvault_certificate_get` | ❌ | -| 5 | 0.436511 | `keyvault_admin_settings_get` | ❌ | +| 1 | 0.919011 | `aks_cluster_get` | ✅ **EXPECTED** | +| 2 | 0.898604 | `kusto_cluster_list` | ❌ | +| 3 | 0.850028 | `aks_nodepool_get` | ❌ | +| 4 | 0.841096 | `search_service_list` | ❌ | +| 5 | 0.833424 | `kusto_cluster_get` | ❌ | --- ## Test 269 -**Expected Tool:** `keyvault_key_get` -**Prompt:** Retrieve key metadata for in vault +**Expected Tool:** `aks_cluster_get` +**Prompt:** Show me my Azure Kubernetes Service clusters ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.518886 | `keyvault_key_get` | ✅ **EXPECTED** | -| 2 | 0.432950 | `storage_account_get` | ❌ | -| 3 | 0.432742 | `keyvault_admin_settings_get` | ❌ | -| 4 | 0.429131 | `keyvault_key_create` | ❌ | -| 5 | 0.422536 | `keyvault_secret_get` | ❌ | +| 1 | 0.869927 | `aks_cluster_get` | ✅ **EXPECTED** | +| 2 | 0.852885 | `kusto_cluster_list` | ❌ | +| 3 | 0.839721 | `aks_nodepool_get` | ❌ | +| 4 | 0.820677 | `search_service_list` | ❌ | +| 5 | 0.814302 | `kusto_cluster_get` | ❌ | --- ## Test 270 -**Expected Tool:** `keyvault_key_list` -**Prompt:** List all keys in the key vault +**Expected Tool:** `aks_cluster_get` +**Prompt:** What AKS clusters do I have? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.701448 | `keyvault_key_list` | ✅ **EXPECTED** | -| 2 | 0.601513 | `keyvault_certificate_list` | ❌ | -| 3 | 0.587427 | `keyvault_secret_list` | ❌ | -| 4 | 0.498767 | `cosmos_account_list` | ❌ | -| 5 | 0.480129 | `keyvault_admin_settings_get` | ❌ | +| 1 | 0.840306 | `aks_cluster_get` | ✅ **EXPECTED** | +| 2 | 0.816615 | `aks_nodepool_get` | ❌ | +| 3 | 0.778774 | `kusto_cluster_list` | ❌ | +| 4 | 0.761782 | `kusto_cluster_get` | ❌ | +| 5 | 0.757916 | `kusto_database_list` | ❌ | --- ## Test 271 -**Expected Tool:** `keyvault_key_list` -**Prompt:** Show me the keys in the key vault +**Expected Tool:** `aks_nodepool_get` +**Prompt:** Get details for nodepool in AKS cluster in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.549453 | `keyvault_key_list` | ✅ **EXPECTED** | -| 2 | 0.506815 | `keyvault_key_get` | ❌ | -| 3 | 0.475507 | `keyvault_certificate_list` | ❌ | -| 4 | 0.472465 | `keyvault_admin_settings_get` | ❌ | -| 5 | 0.455683 | `keyvault_secret_get` | ❌ | +| 1 | 0.887377 | `aks_nodepool_get` | ✅ **EXPECTED** | +| 2 | 0.836887 | `aks_cluster_get` | ❌ | +| 3 | 0.811633 | `kusto_cluster_get` | ❌ | +| 4 | 0.781283 | `virtualdesktop_hostpool_list` | ❌ | +| 5 | 0.778385 | `kusto_cluster_list` | ❌ | --- ## Test 272 -**Expected Tool:** `keyvault_key_list` -**Prompt:** What keys are in the key vault ? +**Expected Tool:** `aks_nodepool_get` +**Prompt:** Show me the configuration for nodepool in AKS cluster in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.581970 | `keyvault_key_list` | ✅ **EXPECTED** | -| 2 | 0.502245 | `keyvault_admin_settings_get` | ❌ | -| 3 | 0.501481 | `keyvault_certificate_list` | ❌ | -| 4 | 0.476470 | `keyvault_key_get` | ❌ | -| 5 | 0.472414 | `keyvault_secret_list` | ❌ | +| 1 | 0.871515 | `aks_nodepool_get` | ✅ **EXPECTED** | +| 2 | 0.820539 | `aks_cluster_get` | ❌ | +| 3 | 0.790116 | `kusto_cluster_get` | ❌ | +| 4 | 0.776492 | `kusto_cluster_list` | ❌ | +| 5 | 0.774670 | `virtualdesktop_hostpool_list` | ❌ | --- ## Test 273 -**Expected Tool:** `keyvault_key_list` -**Prompt:** List key names in vault +**Expected Tool:** `aks_nodepool_get` +**Prompt:** What is the setup of nodepool for AKS cluster in ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.641314 | `keyvault_key_list` | ✅ **EXPECTED** | -| 2 | 0.559550 | `keyvault_certificate_list` | ❌ | -| 3 | 0.553553 | `keyvault_secret_list` | ❌ | -| 4 | 0.486377 | `keyvault_admin_settings_get` | ❌ | -| 5 | 0.475992 | `cosmos_account_list` | ❌ | +| 1 | 0.854484 | `aks_nodepool_get` | ✅ **EXPECTED** | +| 2 | 0.808546 | `aks_cluster_get` | ❌ | +| 3 | 0.768289 | `kusto_cluster_get` | ❌ | +| 4 | 0.762637 | `kusto_cluster_list` | ❌ | +| 5 | 0.759113 | `eventhubs_namespace_get` | ❌ | --- ## Test 274 -**Expected Tool:** `keyvault_key_list` -**Prompt:** Enumerate keys in key vault +**Expected Tool:** `aks_nodepool_get` +**Prompt:** List nodepools for AKS cluster in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.723266 | `keyvault_key_list` | ✅ **EXPECTED** | -| 2 | 0.611366 | `keyvault_certificate_list` | ❌ | -| 3 | 0.611185 | `keyvault_secret_list` | ❌ | -| 4 | 0.473886 | `keyvault_admin_settings_get` | ❌ | -| 5 | 0.441881 | `keyvault_key_get` | ❌ | +| 1 | 0.870076 | `aks_nodepool_get` | ✅ **EXPECTED** | +| 2 | 0.821292 | `aks_cluster_get` | ❌ | +| 3 | 0.812351 | `kusto_cluster_list` | ❌ | +| 4 | 0.802729 | `virtualdesktop_hostpool_list` | ❌ | +| 5 | 0.775537 | `sql_elastic-pool_list` | ❌ | --- ## Test 275 -**Expected Tool:** `keyvault_key_list` -**Prompt:** Show key names in the key vault +**Expected Tool:** `aks_nodepool_get` +**Prompt:** Show me the nodepool list for AKS cluster in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.570444 | `keyvault_key_list` | ✅ **EXPECTED** | -| 2 | 0.501073 | `keyvault_key_get` | ❌ | -| 3 | 0.500103 | `keyvault_certificate_list` | ❌ | -| 4 | 0.496817 | `storage_account_get` | ❌ | -| 5 | 0.490367 | `keyvault_secret_list` | ❌ | +| 1 | 0.886269 | `aks_nodepool_get` | ✅ **EXPECTED** | +| 2 | 0.837311 | `aks_cluster_get` | ❌ | +| 3 | 0.817541 | `kusto_cluster_list` | ❌ | +| 4 | 0.801824 | `virtualdesktop_hostpool_list` | ❌ | +| 5 | 0.791620 | `kusto_database_list` | ❌ | --- ## Test 276 -**Expected Tool:** `keyvault_secret_create` -**Prompt:** Create a new secret called with value in the key vault +**Expected Tool:** `aks_nodepool_get` +**Prompt:** What nodepools do I have for AKS cluster in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.678482 | `keyvault_secret_create` | ✅ **EXPECTED** | -| 2 | 0.553018 | `keyvault_key_create` | ❌ | -| 3 | 0.512856 | `keyvault_secret_get` | ❌ | -| 4 | 0.475097 | `keyvault_certificate_create` | ❌ | -| 5 | 0.461437 | `appconfig_kv_set` | ❌ | +| 1 | 0.872738 | `aks_nodepool_get` | ✅ **EXPECTED** | +| 2 | 0.815806 | `aks_cluster_get` | ❌ | +| 3 | 0.779339 | `kusto_cluster_list` | ❌ | +| 4 | 0.777183 | `virtualdesktop_hostpool_list` | ❌ | +| 5 | 0.762947 | `eventhubs_namespace_get` | ❌ | --- ## Test 277 -**Expected Tool:** `keyvault_secret_create` -**Prompt:** Set a secret named with value in key vault +**Expected Tool:** `loadtesting_test_create` +**Prompt:** Create a basic URL test using the following endpoint URL that runs for 30 minutes with 45 virtual users. The test name is with the test id and the load testing resource is in the resource group in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.663094 | `keyvault_secret_create` | ✅ **EXPECTED** | -| 2 | 0.519601 | `keyvault_secret_get` | ❌ | -| 3 | 0.512233 | `appconfig_kv_set` | ❌ | -| 4 | 0.458502 | `keyvault_key_create` | ❌ | -| 5 | 0.429785 | `appconfig_kv_lock_set` | ❌ | +| 1 | 0.823045 | `loadtesting_test_create` | ✅ **EXPECTED** | +| 2 | 0.821709 | `loadtesting_testresource_create` | ❌ | +| 3 | 0.810933 | `loadtesting_testrun_create` | ❌ | +| 4 | 0.803467 | `loadtesting_testresource_list` | ❌ | +| 5 | 0.782025 | `loadtesting_test_get` | ❌ | --- ## Test 278 -**Expected Tool:** `keyvault_secret_create` -**Prompt:** Store secret value in the key vault +**Expected Tool:** `loadtesting_test_get` +**Prompt:** Get the load test with id in the load test resource in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.639897 | `keyvault_secret_create` | ✅ **EXPECTED** | -| 2 | 0.509674 | `keyvault_secret_get` | ❌ | -| 3 | 0.485203 | `appconfig_kv_set` | ❌ | -| 4 | 0.484680 | `keyvault_key_create` | ❌ | -| 5 | 0.448995 | `appconfig_kv_lock_set` | ❌ | +| 1 | 0.836205 | `loadtesting_test_get` | ✅ **EXPECTED** | +| 2 | 0.823359 | `loadtesting_testresource_create` | ❌ | +| 3 | 0.823186 | `loadtesting_testrun_create` | ❌ | +| 4 | 0.822609 | `loadtesting_testresource_list` | ❌ | +| 5 | 0.808050 | `monitor_webtests_get` | ❌ | --- ## Test 279 -**Expected Tool:** `keyvault_secret_create` -**Prompt:** Add a new version of secret with value in vault +**Expected Tool:** `loadtesting_testresource_create` +**Prompt:** Create a load test resource in the resource group in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.675157 | `keyvault_secret_create` | ✅ **EXPECTED** | -| 2 | 0.499619 | `keyvault_secret_get` | ❌ | -| 3 | 0.498250 | `keyvault_key_create` | ❌ | -| 4 | 0.479179 | `keyvault_certificate_import` | ❌ | -| 5 | 0.458621 | `appconfig_kv_set` | ❌ | +| 1 | 0.850885 | `loadtesting_testresource_list` | ❌ | +| 2 | 0.848438 | `loadtesting_testresource_create` | ✅ **EXPECTED** | +| 3 | 0.825059 | `loadtesting_testrun_create` | ❌ | +| 4 | 0.823651 | `redis_create` | ❌ | +| 5 | 0.823016 | `monitor_webtests_list` | ❌ | --- ## Test 280 -**Expected Tool:** `keyvault_secret_create` -**Prompt:** Update secret to value in the key vault +**Expected Tool:** `loadtesting_testresource_list` +**Prompt:** List all load testing resources in the resource group in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.571923 | `keyvault_secret_create` | ✅ **EXPECTED** | -| 2 | 0.514116 | `keyvault_secret_get` | ❌ | -| 3 | 0.441321 | `appconfig_kv_set` | ❌ | -| 4 | 0.417947 | `appconfig_kv_lock_set` | ❌ | -| 5 | 0.408461 | `keyvault_key_get` | ❌ | +| 1 | 0.903902 | `loadtesting_testresource_list` | ✅ **EXPECTED** | +| 2 | 0.867615 | `monitor_webtests_list` | ❌ | +| 3 | 0.849957 | `group_list` | ❌ | +| 4 | 0.832848 | `redis_list` | ❌ | +| 5 | 0.831770 | `kusto_cluster_list` | ❌ | --- ## Test 281 -**Expected Tool:** `keyvault_secret_get` -**Prompt:** Show me the secret in the key vault +**Expected Tool:** `loadtesting_testrun_create` +**Prompt:** Create a test run using the id for test in the load testing resource in resource group . Use the name of test run and description as ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.602769 | `keyvault_secret_get` | ✅ **EXPECTED** | -| 2 | 0.504212 | `keyvault_key_get` | ❌ | -| 3 | 0.501397 | `keyvault_secret_create` | ❌ | -| 4 | 0.478769 | `keyvault_secret_list` | ❌ | -| 5 | 0.439521 | `keyvault_certificate_get` | ❌ | +| 1 | 0.877261 | `loadtesting_testrun_create` | ✅ **EXPECTED** | +| 2 | 0.831327 | `loadtesting_testresource_create` | ❌ | +| 3 | 0.820081 | `loadtesting_test_create` | ❌ | +| 4 | 0.816240 | `loadtesting_testrun_update` | ❌ | +| 5 | 0.803680 | `loadtesting_testresource_list` | ❌ | --- ## Test 282 -**Expected Tool:** `keyvault_secret_get` -**Prompt:** Show me the details of the secret in the key vault +**Expected Tool:** `loadtesting_testrun_get` +**Prompt:** Get the load test run with id in the load test resource in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.653871 | `keyvault_secret_get` | ✅ **EXPECTED** | -| 2 | 0.566786 | `keyvault_key_get` | ❌ | -| 3 | 0.517547 | `storage_account_get` | ❌ | -| 4 | 0.496050 | `keyvault_certificate_get` | ❌ | -| 5 | 0.485249 | `keyvault_secret_list` | ❌ | +| 1 | 0.845121 | `loadtesting_testrun_create` | ❌ | +| 2 | 0.832314 | `loadtesting_test_get` | ❌ | +| 3 | 0.818933 | `loadtesting_testresource_create` | ❌ | +| 4 | 0.818014 | `monitor_webtests_get` | ❌ | +| 5 | 0.814756 | `loadtesting_testrun_list` | ❌ | --- ## Test 283 -**Expected Tool:** `keyvault_secret_get` -**Prompt:** Get the secret from vault +**Expected Tool:** `loadtesting_testrun_list` +**Prompt:** Get all the load test runs for the test with id in the load test resource in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.578479 | `keyvault_secret_get` | ✅ **EXPECTED** | -| 2 | 0.492213 | `keyvault_key_get` | ❌ | -| 3 | 0.488705 | `keyvault_secret_create` | ❌ | -| 4 | 0.443676 | `keyvault_secret_list` | ❌ | -| 5 | 0.424167 | `keyvault_admin_settings_get` | ❌ | +| 1 | 0.838956 | `loadtesting_testrun_list` | ✅ **EXPECTED** | +| 2 | 0.831750 | `loadtesting_testresource_list` | ❌ | +| 3 | 0.826910 | `loadtesting_testrun_create` | ❌ | +| 4 | 0.812738 | `loadtesting_test_get` | ❌ | +| 5 | 0.812212 | `monitor_webtests_list` | ❌ | --- ## Test 284 -**Expected Tool:** `keyvault_secret_get` -**Prompt:** Display the secret details for in vault +**Expected Tool:** `loadtesting_testrun_update` +**Prompt:** Update a test run display name as for the id for test in the load testing resource in resource group . ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.649267 | `keyvault_secret_get` | ✅ **EXPECTED** | -| 2 | 0.546992 | `keyvault_key_get` | ❌ | -| 3 | 0.497402 | `storage_account_get` | ❌ | -| 4 | 0.492583 | `keyvault_certificate_get` | ❌ | -| 5 | 0.491597 | `keyvault_secret_list` | ❌ | +| 1 | 0.844152 | `loadtesting_testrun_update` | ✅ **EXPECTED** | +| 2 | 0.807072 | `loadtesting_testrun_create` | ❌ | +| 3 | 0.756178 | `eventhubs_namespace_update` | ❌ | +| 4 | 0.749486 | `monitor_webtests_get` | ❌ | +| 5 | 0.749363 | `loadtesting_test_get` | ❌ | --- ## Test 285 -**Expected Tool:** `keyvault_secret_get` -**Prompt:** Retrieve secret metadata for in vault +**Expected Tool:** `grafana_list` +**Prompt:** List all Azure Managed Grafana in one subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.577477 | `keyvault_secret_get` | ✅ **EXPECTED** | -| 2 | 0.475443 | `keyvault_key_get` | ❌ | -| 3 | 0.466890 | `keyvault_secret_create` | ❌ | -| 4 | 0.447602 | `keyvault_secret_list` | ❌ | -| 5 | 0.439583 | `storage_account_get` | ❌ | +| 1 | 0.846512 | `kusto_cluster_list` | ❌ | +| 2 | 0.841949 | `search_service_list` | ❌ | +| 3 | 0.830453 | `redis_list` | ❌ | +| 4 | 0.827182 | `eventgrid_topic_list` | ❌ | +| 5 | 0.824258 | `subscription_list` | ❌ | --- ## Test 286 -**Expected Tool:** `keyvault_secret_list` -**Prompt:** List all secrets in the key vault +**Expected Tool:** `managedlustre_fs_create` +**Prompt:** Create an Azure Managed Lustre filesystem with name , size , SKU , and subnet for availability zone in location . Maintenance should occur on at ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.701227 | `keyvault_secret_list` | ✅ **EXPECTED** | -| 2 | 0.563736 | `keyvault_key_list` | ❌ | -| 3 | 0.538337 | `keyvault_certificate_list` | ❌ | -| 4 | 0.499642 | `keyvault_secret_get` | ❌ | -| 5 | 0.455500 | `cosmos_account_list` | ❌ | +| 1 | 0.883688 | `managedlustre_fs_create` | ✅ **EXPECTED** | +| 2 | 0.872554 | `managedlustre_fs_subnetsize_validate` | ❌ | +| 3 | 0.849380 | `managedlustre_fs_list` | ❌ | +| 4 | 0.841000 | `managedlustre_fs_sku_get` | ❌ | +| 5 | 0.834033 | `managedlustre_fs_update` | ❌ | --- ## Test 287 -**Expected Tool:** `keyvault_secret_list` -**Prompt:** Show me the secrets in the key vault +**Expected Tool:** `managedlustre_fs_list` +**Prompt:** List the Azure Managed Lustre filesystems in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.555681 | `keyvault_secret_list` | ✅ **EXPECTED** | -| 2 | 0.543861 | `keyvault_secret_get` | ❌ | -| 3 | 0.497525 | `keyvault_key_get` | ❌ | -| 4 | 0.464661 | `keyvault_key_list` | ❌ | -| 5 | 0.453130 | `keyvault_admin_settings_get` | ❌ | +| 1 | 0.890120 | `managedlustre_fs_list` | ✅ **EXPECTED** | +| 2 | 0.826604 | `kusto_cluster_list` | ❌ | +| 3 | 0.825453 | `managedlustre_fs_create` | ❌ | +| 4 | 0.821508 | `managedlustre_fs_sku_get` | ❌ | +| 5 | 0.813953 | `managedlustre_fs_subnetsize_validate` | ❌ | --- ## Test 288 -**Expected Tool:** `keyvault_secret_list` -**Prompt:** What secrets are in the key vault ? +**Expected Tool:** `managedlustre_fs_list` +**Prompt:** List the Azure Managed Lustre filesystems in my resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.572540 | `keyvault_secret_list` | ✅ **EXPECTED** | -| 2 | 0.529258 | `keyvault_secret_get` | ❌ | -| 3 | 0.493761 | `keyvault_key_list` | ❌ | -| 4 | 0.487620 | `keyvault_admin_settings_get` | ❌ | -| 5 | 0.475273 | `keyvault_key_get` | ❌ | +| 1 | 0.881088 | `managedlustre_fs_list` | ✅ **EXPECTED** | +| 2 | 0.827106 | `managedlustre_fs_create` | ❌ | +| 3 | 0.816668 | `managedlustre_fs_sku_get` | ❌ | +| 4 | 0.806393 | `loadtesting_testresource_list` | ❌ | +| 5 | 0.793431 | `mysql_server_list` | ❌ | --- ## Test 289 -**Expected Tool:** `keyvault_secret_list` -**Prompt:** List secrets names in vault +**Expected Tool:** `managedlustre_fs_sku_get` +**Prompt:** List the Azure Managed Lustre SKUs available in location ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.624290 | `keyvault_secret_list` | ✅ **EXPECTED** | -| 2 | 0.559681 | `keyvault_key_list` | ❌ | -| 3 | 0.517516 | `keyvault_certificate_list` | ❌ | -| 4 | 0.479547 | `keyvault_secret_get` | ❌ | -| 5 | 0.454288 | `storage_blob_container_get` | ❌ | +| 1 | 0.909059 | `managedlustre_fs_sku_get` | ✅ **EXPECTED** | +| 2 | 0.839205 | `managedlustre_fs_list` | ❌ | +| 3 | 0.826578 | `managedlustre_fs_subnetsize_validate` | ❌ | +| 4 | 0.808577 | `managedlustre_fs_create` | ❌ | +| 5 | 0.800384 | `storage_account_get` | ❌ | --- ## Test 290 -**Expected Tool:** `keyvault_secret_list` -**Prompt:** Enumerate secrets in key vault +**Expected Tool:** `managedlustre_fs_subnetsize_ask` +**Prompt:** Tell me how many IP addresses I need for an Azure Managed Lustre filesystem of size using the SKU ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.742358 | `keyvault_secret_list` | ✅ **EXPECTED** | -| 2 | 0.601183 | `keyvault_key_list` | ❌ | -| 3 | 0.567827 | `keyvault_certificate_list` | ❌ | -| 4 | 0.496127 | `keyvault_secret_get` | ❌ | -| 5 | 0.437560 | `keyvault_admin_settings_get` | ❌ | +| 1 | 0.878047 | `managedlustre_fs_subnetsize_ask` | ✅ **EXPECTED** | +| 2 | 0.865157 | `managedlustre_fs_subnetsize_validate` | ❌ | +| 3 | 0.836563 | `managedlustre_fs_sku_get` | ❌ | +| 4 | 0.806537 | `managedlustre_fs_create` | ❌ | +| 5 | 0.805975 | `managedlustre_fs_list` | ❌ | --- ## Test 291 -**Expected Tool:** `keyvault_secret_list` -**Prompt:** Show secrets names in the key vault +**Expected Tool:** `managedlustre_fs_subnetsize_validate` +**Prompt:** Validate if the network can host Azure Managed Lustre filesystem of size using the SKU ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.567110 | `keyvault_secret_list` | ✅ **EXPECTED** | -| 2 | 0.522399 | `keyvault_secret_get` | ❌ | -| 3 | 0.476309 | `keyvault_key_list` | ❌ | -| 4 | 0.462677 | `keyvault_secret_create` | ❌ | -| 5 | 0.461326 | `keyvault_key_get` | ❌ | +| 1 | 0.949006 | `managedlustre_fs_subnetsize_validate` | ✅ **EXPECTED** | +| 2 | 0.836190 | `managedlustre_fs_subnetsize_ask` | ❌ | +| 3 | 0.823608 | `managedlustre_fs_create` | ❌ | +| 4 | 0.815984 | `managedlustre_fs_sku_get` | ❌ | +| 5 | 0.809808 | `managedlustre_fs_list` | ❌ | --- ## Test 292 -**Expected Tool:** `aks_cluster_get` -**Prompt:** Get the configuration of AKS cluster +**Expected Tool:** `managedlustre_fs_update` +**Prompt:** Update the maintenance window of the Azure Managed Lustre filesystem to at ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.588300 | `aks_cluster_get` | ✅ **EXPECTED** | -| 2 | 0.544302 | `aks_nodepool_get` | ❌ | -| 3 | 0.515827 | `kusto_cluster_get` | ❌ | -| 4 | 0.481416 | `mysql_server_config_get` | ❌ | -| 5 | 0.430976 | `postgres_server_config_get` | ❌ | +| 1 | 0.870692 | `managedlustre_fs_update` | ✅ **EXPECTED** | +| 2 | 0.826744 | `managedlustre_fs_create` | ❌ | +| 3 | 0.796605 | `managedlustre_fs_list` | ❌ | +| 4 | 0.766323 | `managedlustre_fs_subnetsize_validate` | ❌ | +| 5 | 0.753852 | `managedlustre_fs_sku_get` | ❌ | --- ## Test 293 -**Expected Tool:** `aks_cluster_get` -**Prompt:** Show me the details of AKS cluster in resource group +**Expected Tool:** `marketplace_product_get` +**Prompt:** Get details about marketplace product ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.621759 | `aks_cluster_get` | ✅ **EXPECTED** | -| 2 | 0.575626 | `aks_nodepool_get` | ❌ | -| 3 | 0.570591 | `kusto_cluster_get` | ❌ | -| 4 | 0.461466 | `sql_db_show` | ❌ | -| 5 | 0.444814 | `monitor_webtests_get` | ❌ | +| 1 | 0.840110 | `marketplace_product_get` | ✅ **EXPECTED** | +| 2 | 0.838677 | `marketplace_product_list` | ❌ | +| 3 | 0.767924 | `foundry_resource_get` | ❌ | +| 4 | 0.760387 | `storage_account_get` | ❌ | +| 5 | 0.755465 | `search_index_get` | ❌ | --- ## Test 294 -**Expected Tool:** `aks_cluster_get` -**Prompt:** Show me the network configuration for AKS cluster +**Expected Tool:** `marketplace_product_list` +**Prompt:** Search for Microsoft products in the marketplace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.522525 | `aks_cluster_get` | ✅ **EXPECTED** | -| 2 | 0.483220 | `aks_nodepool_get` | ❌ | -| 3 | 0.437144 | `kusto_cluster_get` | ❌ | -| 4 | 0.380301 | `mysql_server_config_get` | ❌ | -| 5 | 0.366689 | `kusto_cluster_list` | ❌ | +| 1 | 0.853722 | `marketplace_product_list` | ✅ **EXPECTED** | +| 2 | 0.795533 | `foundry_agents_get-sdk-sample` | ❌ | +| 3 | 0.783121 | `foundry_threads_list` | ❌ | +| 4 | 0.777058 | `marketplace_product_get` | ❌ | +| 5 | 0.762771 | `cloudarchitect_design` | ❌ | --- ## Test 295 -**Expected Tool:** `aks_cluster_get` -**Prompt:** What are the details of my AKS cluster in ? +**Expected Tool:** `marketplace_product_list` +**Prompt:** Show me marketplace products from publisher ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.588634 | `aks_cluster_get` | ✅ **EXPECTED** | -| 2 | 0.550555 | `aks_nodepool_get` | ❌ | -| 3 | 0.526511 | `kusto_cluster_get` | ❌ | -| 4 | 0.445722 | `storage_account_get` | ❌ | -| 5 | 0.435597 | `foundry_resource_get` | ❌ | +| 1 | 0.857997 | `marketplace_product_list` | ✅ **EXPECTED** | +| 2 | 0.768986 | `marketplace_product_get` | ❌ | +| 3 | 0.767593 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.752001 | `foundry_models_list` | ❌ | +| 5 | 0.750294 | `eventgrid_topic_list` | ❌ | --- ## Test 296 -**Expected Tool:** `aks_cluster_get` -**Prompt:** List all AKS clusters in my subscription +**Expected Tool:** `get_bestpractices_get` +**Prompt:** Get the latest Azure code generation best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.756471 | `aks_cluster_get` | ✅ **EXPECTED** | -| 2 | 0.749416 | `kusto_cluster_list` | ❌ | -| 3 | 0.590166 | `aks_nodepool_get` | ❌ | -| 4 | 0.568559 | `kusto_database_list` | ❌ | -| 5 | 0.562043 | `search_service_list` | ❌ | +| 1 | 0.878380 | `get_bestpractices_get` | ✅ **EXPECTED** | +| 2 | 0.867833 | `azureaibestpractices_get` | ❌ | +| 3 | 0.848190 | `azureterraformbestpractices_get` | ❌ | +| 4 | 0.827076 | `deploy_iac_rules_get` | ❌ | +| 5 | 0.809418 | `foundry_agents_get-sdk-sample` | ❌ | --- ## Test 297 -**Expected Tool:** `aks_cluster_get` -**Prompt:** Show me my Azure Kubernetes Service clusters +**Expected Tool:** `get_bestpractices_get` +**Prompt:** Get the latest Azure deployment best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.612123 | `aks_cluster_get` | ✅ **EXPECTED** | -| 2 | 0.586661 | `kusto_cluster_list` | ❌ | -| 3 | 0.507757 | `aks_nodepool_get` | ❌ | -| 4 | 0.492737 | `kusto_cluster_get` | ❌ | -| 5 | 0.462988 | `kusto_database_list` | ❌ | +| 1 | 0.863870 | `get_bestpractices_get` | ✅ **EXPECTED** | +| 2 | 0.836840 | `azureaibestpractices_get` | ❌ | +| 3 | 0.836743 | `deploy_iac_rules_get` | ❌ | +| 4 | 0.830549 | `azureterraformbestpractices_get` | ❌ | +| 5 | 0.820558 | `deploy_pipeline_guidance_get` | ❌ | --- ## Test 298 -**Expected Tool:** `aks_cluster_get` -**Prompt:** What AKS clusters do I have? +**Expected Tool:** `get_bestpractices_get` +**Prompt:** Get the latest Azure best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.628429 | `aks_cluster_get` | ✅ **EXPECTED** | -| 2 | 0.563189 | `aks_nodepool_get` | ❌ | -| 3 | 0.526756 | `kusto_cluster_list` | ❌ | -| 4 | 0.425639 | `kusto_cluster_get` | ❌ | -| 5 | 0.409185 | `kusto_database_list` | ❌ | +| 1 | 0.868178 | `get_bestpractices_get` | ✅ **EXPECTED** | +| 2 | 0.853941 | `azureaibestpractices_get` | ❌ | +| 3 | 0.852993 | `azureterraformbestpractices_get` | ❌ | +| 4 | 0.820981 | `deploy_iac_rules_get` | ❌ | +| 5 | 0.814134 | `cloudarchitect_design` | ❌ | --- ## Test 299 -**Expected Tool:** `aks_nodepool_get` -**Prompt:** Get details for nodepool in AKS cluster in +**Expected Tool:** `get_bestpractices_get` +**Prompt:** Get the latest Azure Functions code generation best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.728856 | `aks_nodepool_get` | ✅ **EXPECTED** | -| 2 | 0.516048 | `kusto_cluster_get` | ❌ | -| 3 | 0.509946 | `aks_cluster_get` | ❌ | -| 4 | 0.468438 | `virtualdesktop_hostpool_list` | ❌ | -| 5 | 0.463299 | `sql_elastic-pool_list` | ❌ | +| 1 | 0.868122 | `get_bestpractices_get` | ✅ **EXPECTED** | +| 2 | 0.849932 | `azureaibestpractices_get` | ❌ | +| 3 | 0.823458 | `azureterraformbestpractices_get` | ❌ | +| 4 | 0.812991 | `foundry_agents_get-sdk-sample` | ❌ | +| 5 | 0.808704 | `deploy_iac_rules_get` | ❌ | --- ## Test 300 -**Expected Tool:** `aks_nodepool_get` -**Prompt:** Show me the configuration for nodepool in AKS cluster in resource group +**Expected Tool:** `get_bestpractices_get` +**Prompt:** Get the latest Azure Functions deployment best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.654106 | `aks_nodepool_get` | ✅ **EXPECTED** | -| 2 | 0.458596 | `sql_elastic-pool_list` | ❌ | -| 3 | 0.446035 | `aks_cluster_get` | ❌ | -| 4 | 0.440182 | `virtualdesktop_hostpool_list` | ❌ | -| 5 | 0.415685 | `kusto_cluster_get` | ❌ | +| 1 | 0.858253 | `get_bestpractices_get` | ✅ **EXPECTED** | +| 2 | 0.825094 | `azureaibestpractices_get` | ❌ | +| 3 | 0.819017 | `deploy_iac_rules_get` | ❌ | +| 4 | 0.810336 | `extension_cli_install` | ❌ | +| 5 | 0.808484 | `azureterraformbestpractices_get` | ❌ | --- ## Test 301 -**Expected Tool:** `aks_nodepool_get` -**Prompt:** What is the setup of nodepool for AKS cluster in ? +**Expected Tool:** `get_bestpractices_get` +**Prompt:** Get the latest Azure Functions best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.592806 | `aks_nodepool_get` | ✅ **EXPECTED** | -| 2 | 0.402556 | `aks_cluster_get` | ❌ | -| 3 | 0.385173 | `virtualdesktop_hostpool_list` | ❌ | -| 4 | 0.383045 | `sql_elastic-pool_list` | ❌ | -| 5 | 0.353691 | `kusto_cluster_get` | ❌ | +| 1 | 0.859503 | `get_bestpractices_get` | ✅ **EXPECTED** | +| 2 | 0.837761 | `azureaibestpractices_get` | ❌ | +| 3 | 0.823916 | `azureterraformbestpractices_get` | ❌ | +| 4 | 0.808443 | `extension_cli_install` | ❌ | +| 5 | 0.800786 | `cloudarchitect_design` | ❌ | --- ## Test 302 -**Expected Tool:** `aks_nodepool_get` -**Prompt:** List nodepools for AKS cluster in +**Expected Tool:** `get_bestpractices_get` +**Prompt:** Get the latest Azure Static Web Apps best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.692231 | `aks_nodepool_get` | ✅ **EXPECTED** | -| 2 | 0.519037 | `aks_cluster_get` | ❌ | -| 3 | 0.506624 | `virtualdesktop_hostpool_list` | ❌ | -| 4 | 0.500749 | `kusto_cluster_list` | ❌ | -| 5 | 0.487707 | `sql_elastic-pool_list` | ❌ | +| 1 | 0.847052 | `get_bestpractices_get` | ✅ **EXPECTED** | +| 2 | 0.834727 | `azureaibestpractices_get` | ❌ | +| 3 | 0.819317 | `deploy_iac_rules_get` | ❌ | +| 4 | 0.817462 | `azureterraformbestpractices_get` | ❌ | +| 5 | 0.815950 | `cloudarchitect_design` | ❌ | --- ## Test 303 -**Expected Tool:** `aks_nodepool_get` -**Prompt:** Show me the nodepool list for AKS cluster in +**Expected Tool:** `get_bestpractices_get` +**Prompt:** What are azure function best practices? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.732132 | `aks_nodepool_get` | ✅ **EXPECTED** | -| 2 | 0.561829 | `aks_cluster_get` | ❌ | -| 3 | 0.510269 | `sql_elastic-pool_list` | ❌ | -| 4 | 0.509732 | `virtualdesktop_hostpool_list` | ❌ | -| 5 | 0.486700 | `kusto_cluster_list` | ❌ | +| 1 | 0.841439 | `get_bestpractices_get` | ✅ **EXPECTED** | +| 2 | 0.822991 | `azureaibestpractices_get` | ❌ | +| 3 | 0.816198 | `cloudarchitect_design` | ❌ | +| 4 | 0.815282 | `azureterraformbestpractices_get` | ❌ | +| 5 | 0.799818 | `extension_cli_install` | ❌ | --- ## Test 304 -**Expected Tool:** `aks_nodepool_get` -**Prompt:** What nodepools do I have for AKS cluster in +**Expected Tool:** `get_bestpractices_get` +**Prompt:** configure azure mcp in coding agent for my repo ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.629358 | `aks_nodepool_get` | ✅ **EXPECTED** | -| 2 | 0.456911 | `aks_cluster_get` | ❌ | -| 3 | 0.443902 | `virtualdesktop_hostpool_list` | ❌ | -| 4 | 0.433006 | `kusto_cluster_list` | ❌ | -| 5 | 0.425448 | `sql_elastic-pool_list` | ❌ | +| 1 | 0.820628 | `deploy_plan_get` | ❌ | +| 2 | 0.801789 | `deploy_pipeline_guidance_get` | ❌ | +| 3 | 0.800343 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.799225 | `deploy_iac_rules_get` | ❌ | +| 5 | 0.776799 | `azureaibestpractices_get` | ❌ | --- ## Test 305 -**Expected Tool:** `loadtesting_test_create` -**Prompt:** Create a basic URL test using the following endpoint URL that runs for 30 minutes with 45 virtual users. The test name is with the test id and the load testing resource is in the resource group in my subscription +**Expected Tool:** `monitor_activitylog_list` +**Prompt:** List the activity logs of the last month for ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.577811 | `loadtesting_test_create` | ✅ **EXPECTED** | -| 2 | 0.519418 | `loadtesting_testresource_create` | ❌ | -| 3 | 0.512099 | `loadtesting_testrun_create` | ❌ | -| 4 | 0.472753 | `monitor_webtests_create` | ❌ | -| 5 | 0.460717 | `loadtesting_testresource_list` | ❌ | +| 1 | 0.813939 | `monitor_activitylog_list` | ✅ **EXPECTED** | +| 2 | 0.787050 | `monitor_resource_log_query` | ❌ | +| 3 | 0.780778 | `monitor_workspace_log_query` | ❌ | +| 4 | 0.767093 | `deploy_app_logs_get` | ❌ | +| 5 | 0.747243 | `resourcehealth_health-events_list` | ❌ | --- ## Test 306 -**Expected Tool:** `loadtesting_test_get` -**Prompt:** Get the load test with id in the load test resource in resource group +**Expected Tool:** `monitor_healthmodels_entity_get` +**Prompt:** Show me the health status of entity using the health model ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.626219 | `loadtesting_testresource_list` | ❌ | -| 2 | 0.620023 | `loadtesting_test_get` | ✅ **EXPECTED** | -| 3 | 0.594766 | `loadtesting_testresource_create` | ❌ | -| 4 | 0.591094 | `monitor_webtests_get` | ❌ | -| 5 | 0.536200 | `monitor_webtests_list` | ❌ | +| 1 | 0.836867 | `monitor_healthmodels_entity_get` | ✅ **EXPECTED** | +| 2 | 0.798169 | `resourcehealth_availability-status_get` | ❌ | +| 3 | 0.745048 | `resourcehealth_health-events_list` | ❌ | +| 4 | 0.724226 | `resourcehealth_availability-status_list` | ❌ | +| 5 | 0.721744 | `foundry_models_deployments_list` | ❌ | --- ## Test 307 -**Expected Tool:** `loadtesting_testresource_create` -**Prompt:** Create a load test resource in the resource group in my subscription +**Expected Tool:** `monitor_metrics_definitions` +**Prompt:** Get metric definitions for from the namespace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.645537 | `loadtesting_testresource_create` | ✅ **EXPECTED** | -| 2 | 0.618773 | `loadtesting_testresource_list` | ❌ | -| 3 | 0.541746 | `loadtesting_test_create` | ❌ | -| 4 | 0.539771 | `loadtesting_testrun_create` | ❌ | -| 5 | 0.526684 | `monitor_webtests_list` | ❌ | +| 1 | 0.818978 | `monitor_metrics_definitions` | ✅ **EXPECTED** | +| 2 | 0.769441 | `monitor_metrics_query` | ❌ | +| 3 | 0.740290 | `datadog_monitoredresources_list` | ❌ | +| 4 | 0.734430 | `applens_resource_diagnose` | ❌ | +| 5 | 0.734107 | `eventhubs_namespace_get` | ❌ | --- ## Test 308 -**Expected Tool:** `loadtesting_testresource_list` -**Prompt:** List all load testing resources in the resource group in my subscription +**Expected Tool:** `monitor_metrics_definitions` +**Prompt:** Show me all available metrics and their definitions for storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.794326 | `loadtesting_testresource_list` | ✅ **EXPECTED** | -| 2 | 0.653165 | `monitor_webtests_list` | ❌ | -| 3 | 0.577408 | `group_list` | ❌ | -| 4 | 0.575172 | `loadtesting_testresource_create` | ❌ | -| 5 | 0.565565 | `datadog_monitoredresources_list` | ❌ | +| 1 | 0.844158 | `monitor_metrics_definitions` | ✅ **EXPECTED** | +| 2 | 0.829681 | `storage_account_get` | ❌ | +| 3 | 0.818100 | `storage_blob_container_get` | ❌ | +| 4 | 0.802592 | `quota_usage_check` | ❌ | +| 5 | 0.802175 | `storage_blob_get` | ❌ | --- ## Test 309 -**Expected Tool:** `loadtesting_testrun_create` -**Prompt:** Create a test run using the id for test in the load testing resource in resource group . Use the name of test run and description as +**Expected Tool:** `monitor_metrics_definitions` +**Prompt:** What metric definitions are available for the Application Insights resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.688976 | `loadtesting_testrun_create` | ✅ **EXPECTED** | -| 2 | 0.594879 | `loadtesting_testrun_update` | ❌ | -| 3 | 0.558636 | `loadtesting_test_create` | ❌ | -| 4 | 0.547102 | `loadtesting_testresource_create` | ❌ | -| 5 | 0.496224 | `loadtesting_testresource_list` | ❌ | +| 1 | 0.864559 | `monitor_metrics_definitions` | ✅ **EXPECTED** | +| 2 | 0.808641 | `monitor_metrics_query` | ❌ | +| 3 | 0.804435 | `applens_resource_diagnose` | ❌ | +| 4 | 0.770012 | `bicepschema_get` | ❌ | +| 5 | 0.769714 | `quota_usage_check` | ❌ | --- ## Test 310 -**Expected Tool:** `loadtesting_testrun_get` -**Prompt:** Get the load test run with id in the load test resource in resource group +**Expected Tool:** `monitor_metrics_query` +**Prompt:** Analyze the performance trends and response times for Application Insights resource over the last ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.619210 | `loadtesting_testresource_list` | ❌ | -| 2 | 0.601971 | `loadtesting_test_get` | ❌ | -| 3 | 0.597542 | `loadtesting_testresource_create` | ❌ | -| 4 | 0.577842 | `monitor_webtests_get` | ❌ | -| 5 | 0.565971 | `loadtesting_testrun_list` | ❌ | +| 1 | 0.825544 | `monitor_metrics_query` | ✅ **EXPECTED** | +| 2 | 0.795088 | `applens_resource_diagnose` | ❌ | +| 3 | 0.775598 | `applicationinsights_recommendation_list` | ❌ | +| 4 | 0.772098 | `resourcehealth_health-events_list` | ❌ | +| 5 | 0.771591 | `monitor_resource_log_query` | ❌ | --- ## Test 311 -**Expected Tool:** `loadtesting_testrun_list` -**Prompt:** Get all the load test runs for the test with id in the load test resource in resource group +**Expected Tool:** `monitor_metrics_query` +**Prompt:** Check the availability metrics for my Application Insights resource for the last ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.669185 | `loadtesting_testresource_list` | ❌ | -| 2 | 0.640359 | `loadtesting_testrun_list` | ✅ **EXPECTED** | -| 3 | 0.600996 | `loadtesting_test_get` | ❌ | -| 4 | 0.577352 | `loadtesting_testresource_create` | ❌ | -| 5 | 0.569751 | `monitor_webtests_get` | ❌ | +| 1 | 0.826194 | `monitor_metrics_query` | ✅ **EXPECTED** | +| 2 | 0.800088 | `quota_usage_check` | ❌ | +| 3 | 0.797355 | `applens_resource_diagnose` | ❌ | +| 4 | 0.796373 | `resourcehealth_availability-status_list` | ❌ | +| 5 | 0.792043 | `resourcehealth_availability-status_get` | ❌ | --- ## Test 312 -**Expected Tool:** `loadtesting_testrun_update` -**Prompt:** Update a test run display name as for the id for test in the load testing resource in resource group . +**Expected Tool:** `monitor_metrics_query` +**Prompt:** Get the metric for over the last with intervals ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.706747 | `loadtesting_testrun_update` | ✅ **EXPECTED** | -| 2 | 0.514428 | `loadtesting_testrun_create` | ❌ | -| 3 | 0.486949 | `monitor_webtests_update` | ❌ | -| 4 | 0.470337 | `loadtesting_testresource_list` | ❌ | -| 5 | 0.468109 | `monitor_webtests_get` | ❌ | +| 1 | 0.782679 | `monitor_metrics_query` | ✅ **EXPECTED** | +| 2 | 0.722626 | `monitor_metrics_definitions` | ❌ | +| 3 | 0.707861 | `datadog_monitoredresources_list` | ❌ | +| 4 | 0.704787 | `resourcehealth_health-events_list` | ❌ | +| 5 | 0.703109 | `resourcehealth_availability-status_get` | ❌ | --- ## Test 313 -**Expected Tool:** `grafana_list` -**Prompt:** List all Azure Managed Grafana in one subscription +**Expected Tool:** `monitor_metrics_query` +**Prompt:** Investigate error rates and failed requests for Application Insights resource for the last ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.599427 | `kusto_cluster_list` | ❌ | -| 2 | 0.578892 | `grafana_list` | ✅ **EXPECTED** | -| 3 | 0.551851 | `search_service_list` | ❌ | -| 4 | 0.550360 | `subscription_list` | ❌ | -| 5 | 0.531259 | `redis_list` | ❌ | +| 1 | 0.798339 | `monitor_metrics_query` | ✅ **EXPECTED** | +| 2 | 0.779818 | `applens_resource_diagnose` | ❌ | +| 3 | 0.768299 | `resourcehealth_health-events_list` | ❌ | +| 4 | 0.766578 | `monitor_activitylog_list` | ❌ | +| 5 | 0.765061 | `quota_usage_check` | ❌ | --- ## Test 314 -**Expected Tool:** `managedlustre_fs_create` -**Prompt:** Create an Azure Managed Lustre filesystem with name , size , SKU , and subnet for availability zone in location . Maintenance should occur on at +**Expected Tool:** `monitor_metrics_query` +**Prompt:** Query the metric for for the last ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.728113 | `managedlustre_fs_create` | ✅ **EXPECTED** | -| 2 | 0.616164 | `managedlustre_fs_list` | ❌ | -| 3 | 0.605775 | `managedlustre_fs_sku_get` | ❌ | -| 4 | 0.598255 | `managedlustre_fs_update` | ❌ | -| 5 | 0.557720 | `managedlustre_fs_subnetsize_validate` | ❌ | +| 1 | 0.811105 | `monitor_metrics_query` | ✅ **EXPECTED** | +| 2 | 0.750347 | `monitor_resource_log_query` | ❌ | +| 3 | 0.741873 | `monitor_workspace_log_query` | ❌ | +| 4 | 0.730939 | `datadog_monitoredresources_list` | ❌ | +| 5 | 0.727139 | `monitor_metrics_definitions` | ❌ | --- ## Test 315 -**Expected Tool:** `managedlustre_fs_list` -**Prompt:** List the Azure Managed Lustre filesystems in my subscription +**Expected Tool:** `monitor_metrics_query` +**Prompt:** What's the request per second rate for my Application Insights resource over the last ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.750675 | `managedlustre_fs_list` | ✅ **EXPECTED** | -| 2 | 0.631770 | `managedlustre_fs_sku_get` | ❌ | -| 3 | 0.582660 | `managedlustre_fs_create` | ❌ | -| 4 | 0.562377 | `kusto_cluster_list` | ❌ | -| 5 | 0.513156 | `search_service_list` | ❌ | +| 1 | 0.781554 | `monitor_metrics_query` | ✅ **EXPECTED** | +| 2 | 0.751204 | `applens_resource_diagnose` | ❌ | +| 3 | 0.748179 | `applicationinsights_recommendation_list` | ❌ | +| 4 | 0.743912 | `quota_usage_check` | ❌ | +| 5 | 0.737125 | `resourcehealth_health-events_list` | ❌ | --- ## Test 316 -**Expected Tool:** `managedlustre_fs_list` -**Prompt:** List the Azure Managed Lustre filesystems in my resource group +**Expected Tool:** `monitor_resource_log_query` +**Prompt:** Show me the logs for the past hour for the resource in the Log Analytics workspace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.743903 | `managedlustre_fs_list` | ✅ **EXPECTED** | -| 2 | 0.613217 | `managedlustre_fs_sku_get` | ❌ | -| 3 | 0.565856 | `managedlustre_fs_create` | ❌ | -| 4 | 0.519986 | `datadog_monitoredresources_list` | ❌ | -| 5 | 0.515433 | `loadtesting_testresource_list` | ❌ | +| 1 | 0.864572 | `monitor_workspace_log_query` | ❌ | +| 2 | 0.858333 | `monitor_resource_log_query` | ✅ **EXPECTED** | +| 3 | 0.848487 | `deploy_app_logs_get` | ❌ | +| 4 | 0.840408 | `monitor_activitylog_list` | ❌ | +| 5 | 0.805815 | `monitor_workspace_list` | ❌ | --- ## Test 317 -**Expected Tool:** `managedlustre_fs_sku_get` -**Prompt:** List the Azure Managed Lustre SKUs available in location +**Expected Tool:** `monitor_table_list` +**Prompt:** List all tables in the Log Analytics workspace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.827381 | `managedlustre_fs_sku_get` | ✅ **EXPECTED** | -| 2 | 0.613674 | `managedlustre_fs_list` | ❌ | -| 3 | 0.513242 | `managedlustre_fs_create` | ❌ | -| 4 | 0.496242 | `managedlustre_fs_subnetsize_validate` | ❌ | -| 5 | 0.470241 | `kusto_cluster_list` | ❌ | +| 1 | 0.911959 | `monitor_table_list` | ✅ **EXPECTED** | +| 2 | 0.884489 | `monitor_table_type_list` | ❌ | +| 3 | 0.853296 | `monitor_workspace_list` | ❌ | +| 4 | 0.831372 | `postgres_table_list` | ❌ | +| 5 | 0.829730 | `monitor_workspace_log_query` | ❌ | --- ## Test 318 -**Expected Tool:** `managedlustre_fs_subnetsize_ask` -**Prompt:** Tell me how many IP addresses I need for an Azure Managed Lustre filesystem of size using the SKU +**Expected Tool:** `monitor_table_list` +**Prompt:** Show me the tables in the Log Analytics workspace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.739766 | `managedlustre_fs_subnetsize_ask` | ✅ **EXPECTED** | -| 2 | 0.651598 | `managedlustre_fs_subnetsize_validate` | ❌ | -| 3 | 0.594585 | `managedlustre_fs_sku_get` | ❌ | -| 4 | 0.559498 | `managedlustre_fs_list` | ❌ | -| 5 | 0.533684 | `managedlustre_fs_create` | ❌ | +| 1 | 0.882736 | `monitor_table_list` | ✅ **EXPECTED** | +| 2 | 0.866055 | `monitor_table_type_list` | ❌ | +| 3 | 0.844022 | `monitor_workspace_list` | ❌ | +| 4 | 0.840771 | `deploy_app_logs_get` | ❌ | +| 5 | 0.831257 | `monitor_workspace_log_query` | ❌ | --- ## Test 319 -**Expected Tool:** `managedlustre_fs_subnetsize_validate` -**Prompt:** Validate if the network can host Azure Managed Lustre filesystem of size using the SKU +**Expected Tool:** `monitor_table_type_list` +**Prompt:** List all available table types in the Log Analytics workspace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.879389 | `managedlustre_fs_subnetsize_validate` | ✅ **EXPECTED** | -| 2 | 0.622463 | `managedlustre_fs_subnetsize_ask` | ❌ | -| 3 | 0.542808 | `managedlustre_fs_sku_get` | ❌ | -| 4 | 0.515935 | `managedlustre_fs_create` | ❌ | -| 5 | 0.480855 | `managedlustre_fs_list` | ❌ | +| 1 | 0.948425 | `monitor_table_type_list` | ✅ **EXPECTED** | +| 2 | 0.875367 | `monitor_table_list` | ❌ | +| 3 | 0.821336 | `monitor_workspace_list` | ❌ | +| 4 | 0.806486 | `postgres_table_list` | ❌ | +| 5 | 0.796210 | `deploy_app_logs_get` | ❌ | --- ## Test 320 -**Expected Tool:** `managedlustre_fs_update` -**Prompt:** Update the maintenance window of the Azure Managed Lustre filesystem to at +**Expected Tool:** `monitor_table_type_list` +**Prompt:** Show me the available table types in the Log Analytics workspace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.738949 | `managedlustre_fs_update` | ✅ **EXPECTED** | -| 2 | 0.527679 | `managedlustre_fs_create` | ❌ | -| 3 | 0.487402 | `managedlustre_fs_list` | ❌ | -| 4 | 0.385290 | `managedlustre_fs_sku_get` | ❌ | -| 5 | 0.345040 | `managedlustre_fs_subnetsize_validate` | ❌ | +| 1 | 0.930297 | `monitor_table_type_list` | ✅ **EXPECTED** | +| 2 | 0.858293 | `monitor_table_list` | ❌ | +| 3 | 0.820683 | `monitor_workspace_list` | ❌ | +| 4 | 0.814090 | `deploy_app_logs_get` | ❌ | +| 5 | 0.804498 | `monitor_workspace_log_query` | ❌ | --- ## Test 321 -**Expected Tool:** `marketplace_product_get` -**Prompt:** Get details about marketplace product +**Expected Tool:** `monitor_webtests_create` +**Prompt:** Create a new Standard Web Test with name in my subscription in in a given ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.570145 | `marketplace_product_get` | ✅ **EXPECTED** | -| 2 | 0.499184 | `marketplace_product_list` | ❌ | -| 3 | 0.353256 | `servicebus_topic_subscription_details` | ❌ | -| 4 | 0.333160 | `servicebus_topic_details` | ❌ | -| 5 | 0.330935 | `servicebus_queue_details` | ❌ | +| 1 | 0.814130 | `monitor_webtests_list` | ❌ | +| 2 | 0.801618 | `monitor_webtests_create` | ✅ **EXPECTED** | +| 3 | 0.798365 | `loadtesting_testresource_create` | ❌ | +| 4 | 0.797600 | `monitor_webtests_get` | ❌ | +| 5 | 0.781952 | `loadtesting_testrun_create` | ❌ | --- ## Test 322 -**Expected Tool:** `marketplace_product_list` -**Prompt:** Search for Microsoft products in the marketplace +**Expected Tool:** `monitor_webtests_get` +**Prompt:** Get Web Test details for in my subscription in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.607917 | `marketplace_product_list` | ✅ **EXPECTED** | -| 2 | 0.443133 | `marketplace_product_get` | ❌ | -| 3 | 0.343549 | `search_service_list` | ❌ | -| 4 | 0.330500 | `foundry_models_list` | ❌ | -| 5 | 0.328676 | `managedlustre_fs_sku_get` | ❌ | +| 1 | 0.891564 | `monitor_webtests_get` | ✅ **EXPECTED** | +| 2 | 0.875106 | `monitor_webtests_list` | ❌ | +| 3 | 0.832489 | `loadtesting_testresource_list` | ❌ | +| 4 | 0.795049 | `eventgrid_topic_list` | ❌ | +| 5 | 0.793058 | `group_list` | ❌ | --- ## Test 323 -**Expected Tool:** `marketplace_product_list` -**Prompt:** Show me marketplace products from publisher +**Expected Tool:** `monitor_webtests_list` +**Prompt:** List all Web Test resources in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.537726 | `marketplace_product_list` | ✅ **EXPECTED** | -| 2 | 0.385167 | `marketplace_product_get` | ❌ | -| 3 | 0.308769 | `foundry_models_list` | ❌ | -| 4 | 0.288006 | `redis_list` | ❌ | -| 5 | 0.260387 | `managedlustre_fs_sku_get` | ❌ | +| 1 | 0.887937 | `monitor_webtests_list` | ✅ **EXPECTED** | +| 2 | 0.839863 | `loadtesting_testresource_list` | ❌ | +| 3 | 0.829709 | `eventgrid_subscription_list` | ❌ | +| 4 | 0.821453 | `eventgrid_topic_list` | ❌ | +| 5 | 0.820228 | `redis_list` | ❌ | --- ## Test 324 -**Expected Tool:** `get_bestpractices_get` -**Prompt:** Get the latest Azure code generation best practices +**Expected Tool:** `monitor_webtests_list` +**Prompt:** List all Web Test resources in my subscription in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.646844 | `get_bestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.641125 | `azureaibestpractices_get` | ❌ | -| 3 | 0.635406 | `azureterraformbestpractices_get` | ❌ | -| 4 | 0.586907 | `deploy_iac_rules_get` | ❌ | -| 5 | 0.531728 | `deploy_pipeline_guidance_get` | ❌ | +| 1 | 0.917395 | `monitor_webtests_list` | ✅ **EXPECTED** | +| 2 | 0.862023 | `loadtesting_testresource_list` | ❌ | +| 3 | 0.846033 | `group_list` | ❌ | +| 4 | 0.830941 | `monitor_webtests_get` | ❌ | +| 5 | 0.825441 | `eventgrid_topic_list` | ❌ | --- ## Test 325 -**Expected Tool:** `get_bestpractices_get` -**Prompt:** Get the latest Azure deployment best practices +**Expected Tool:** `monitor_webtests_update` +**Prompt:** Update an existing Standard Web Test with name in my subscription in in a given ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.600903 | `get_bestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.548542 | `azureterraformbestpractices_get` | ❌ | -| 3 | 0.541091 | `deploy_iac_rules_get` | ❌ | -| 4 | 0.516852 | `deploy_plan_get` | ❌ | -| 5 | 0.516443 | `deploy_pipeline_guidance_get` | ❌ | +| 1 | 0.838960 | `monitor_webtests_update` | ✅ **EXPECTED** | +| 2 | 0.804943 | `monitor_webtests_get` | ❌ | +| 3 | 0.797730 | `monitor_webtests_list` | ❌ | +| 4 | 0.777279 | `monitor_webtests_create` | ❌ | +| 5 | 0.760733 | `loadtesting_testrun_create` | ❌ | --- ## Test 326 -**Expected Tool:** `get_bestpractices_get` -**Prompt:** Get the latest Azure best practices +**Expected Tool:** `monitor_workspace_list` +**Prompt:** List all Log Analytics workspaces in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.625259 | `get_bestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.594323 | `azureterraformbestpractices_get` | ❌ | -| 3 | 0.532363 | `azureaibestpractices_get` | ❌ | -| 4 | 0.518643 | `deploy_iac_rules_get` | ❌ | -| 5 | 0.465573 | `deploy_pipeline_guidance_get` | ❌ | +| 1 | 0.918591 | `monitor_workspace_list` | ✅ **EXPECTED** | +| 2 | 0.850868 | `grafana_list` | ❌ | +| 3 | 0.842137 | `deploy_app_logs_get` | ❌ | +| 4 | 0.836532 | `kusto_cluster_list` | ❌ | +| 5 | 0.832656 | `monitor_table_list` | ❌ | --- ## Test 327 -**Expected Tool:** `get_bestpractices_get` -**Prompt:** Get the latest Azure Functions code generation best practices +**Expected Tool:** `monitor_workspace_list` +**Prompt:** Show me my Log Analytics workspaces ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.624273 | `get_bestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.577768 | `azureaibestpractices_get` | ❌ | -| 3 | 0.570488 | `azureterraformbestpractices_get` | ❌ | -| 4 | 0.522998 | `deploy_iac_rules_get` | ❌ | -| 5 | 0.493998 | `deploy_pipeline_guidance_get` | ❌ | +| 1 | 0.870767 | `deploy_app_logs_get` | ❌ | +| 2 | 0.861385 | `monitor_workspace_list` | ✅ **EXPECTED** | +| 3 | 0.828604 | `monitor_workspace_log_query` | ❌ | +| 4 | 0.814130 | `monitor_table_list` | ❌ | +| 5 | 0.807481 | `monitor_table_type_list` | ❌ | --- ## Test 328 -**Expected Tool:** `get_bestpractices_get` -**Prompt:** Get the latest Azure Functions deployment best practices +**Expected Tool:** `monitor_workspace_list` +**Prompt:** Show me the Log Analytics workspaces in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.581850 | `get_bestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.497350 | `deploy_pipeline_guidance_get` | ❌ | -| 3 | 0.495659 | `deploy_iac_rules_get` | ❌ | -| 4 | 0.486886 | `azureterraformbestpractices_get` | ❌ | -| 5 | 0.474511 | `deploy_plan_get` | ❌ | +| 1 | 0.897758 | `monitor_workspace_list` | ✅ **EXPECTED** | +| 2 | 0.857938 | `deploy_app_logs_get` | ❌ | +| 3 | 0.826091 | `monitor_workspace_log_query` | ❌ | +| 4 | 0.822058 | `grafana_list` | ❌ | +| 5 | 0.820925 | `eventgrid_subscription_list` | ❌ | --- ## Test 329 -**Expected Tool:** `get_bestpractices_get` -**Prompt:** Get the latest Azure Functions best practices +**Expected Tool:** `monitor_workspace_log_query` +**Prompt:** Show me the logs for the past hour in the Log Analytics workspace ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.610986 | `get_bestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.532790 | `azureterraformbestpractices_get` | ❌ | -| 3 | 0.513612 | `azureaibestpractices_get` | ❌ | -| 4 | 0.487322 | `deploy_iac_rules_get` | ❌ | -| 5 | 0.458060 | `deploy_pipeline_guidance_get` | ❌ | +| 1 | 0.855186 | `deploy_app_logs_get` | ❌ | +| 2 | 0.853231 | `monitor_workspace_log_query` | ✅ **EXPECTED** | +| 3 | 0.827999 | `monitor_resource_log_query` | ❌ | +| 4 | 0.823474 | `monitor_activitylog_list` | ❌ | +| 5 | 0.815853 | `monitor_workspace_list` | ❌ | --- ## Test 330 -**Expected Tool:** `get_bestpractices_get` -**Prompt:** Get the latest Azure Static Web Apps best practices +**Expected Tool:** `datadog_monitoredresources_list` +**Prompt:** List all monitored resources in the Datadog resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.557862 | `get_bestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.513262 | `azureterraformbestpractices_get` | ❌ | -| 3 | 0.505123 | `deploy_iac_rules_get` | ❌ | -| 4 | 0.502874 | `azureaibestpractices_get` | ❌ | -| 5 | 0.483705 | `deploy_pipeline_guidance_get` | ❌ | +| 1 | 0.893413 | `datadog_monitoredresources_list` | ✅ **EXPECTED** | +| 2 | 0.802297 | `redis_list` | ❌ | +| 3 | 0.777619 | `loadtesting_testresource_list` | ❌ | +| 4 | 0.776555 | `group_list` | ❌ | +| 5 | 0.774190 | `monitor_metrics_query` | ❌ | --- ## Test 331 -**Expected Tool:** `get_bestpractices_get` -**Prompt:** What are azure function best practices? +**Expected Tool:** `datadog_monitoredresources_list` +**Prompt:** Show me the monitored resources in the Datadog resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.582541 | `get_bestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.500368 | `azureterraformbestpractices_get` | ❌ | -| 3 | 0.477057 | `azureaibestpractices_get` | ❌ | -| 4 | 0.472112 | `deploy_iac_rules_get` | ❌ | -| 5 | 0.433134 | `deploy_pipeline_guidance_get` | ❌ | +| 1 | 0.877462 | `datadog_monitoredresources_list` | ✅ **EXPECTED** | +| 2 | 0.805276 | `redis_list` | ❌ | +| 3 | 0.790049 | `monitor_metrics_query` | ❌ | +| 4 | 0.780457 | `quota_usage_check` | ❌ | +| 5 | 0.779392 | `deploy_app_logs_get` | ❌ | --- ## Test 332 -**Expected Tool:** `get_bestpractices_get` -**Prompt:** configure azure mcp in coding agent for my repo +**Expected Tool:** `extension_azqr` +**Prompt:** Check my Azure subscription for any compliance issues or recommendations ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.488855 | `deploy_plan_get` | ❌ | -| 2 | 0.460956 | `deploy_pipeline_guidance_get` | ❌ | -| 3 | 0.390270 | `deploy_iac_rules_get` | ❌ | -| 4 | 0.370298 | `azureterraformbestpractices_get` | ❌ | -| 5 | 0.369169 | `extension_cli_install` | ❌ | +| 1 | 0.843977 | `quota_usage_check` | ❌ | +| 2 | 0.835160 | `applens_resource_diagnose` | ❌ | +| 3 | 0.818047 | `subscription_list` | ❌ | +| 4 | 0.813051 | `extension_azqr` | ✅ **EXPECTED** | +| 5 | 0.811721 | `marketplace_product_list` | ❌ | --- ## Test 333 -**Expected Tool:** `monitor_activitylog_list` -**Prompt:** List the activity logs of the last month for +**Expected Tool:** `extension_azqr` +**Prompt:** Provide compliance recommendations for my current Azure subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.537893 | `monitor_activitylog_list` | ✅ **EXPECTED** | -| 2 | 0.506212 | `monitor_resource_log_query` | ❌ | -| 3 | 0.371728 | `monitor_workspace_log_query` | ❌ | -| 4 | 0.363798 | `resourcehealth_health-events_list` | ❌ | -| 5 | 0.344629 | `datadog_monitoredresources_list` | ❌ | +| 1 | 0.827239 | `quota_usage_check` | ❌ | +| 2 | 0.820824 | `applens_resource_diagnose` | ❌ | +| 3 | 0.818905 | `cloudarchitect_design` | ❌ | +| 4 | 0.814683 | `subscription_list` | ❌ | +| 5 | 0.810504 | `marketplace_product_list` | ❌ | --- ## Test 334 -**Expected Tool:** `monitor_healthmodels_entity_get` -**Prompt:** Show me the health status of entity using the health model +**Expected Tool:** `extension_azqr` +**Prompt:** Scan my Azure subscription for compliance recommendations ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.661413 | `monitor_healthmodels_entity_get` | ✅ **EXPECTED** | -| 2 | 0.609276 | `resourcehealth_availability-status_get` | ❌ | -| 3 | 0.351697 | `resourcehealth_availability-status_list` | ❌ | -| 4 | 0.328321 | `resourcehealth_health-events_list` | ❌ | -| 5 | 0.288705 | `foundry_models_deployments_list` | ❌ | +| 1 | 0.839099 | `quota_usage_check` | ❌ | +| 2 | 0.825566 | `applens_resource_diagnose` | ❌ | +| 3 | 0.822292 | `search_service_list` | ❌ | +| 4 | 0.821161 | `extension_azqr` | ✅ **EXPECTED** | +| 5 | 0.815321 | `marketplace_product_list` | ❌ | --- ## Test 335 -**Expected Tool:** `monitor_metrics_definitions` -**Prompt:** Get metric definitions for from the namespace +**Expected Tool:** `quota_region_availability_list` +**Prompt:** Show me the available regions for these resource types ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.592753 | `monitor_metrics_definitions` | ✅ **EXPECTED** | -| 2 | 0.424141 | `monitor_metrics_query` | ❌ | -| 3 | 0.368319 | `bicepschema_get` | ❌ | -| 4 | 0.332378 | `monitor_table_type_list` | ❌ | -| 5 | 0.324986 | `resourcehealth_availability-status_get` | ❌ | +| 1 | 0.849752 | `quota_region_availability_list` | ✅ **EXPECTED** | +| 2 | 0.795933 | `quota_usage_check` | ❌ | +| 3 | 0.777360 | `redis_list` | ❌ | +| 4 | 0.759715 | `group_list` | ❌ | +| 5 | 0.756750 | `eventgrid_topic_list` | ❌ | --- ## Test 336 -**Expected Tool:** `monitor_metrics_definitions` -**Prompt:** Show me all available metrics and their definitions for storage account +**Expected Tool:** `quota_usage_check` +**Prompt:** Check usage information for in region ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.607600 | `storage_account_get` | ❌ | -| 2 | 0.587779 | `monitor_metrics_definitions` | ✅ **EXPECTED** | -| 3 | 0.544781 | `storage_blob_container_get` | ❌ | -| 4 | 0.495829 | `storage_blob_get` | ❌ | -| 5 | 0.473421 | `managedlustre_fs_list` | ❌ | +| 1 | 0.858891 | `quota_usage_check` | ✅ **EXPECTED** | +| 2 | 0.793713 | `quota_region_availability_list` | ❌ | +| 3 | 0.766212 | `applens_resource_diagnose` | ❌ | +| 4 | 0.763981 | `resourcehealth_availability-status_list` | ❌ | +| 5 | 0.763012 | `resourcehealth_availability-status_get` | ❌ | --- ## Test 337 -**Expected Tool:** `monitor_metrics_definitions` -**Prompt:** What metric definitions are available for the Application Insights resource +**Expected Tool:** `role_assignment_list` +**Prompt:** List all available role assignments in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.633225 | `monitor_metrics_definitions` | ✅ **EXPECTED** | -| 2 | 0.495513 | `monitor_metrics_query` | ❌ | -| 3 | 0.433945 | `monitor_resource_log_query` | ❌ | -| 4 | 0.392960 | `loadtesting_testresource_list` | ❌ | -| 5 | 0.388750 | `bicepschema_get` | ❌ | +| 1 | 0.834996 | `role_assignment_list` | ✅ **EXPECTED** | +| 2 | 0.812028 | `subscription_list` | ❌ | +| 3 | 0.808743 | `kusto_cluster_list` | ❌ | +| 4 | 0.805561 | `search_service_list` | ❌ | +| 5 | 0.804305 | `eventgrid_subscription_list` | ❌ | --- ## Test 338 -**Expected Tool:** `monitor_metrics_query` -**Prompt:** Analyze the performance trends and response times for Application Insights resource over the last +**Expected Tool:** `role_assignment_list` +**Prompt:** Show me the available role assignments in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.555377 | `monitor_metrics_query` | ✅ **EXPECTED** | -| 2 | 0.527530 | `monitor_resource_log_query` | ❌ | -| 3 | 0.464743 | `applens_resource_diagnose` | ❌ | -| 4 | 0.420462 | `resourcehealth_health-events_list` | ❌ | -| 5 | 0.413282 | `applicationinsights_recommendation_list` | ❌ | +| 1 | 0.826482 | `role_assignment_list` | ✅ **EXPECTED** | +| 2 | 0.816682 | `eventgrid_subscription_list` | ❌ | +| 3 | 0.808661 | `subscription_list` | ❌ | +| 4 | 0.804538 | `eventgrid_topic_list` | ❌ | +| 5 | 0.802028 | `redis_list` | ❌ | --- ## Test 339 -**Expected Tool:** `monitor_metrics_query` -**Prompt:** Check the availability metrics for my Application Insights resource for the last +**Expected Tool:** `redis_create` +**Prompt:** Create a new Redis resource named with SKU in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.557905 | `monitor_metrics_query` | ✅ **EXPECTED** | -| 2 | 0.476427 | `monitor_resource_log_query` | ❌ | -| 3 | 0.460069 | `resourcehealth_availability-status_list` | ❌ | -| 4 | 0.455978 | `quota_usage_check` | ❌ | -| 5 | 0.437566 | `monitor_metrics_definitions` | ❌ | +| 1 | 0.844397 | `redis_create` | ✅ **EXPECTED** | +| 2 | 0.801319 | `storage_account_create` | ❌ | +| 3 | 0.785885 | `eventhubs_namespace_update` | ❌ | +| 4 | 0.775091 | `redis_list` | ❌ | +| 5 | 0.761478 | `workbooks_create` | ❌ | --- ## Test 340 -**Expected Tool:** `monitor_metrics_query` -**Prompt:** Get the metric for over the last with intervals +**Expected Tool:** `redis_create` +**Prompt:** Create a new Redis resource for me ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.461177 | `monitor_metrics_query` | ✅ **EXPECTED** | -| 2 | 0.390146 | `monitor_metrics_definitions` | ❌ | -| 3 | 0.338510 | `monitor_resource_log_query` | ❌ | -| 4 | 0.334447 | `resourcehealth_availability-status_get` | ❌ | -| 5 | 0.306284 | `resourcehealth_availability-status_list` | ❌ | +| 1 | 0.858770 | `redis_create` | ✅ **EXPECTED** | +| 2 | 0.796574 | `redis_list` | ❌ | +| 3 | 0.763828 | `loadtesting_testresource_create` | ❌ | +| 4 | 0.734660 | `loadtesting_testrun_create` | ❌ | +| 5 | 0.732287 | `foundry_models_deploy` | ❌ | --- ## Test 341 -**Expected Tool:** `monitor_metrics_query` -**Prompt:** Investigate error rates and failed requests for Application Insights resource for the last +**Expected Tool:** `redis_create` +**Prompt:** Create a Redis cache named with SKU in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.496878 | `monitor_resource_log_query` | ❌ | -| 2 | 0.492138 | `monitor_metrics_query` | ✅ **EXPECTED** | -| 3 | 0.448148 | `applens_resource_diagnose` | ❌ | -| 4 | 0.412184 | `resourcehealth_health-events_list` | ❌ | -| 5 | 0.397335 | `quota_usage_check` | ❌ | +| 1 | 0.836986 | `redis_create` | ✅ **EXPECTED** | +| 2 | 0.787470 | `storage_account_create` | ❌ | +| 3 | 0.784911 | `redis_list` | ❌ | +| 4 | 0.771132 | `eventhubs_namespace_update` | ❌ | +| 5 | 0.751311 | `managedlustre_fs_sku_get` | ❌ | --- ## Test 342 -**Expected Tool:** `monitor_metrics_query` -**Prompt:** Query the metric for for the last +**Expected Tool:** `redis_create` +**Prompt:** Create a new Redis cluster with name , SKU ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.525627 | `monitor_metrics_query` | ✅ **EXPECTED** | -| 2 | 0.406251 | `monitor_resource_log_query` | ❌ | -| 3 | 0.384654 | `monitor_metrics_definitions` | ❌ | -| 4 | 0.347746 | `monitor_workspace_log_query` | ❌ | -| 5 | 0.330761 | `resourcehealth_availability-status_get` | ❌ | +| 1 | 0.834538 | `redis_create` | ✅ **EXPECTED** | +| 2 | 0.782674 | `redis_list` | ❌ | +| 3 | 0.773686 | `eventhubs_namespace_update` | ❌ | +| 4 | 0.769323 | `managedlustre_fs_sku_get` | ❌ | +| 5 | 0.762878 | `storage_account_create` | ❌ | --- ## Test 343 -**Expected Tool:** `monitor_metrics_query` -**Prompt:** What's the request per second rate for my Application Insights resource over the last +**Expected Tool:** `redis_list` +**Prompt:** List all Redis resources in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.480099 | `monitor_metrics_query` | ✅ **EXPECTED** | -| 2 | 0.444791 | `monitor_resource_log_query` | ❌ | -| 3 | 0.388406 | `applens_resource_diagnose` | ❌ | -| 4 | 0.363488 | `quota_usage_check` | ❌ | -| 5 | 0.350097 | `resourcehealth_health-events_list` | ❌ | +| 1 | 0.919341 | `redis_list` | ✅ **EXPECTED** | +| 2 | 0.833124 | `redis_create` | ❌ | +| 3 | 0.819484 | `group_list` | ❌ | +| 4 | 0.817669 | `kusto_cluster_list` | ❌ | +| 5 | 0.813246 | `grafana_list` | ❌ | --- ## Test 344 -**Expected Tool:** `monitor_resource_log_query` -**Prompt:** Show me the logs for the past hour for the resource in the Log Analytics workspace +**Expected Tool:** `redis_list` +**Prompt:** Show me my Redis resources ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.687852 | `monitor_resource_log_query` | ✅ **EXPECTED** | -| 2 | 0.621919 | `monitor_workspace_log_query` | ❌ | -| 3 | 0.598393 | `monitor_activitylog_list` | ❌ | -| 4 | 0.485633 | `deploy_app_logs_get` | ❌ | -| 5 | 0.469703 | `monitor_metrics_query` | ❌ | +| 1 | 0.868860 | `redis_list` | ✅ **EXPECTED** | +| 2 | 0.828893 | `redis_create` | ❌ | +| 3 | 0.779446 | `quota_usage_check` | ❌ | +| 4 | 0.766882 | `datadog_monitoredresources_list` | ❌ | +| 5 | 0.754078 | `deploy_app_logs_get` | ❌ | --- ## Test 345 -**Expected Tool:** `monitor_table_list` -**Prompt:** List all tables in the Log Analytics workspace +**Expected Tool:** `redis_list` +**Prompt:** Show me the Redis resources in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.851109 | `monitor_table_list` | ✅ **EXPECTED** | -| 2 | 0.725756 | `monitor_table_type_list` | ❌ | -| 3 | 0.620445 | `monitor_workspace_list` | ❌ | -| 4 | 0.541928 | `kusto_table_list` | ❌ | -| 5 | 0.539481 | `monitor_workspace_log_query` | ❌ | +| 1 | 0.907147 | `redis_list` | ✅ **EXPECTED** | +| 2 | 0.838890 | `redis_create` | ❌ | +| 3 | 0.805796 | `eventgrid_subscription_list` | ❌ | +| 4 | 0.805620 | `eventgrid_topic_list` | ❌ | +| 5 | 0.792343 | `group_list` | ❌ | --- ## Test 346 -**Expected Tool:** `monitor_table_list` -**Prompt:** Show me the tables in the Log Analytics workspace +**Expected Tool:** `redis_list` +**Prompt:** Show me my Redis caches ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.798487 | `monitor_table_list` | ✅ **EXPECTED** | -| 2 | 0.701173 | `monitor_table_type_list` | ❌ | -| 3 | 0.599916 | `monitor_workspace_list` | ❌ | -| 4 | 0.542820 | `monitor_workspace_log_query` | ❌ | -| 5 | 0.502882 | `monitor_resource_log_query` | ❌ | +| 1 | 0.829540 | `redis_list` | ✅ **EXPECTED** | +| 2 | 0.792130 | `redis_create` | ❌ | +| 3 | 0.761341 | `postgres_database_list` | ❌ | +| 4 | 0.749666 | `postgres_table_list` | ❌ | +| 5 | 0.743058 | `quota_usage_check` | ❌ | --- ## Test 347 -**Expected Tool:** `monitor_table_type_list` -**Prompt:** List all available table types in the Log Analytics workspace +**Expected Tool:** `redis_list` +**Prompt:** Get Redis clusters ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.881511 | `monitor_table_type_list` | ✅ **EXPECTED** | -| 2 | 0.765592 | `monitor_table_list` | ❌ | -| 3 | 0.569921 | `monitor_workspace_list` | ❌ | -| 4 | 0.504683 | `mysql_table_list` | ❌ | -| 5 | 0.497622 | `monitor_workspace_log_query` | ❌ | +| 1 | 0.789084 | `redis_list` | ✅ **EXPECTED** | +| 2 | 0.783134 | `redis_create` | ❌ | +| 3 | 0.782061 | `kusto_cluster_list` | ❌ | +| 4 | 0.768549 | `aks_cluster_get` | ❌ | +| 5 | 0.757925 | `kusto_database_list` | ❌ | --- ## Test 348 -**Expected Tool:** `monitor_table_type_list` -**Prompt:** Show me the available table types in the Log Analytics workspace +**Expected Tool:** `group_list` +**Prompt:** List all resource groups in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.843198 | `monitor_table_type_list` | ✅ **EXPECTED** | -| 2 | 0.736752 | `monitor_table_list` | ❌ | -| 3 | 0.576816 | `monitor_workspace_list` | ❌ | -| 4 | 0.509589 | `monitor_workspace_log_query` | ❌ | -| 5 | 0.481205 | `mysql_table_list` | ❌ | +| 1 | 0.912210 | `group_list` | ✅ **EXPECTED** | +| 2 | 0.853428 | `eventgrid_topic_list` | ❌ | +| 3 | 0.846332 | `kusto_cluster_list` | ❌ | +| 4 | 0.845507 | `redis_list` | ❌ | +| 5 | 0.842132 | `loadtesting_testresource_list` | ❌ | --- ## Test 349 -**Expected Tool:** `monitor_webtests_create` -**Prompt:** Create a new Standard Web Test with name in my subscription in in a given +**Expected Tool:** `group_list` +**Prompt:** Show me my resource groups ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.650793 | `monitor_webtests_create` | ✅ **EXPECTED** | -| 2 | 0.570415 | `monitor_webtests_list` | ❌ | -| 3 | 0.550140 | `monitor_webtests_update` | ❌ | -| 4 | 0.533453 | `monitor_webtests_get` | ❌ | -| 5 | 0.482308 | `loadtesting_testresource_create` | ❌ | +| 1 | 0.831722 | `group_list` | ✅ **EXPECTED** | +| 2 | 0.810008 | `redis_list` | ❌ | +| 3 | 0.807500 | `eventgrid_topic_list` | ❌ | +| 4 | 0.803914 | `quota_usage_check` | ❌ | +| 5 | 0.800318 | `loadtesting_testresource_list` | ❌ | --- ## Test 350 -**Expected Tool:** `monitor_webtests_get` -**Prompt:** Get Web Test details for in my subscription in +**Expected Tool:** `group_list` +**Prompt:** Show me the resource groups in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.759187 | `monitor_webtests_get` | ✅ **EXPECTED** | -| 2 | 0.725442 | `monitor_webtests_list` | ❌ | -| 3 | 0.583815 | `loadtesting_testresource_list` | ❌ | -| 4 | 0.562819 | `monitor_webtests_update` | ❌ | -| 5 | 0.530557 | `monitor_webtests_create` | ❌ | +| 1 | 0.885575 | `group_list` | ✅ **EXPECTED** | +| 2 | 0.857088 | `eventgrid_topic_list` | ❌ | +| 3 | 0.855957 | `redis_list` | ❌ | +| 4 | 0.847026 | `eventgrid_subscription_list` | ❌ | +| 5 | 0.829705 | `quota_region_availability_list` | ❌ | --- ## Test 351 -**Expected Tool:** `monitor_webtests_list` -**Prompt:** List all Web Test resources in my subscription +**Expected Tool:** `resourcehealth_availability-status_get` +**Prompt:** Get the availability status for resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.730601 | `monitor_webtests_list` | ✅ **EXPECTED** | -| 2 | 0.610207 | `loadtesting_testresource_list` | ❌ | -| 3 | 0.547723 | `grafana_list` | ❌ | -| 4 | 0.520872 | `redis_list` | ❌ | -| 5 | 0.496298 | `monitor_webtests_get` | ❌ | +| 1 | 0.840651 | `resourcehealth_availability-status_get` | ✅ **EXPECTED** | +| 2 | 0.821593 | `resourcehealth_availability-status_list` | ❌ | +| 3 | 0.767782 | `quota_usage_check` | ❌ | +| 4 | 0.756824 | `monitor_metrics_definitions` | ❌ | +| 5 | 0.752841 | `monitor_healthmodels_entity_get` | ❌ | --- ## Test 352 -**Expected Tool:** `monitor_webtests_list` -**Prompt:** List all Web Test resources in my subscription in +**Expected Tool:** `resourcehealth_availability-status_get` +**Prompt:** Show me the health status of the storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.793807 | `monitor_webtests_list` | ✅ **EXPECTED** | -| 2 | 0.675965 | `loadtesting_testresource_list` | ❌ | -| 3 | 0.584671 | `monitor_webtests_get` | ❌ | -| 4 | 0.573602 | `group_list` | ❌ | -| 5 | 0.546088 | `resourcehealth_availability-status_list` | ❌ | +| 1 | 0.844127 | `resourcehealth_availability-status_get` | ✅ **EXPECTED** | +| 2 | 0.824312 | `storage_account_get` | ❌ | +| 3 | 0.821574 | `storage_blob_container_get` | ❌ | +| 4 | 0.813201 | `resourcehealth_availability-status_list` | ❌ | +| 5 | 0.807901 | `quota_usage_check` | ❌ | --- ## Test 353 -**Expected Tool:** `monitor_webtests_update` -**Prompt:** Update an existing Standard Web Test with name in my subscription in in a given +**Expected Tool:** `resourcehealth_availability-status_get` +**Prompt:** What is the availability status of virtual machine in resource group ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.686394 | `monitor_webtests_update` | ✅ **EXPECTED** | -| 2 | 0.559036 | `monitor_webtests_get` | ❌ | -| 3 | 0.558234 | `monitor_webtests_create` | ❌ | -| 4 | 0.553726 | `monitor_webtests_list` | ❌ | -| 5 | 0.508736 | `loadtesting_testrun_update` | ❌ | +| 1 | 0.838295 | `resourcehealth_availability-status_get` | ✅ **EXPECTED** | +| 2 | 0.832085 | `resourcehealth_availability-status_list` | ❌ | +| 3 | 0.791186 | `quota_usage_check` | ❌ | +| 4 | 0.771561 | `applens_resource_diagnose` | ❌ | +| 5 | 0.770734 | `managedlustre_fs_list` | ❌ | --- ## Test 354 -**Expected Tool:** `monitor_workspace_list` -**Prompt:** List all Log Analytics workspaces in my subscription +**Expected Tool:** `resourcehealth_availability-status_list` +**Prompt:** List availability status for all resources in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.813902 | `monitor_workspace_list` | ✅ **EXPECTED** | -| 2 | 0.680201 | `grafana_list` | ❌ | -| 3 | 0.660255 | `monitor_table_list` | ❌ | -| 4 | 0.610623 | `kusto_cluster_list` | ❌ | -| 5 | 0.600802 | `search_service_list` | ❌ | +| 1 | 0.909545 | `resourcehealth_availability-status_list` | ✅ **EXPECTED** | +| 2 | 0.852783 | `redis_list` | ❌ | +| 3 | 0.838126 | `loadtesting_testresource_list` | ❌ | +| 4 | 0.837929 | `resourcehealth_availability-status_get` | ❌ | +| 5 | 0.826610 | `search_service_list` | ❌ | --- ## Test 355 -**Expected Tool:** `monitor_workspace_list` -**Prompt:** Show me my Log Analytics workspaces +**Expected Tool:** `resourcehealth_availability-status_list` +**Prompt:** Show me the health status of all my Azure resources ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.656194 | `monitor_workspace_list` | ✅ **EXPECTED** | -| 2 | 0.585576 | `monitor_table_list` | ❌ | -| 3 | 0.531180 | `monitor_table_type_list` | ❌ | -| 4 | 0.518254 | `grafana_list` | ❌ | -| 5 | 0.506772 | `monitor_workspace_log_query` | ❌ | +| 1 | 0.869075 | `resourcehealth_availability-status_list` | ✅ **EXPECTED** | +| 2 | 0.852189 | `quota_usage_check` | ❌ | +| 3 | 0.845670 | `resourcehealth_availability-status_get` | ❌ | +| 4 | 0.822780 | `applens_resource_diagnose` | ❌ | +| 5 | 0.819445 | `loadtesting_testresource_list` | ❌ | --- ## Test 356 -**Expected Tool:** `monitor_workspace_list` -**Prompt:** Show me the Log Analytics workspaces in my subscription +**Expected Tool:** `resourcehealth_availability-status_list` +**Prompt:** What resources in resource group have health issues? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.732962 | `monitor_workspace_list` | ✅ **EXPECTED** | -| 2 | 0.601481 | `grafana_list` | ❌ | -| 3 | 0.580418 | `monitor_table_list` | ❌ | -| 4 | 0.523782 | `monitor_workspace_log_query` | ❌ | -| 5 | 0.522749 | `kusto_cluster_list` | ❌ | +| 1 | 0.804274 | `resourcehealth_availability-status_get` | ❌ | +| 2 | 0.802309 | `resourcehealth_availability-status_list` | ✅ **EXPECTED** | +| 3 | 0.802099 | `applens_resource_diagnose` | ❌ | +| 4 | 0.785235 | `resourcehealth_health-events_list` | ❌ | +| 5 | 0.781158 | `quota_usage_check` | ❌ | --- ## Test 357 -**Expected Tool:** `monitor_workspace_log_query` -**Prompt:** Show me the logs for the past hour in the Log Analytics workspace +**Expected Tool:** `resourcehealth_health-events_list` +**Prompt:** List all service health events in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.610115 | `monitor_workspace_log_query` | ✅ **EXPECTED** | -| 2 | 0.587614 | `monitor_resource_log_query` | ❌ | -| 3 | 0.527733 | `monitor_activitylog_list` | ❌ | -| 4 | 0.498269 | `deploy_app_logs_get` | ❌ | -| 5 | 0.486044 | `monitor_table_list` | ❌ | +| 1 | 0.861712 | `resourcehealth_health-events_list` | ✅ **EXPECTED** | +| 2 | 0.835872 | `eventgrid_subscription_list` | ❌ | +| 3 | 0.832761 | `search_service_list` | ❌ | +| 4 | 0.831846 | `eventgrid_topic_list` | ❌ | +| 5 | 0.815611 | `kusto_cluster_list` | ❌ | --- ## Test 358 -**Expected Tool:** `datadog_monitoredresources_list` -**Prompt:** List all monitored resources in the Datadog resource +**Expected Tool:** `resourcehealth_health-events_list` +**Prompt:** Show me Azure service health events for subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.668828 | `datadog_monitoredresources_list` | ✅ **EXPECTED** | -| 2 | 0.454270 | `redis_list` | ❌ | -| 3 | 0.413661 | `loadtesting_testresource_list` | ❌ | -| 4 | 0.413173 | `monitor_metrics_query` | ❌ | -| 5 | 0.401731 | `grafana_list` | ❌ | +| 1 | 0.864582 | `resourcehealth_health-events_list` | ✅ **EXPECTED** | +| 2 | 0.844059 | `search_service_list` | ❌ | +| 3 | 0.832410 | `eventgrid_topic_list` | ❌ | +| 4 | 0.825710 | `eventgrid_subscription_list` | ❌ | +| 5 | 0.822477 | `resourcehealth_availability-status_list` | ❌ | --- ## Test 359 -**Expected Tool:** `datadog_monitoredresources_list` -**Prompt:** Show me the monitored resources in the Datadog resource +**Expected Tool:** `resourcehealth_health-events_list` +**Prompt:** What service issues have occurred in the last 30 days? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.624066 | `datadog_monitoredresources_list` | ✅ **EXPECTED** | -| 2 | 0.443481 | `monitor_metrics_query` | ❌ | -| 3 | 0.440052 | `redis_list` | ❌ | -| 4 | 0.424391 | `monitor_resource_log_query` | ❌ | -| 5 | 0.385122 | `loadtesting_testresource_list` | ❌ | +| 1 | 0.791286 | `resourcehealth_health-events_list` | ✅ **EXPECTED** | +| 2 | 0.759813 | `applens_resource_diagnose` | ❌ | +| 3 | 0.737538 | `deploy_app_logs_get` | ❌ | +| 4 | 0.729654 | `quota_usage_check` | ❌ | +| 5 | 0.728369 | `cloudarchitect_design` | ❌ | --- ## Test 360 -**Expected Tool:** `extension_azqr` -**Prompt:** Check my Azure subscription for any compliance issues or recommendations +**Expected Tool:** `resourcehealth_health-events_list` +**Prompt:** List active service health events in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.533164 | `quota_usage_check` | ❌ | -| 2 | 0.481143 | `azureterraformbestpractices_get` | ❌ | -| 3 | 0.476826 | `extension_azqr` | ✅ **EXPECTED** | -| 4 | 0.471559 | `subscription_list` | ❌ | -| 5 | 0.468404 | `applens_resource_diagnose` | ❌ | +| 1 | 0.833717 | `resourcehealth_health-events_list` | ✅ **EXPECTED** | +| 2 | 0.829280 | `eventgrid_subscription_list` | ❌ | +| 3 | 0.811795 | `eventgrid_topic_list` | ❌ | +| 4 | 0.802959 | `search_service_list` | ❌ | +| 5 | 0.774852 | `kusto_cluster_list` | ❌ | --- ## Test 361 -**Expected Tool:** `extension_azqr` -**Prompt:** Provide compliance recommendations for my current Azure subscription +**Expected Tool:** `resourcehealth_health-events_list` +**Prompt:** Show me planned maintenance events for my Azure services ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.532792 | `azureterraformbestpractices_get` | ❌ | -| 2 | 0.492863 | `get_bestpractices_get` | ❌ | -| 3 | 0.476164 | `applicationinsights_recommendation_list` | ❌ | -| 4 | 0.473365 | `deploy_iac_rules_get` | ❌ | -| 5 | 0.468194 | `azureaibestpractices_get` | ❌ | +| 1 | 0.835632 | `resourcehealth_health-events_list` | ✅ **EXPECTED** | +| 2 | 0.821663 | `quota_usage_check` | ❌ | +| 3 | 0.817334 | `search_service_list` | ❌ | +| 4 | 0.802165 | `applens_resource_diagnose` | ❌ | +| 5 | 0.800477 | `deploy_app_logs_get` | ❌ | --- ## Test 362 -**Expected Tool:** `extension_azqr` -**Prompt:** Scan my Azure subscription for compliance recommendations +**Expected Tool:** `servicebus_queue_details` +**Prompt:** Show me the details of service bus queue ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.536934 | `azureterraformbestpractices_get` | ❌ | -| 2 | 0.516925 | `extension_azqr` | ✅ **EXPECTED** | -| 3 | 0.514978 | `applicationinsights_recommendation_list` | ❌ | -| 4 | 0.504673 | `quota_usage_check` | ❌ | -| 5 | 0.494872 | `deploy_plan_get` | ❌ | +| 1 | 0.831616 | `servicebus_queue_details` | ✅ **EXPECTED** | +| 2 | 0.814519 | `servicebus_topic_details` | ❌ | +| 3 | 0.785992 | `servicebus_topic_subscription_details` | ❌ | +| 4 | 0.783941 | `redis_list` | ❌ | +| 5 | 0.782057 | `sql_server_show` | ❌ | --- ## Test 363 -**Expected Tool:** `quota_region_availability_list` -**Prompt:** Show me the available regions for these resource types +**Expected Tool:** `servicebus_topic_details` +**Prompt:** Show me the details of service bus topic ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.590878 | `quota_region_availability_list` | ✅ **EXPECTED** | -| 2 | 0.413274 | `quota_usage_check` | ❌ | -| 3 | 0.391332 | `redis_list` | ❌ | -| 4 | 0.372940 | `resourcehealth_availability-status_list` | ❌ | -| 5 | 0.369855 | `managedlustre_fs_sku_get` | ❌ | +| 1 | 0.851454 | `servicebus_topic_details` | ✅ **EXPECTED** | +| 2 | 0.806562 | `servicebus_topic_subscription_details` | ❌ | +| 3 | 0.795960 | `eventgrid_topic_list` | ❌ | +| 4 | 0.790087 | `eventgrid_subscription_list` | ❌ | +| 5 | 0.783687 | `redis_list` | ❌ | --- ## Test 364 -**Expected Tool:** `quota_usage_check` -**Prompt:** Check usage information for in region +**Expected Tool:** `servicebus_topic_subscription_details` +**Prompt:** Show me the details of service bus subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.609244 | `quota_usage_check` | ✅ **EXPECTED** | -| 2 | 0.491058 | `quota_region_availability_list` | ❌ | -| 3 | 0.384350 | `resourcehealth_availability-status_list` | ❌ | -| 4 | 0.376368 | `resourcehealth_availability-status_get` | ❌ | -| 5 | 0.371407 | `redis_list` | ❌ | +| 1 | 0.840195 | `servicebus_topic_subscription_details` | ✅ **EXPECTED** | +| 2 | 0.834069 | `servicebus_topic_details` | ❌ | +| 3 | 0.826504 | `redis_list` | ❌ | +| 4 | 0.823696 | `kusto_cluster_get` | ❌ | +| 5 | 0.815687 | `search_service_list` | ❌ | --- ## Test 365 -**Expected Tool:** `role_assignment_list` -**Prompt:** List all available role assignments in my subscription +**Expected Tool:** `signalr_runtime_get` +**Prompt:** Show me the details of SignalR ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.645258 | `role_assignment_list` | ✅ **EXPECTED** | -| 2 | 0.539843 | `subscription_list` | ❌ | -| 3 | 0.483988 | `group_list` | ❌ | -| 4 | 0.478700 | `grafana_list` | ❌ | -| 5 | 0.471364 | `cosmos_account_list` | ❌ | +| 1 | 0.809383 | `signalr_runtime_get` | ✅ **EXPECTED** | +| 2 | 0.772240 | `eventhubs_eventhub_get` | ❌ | +| 3 | 0.767936 | `sql_server_show` | ❌ | +| 4 | 0.762918 | `eventgrid_subscription_list` | ❌ | +| 5 | 0.761571 | `redis_list` | ❌ | --- ## Test 366 -**Expected Tool:** `role_assignment_list` -**Prompt:** Show me the available role assignments in my subscription +**Expected Tool:** `signalr_runtime_get` +**Prompt:** Show me the network information of SignalR runtime ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.609704 | `role_assignment_list` | ✅ **EXPECTED** | -| 2 | 0.514778 | `subscription_list` | ❌ | -| 3 | 0.456956 | `grafana_list` | ❌ | -| 4 | 0.449210 | `eventgrid_subscription_list` | ❌ | -| 5 | 0.445149 | `redis_list` | ❌ | +| 1 | 0.839342 | `signalr_runtime_get` | ✅ **EXPECTED** | +| 2 | 0.767355 | `sql_server_show` | ❌ | +| 3 | 0.746593 | `eventhubs_eventhub_get` | ❌ | +| 4 | 0.743524 | `servicebus_topic_details` | ❌ | +| 5 | 0.738830 | `foundry_resource_get` | ❌ | --- ## Test 367 -**Expected Tool:** `redis_create` -**Prompt:** Create a new Redis resource named with SKU in resource group +**Expected Tool:** `signalr_runtime_get` +**Prompt:** Describe the SignalR runtime in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.491740 | `redis_list` | ❌ | -| 2 | 0.489419 | `storage_account_create` | ❌ | -| 3 | 0.457104 | `workbooks_create` | ❌ | -| 4 | 0.440737 | `eventhubs_namespace_update` | ❌ | -| 5 | 0.409876 | `loadtesting_testresource_create` | ❌ | +| 1 | 0.851664 | `signalr_runtime_get` | ✅ **EXPECTED** | +| 2 | 0.771807 | `redis_create` | ❌ | +| 3 | 0.769196 | `foundry_models_deploy` | ❌ | +| 4 | 0.768974 | `loadtesting_testresource_create` | ❌ | +| 5 | 0.766785 | `eventhubs_namespace_get` | ❌ | --- ## Test 368 -**Expected Tool:** `redis_create` -**Prompt:** Create a new Redis resource for me +**Expected Tool:** `signalr_runtime_get` +**Prompt:** Get information about my SignalR runtime in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.479139 | `redis_list` | ❌ | -| 2 | 0.374539 | `loadtesting_testresource_create` | ❌ | -| 3 | 0.318545 | `loadtesting_testrun_create` | ❌ | -| 4 | 0.287562 | `workbooks_create` | ❌ | -| 5 | 0.286260 | `sql_db_create` | ❌ | +| 1 | 0.877625 | `signalr_runtime_get` | ✅ **EXPECTED** | +| 2 | 0.780888 | `foundry_resource_get` | ❌ | +| 3 | 0.778225 | `eventhubs_eventhub_get` | ❌ | +| 4 | 0.777934 | `quota_usage_check` | ❌ | +| 5 | 0.776913 | `servicebus_topic_details` | ❌ | --- ## Test 369 -**Expected Tool:** `redis_create` -**Prompt:** Create a Redis cache named with SKU in resource group +**Expected Tool:** `signalr_runtime_get` +**Prompt:** Show all the SignalRs information in ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.475862 | `storage_account_create` | ❌ | -| 2 | 0.464155 | `redis_list` | ❌ | -| 3 | 0.419043 | `eventhubs_namespace_update` | ❌ | -| 4 | 0.408170 | `workbooks_create` | ❌ | -| 5 | 0.372512 | `sql_db_create` | ❌ | +| 1 | 0.801139 | `signalr_runtime_get` | ✅ **EXPECTED** | +| 2 | 0.788870 | `redis_list` | ❌ | +| 3 | 0.785706 | `group_list` | ❌ | +| 4 | 0.782545 | `resourcehealth_availability-status_list` | ❌ | +| 5 | 0.773956 | `loadtesting_testresource_list` | ❌ | --- ## Test 370 -**Expected Tool:** `redis_create` -**Prompt:** Create a new Redis cluster with name , SKU +**Expected Tool:** `signalr_runtime_get` +**Prompt:** List all SignalRs in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.425855 | `redis_list` | ❌ | -| 2 | 0.401168 | `kusto_cluster_get` | ❌ | -| 3 | 0.377350 | `eventhubs_namespace_update` | ❌ | -| 4 | 0.361883 | `storage_account_create` | ❌ | -| 5 | 0.348511 | `kusto_cluster_list` | ❌ | +| 1 | 0.794619 | `eventgrid_subscription_list` | ❌ | +| 2 | 0.794190 | `redis_list` | ❌ | +| 3 | 0.792777 | `search_service_list` | ❌ | +| 4 | 0.792107 | `eventgrid_topic_list` | ❌ | +| 5 | 0.777033 | `signalr_runtime_get` | ✅ **EXPECTED** | --- ## Test 371 -**Expected Tool:** `redis_list` -**Prompt:** List all Redis resources in my subscription +**Expected Tool:** `sql_db_create` +**Prompt:** Create a new SQL database named in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.810504 | `redis_list` | ✅ **EXPECTED** | -| 2 | 0.587836 | `grafana_list` | ❌ | -| 3 | 0.512954 | `kusto_cluster_list` | ❌ | -| 4 | 0.508532 | `datadog_monitoredresources_list` | ❌ | -| 5 | 0.501218 | `postgres_server_list` | ❌ | +| 1 | 0.791527 | `sql_db_create` | ✅ **EXPECTED** | +| 2 | 0.791206 | `postgres_database_list` | ❌ | +| 3 | 0.781417 | `mysql_table_list` | ❌ | +| 4 | 0.770893 | `mysql_database_list` | ❌ | +| 5 | 0.769424 | `sql_db_rename` | ❌ | --- ## Test 372 -**Expected Tool:** `redis_list` -**Prompt:** Show me my Redis resources +**Expected Tool:** `sql_db_create` +**Prompt:** Create a SQL database with Basic tier in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.685128 | `redis_list` | ✅ **EXPECTED** | -| 2 | 0.374327 | `grafana_list` | ❌ | -| 3 | 0.364197 | `datadog_monitoredresources_list` | ❌ | -| 4 | 0.359659 | `mysql_server_list` | ❌ | -| 5 | 0.331502 | `mysql_database_list` | ❌ | +| 1 | 0.808905 | `sql_db_create` | ✅ **EXPECTED** | +| 2 | 0.778982 | `mysql_table_list` | ❌ | +| 3 | 0.775333 | `mysql_database_query` | ❌ | +| 4 | 0.774103 | `postgres_database_list` | ❌ | +| 5 | 0.771927 | `sql_server_show` | ❌ | --- ## Test 373 -**Expected Tool:** `redis_list` -**Prompt:** Show me the Redis resources in my subscription +**Expected Tool:** `sql_db_create` +**Prompt:** Create a new database called on SQL server in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.781228 | `redis_list` | ✅ **EXPECTED** | -| 2 | 0.539177 | `grafana_list` | ❌ | -| 3 | 0.449276 | `datadog_monitoredresources_list` | ❌ | -| 4 | 0.449014 | `postgres_server_list` | ❌ | -| 5 | 0.442854 | `kusto_cluster_list` | ❌ | +| 1 | 0.823521 | `sql_server_create` | ❌ | +| 2 | 0.817751 | `sql_db_create` | ✅ **EXPECTED** | +| 3 | 0.802433 | `sql_db_rename` | ❌ | +| 4 | 0.798566 | `mysql_server_list` | ❌ | +| 5 | 0.797166 | `appservice_database_add` | ❌ | --- ## Test 374 -**Expected Tool:** `redis_list` -**Prompt:** Show me my Redis caches +**Expected Tool:** `sql_db_delete` +**Prompt:** Delete the SQL database from server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.572767 | `redis_list` | ✅ **EXPECTED** | -| 2 | 0.316630 | `mysql_database_list` | ❌ | -| 3 | 0.301786 | `postgres_database_list` | ❌ | -| 4 | 0.286513 | `mysql_server_list` | ❌ | -| 5 | 0.273014 | `kusto_cluster_list` | ❌ | +| 1 | 0.826648 | `sql_server_delete` | ❌ | +| 2 | 0.824912 | `sql_db_delete` | ✅ **EXPECTED** | +| 3 | 0.767749 | `postgres_database_list` | ❌ | +| 4 | 0.760572 | `mysql_table_list` | ❌ | +| 5 | 0.752778 | `mysql_database_query` | ❌ | --- ## Test 375 -**Expected Tool:** `redis_list` -**Prompt:** Get Redis clusters +**Expected Tool:** `sql_db_delete` +**Prompt:** Remove database from SQL server in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.478070 | `redis_list` | ✅ **EXPECTED** | -| 2 | 0.456308 | `kusto_cluster_list` | ❌ | -| 3 | 0.383821 | `kusto_cluster_get` | ❌ | -| 4 | 0.359479 | `kusto_database_list` | ❌ | -| 5 | 0.343305 | `aks_cluster_get` | ❌ | +| 1 | 0.838652 | `sql_server_delete` | ❌ | +| 2 | 0.832480 | `sql_db_delete` | ✅ **EXPECTED** | +| 3 | 0.785882 | `sql_db_rename` | ❌ | +| 4 | 0.772417 | `mysql_server_list` | ❌ | +| 5 | 0.771258 | `appservice_database_add` | ❌ | --- ## Test 376 -**Expected Tool:** `group_list` -**Prompt:** List all resource groups in my subscription +**Expected Tool:** `sql_db_delete` +**Prompt:** Delete the database called on server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.755935 | `group_list` | ✅ **EXPECTED** | -| 2 | 0.566552 | `workbooks_list` | ❌ | -| 3 | 0.564566 | `loadtesting_testresource_list` | ❌ | -| 4 | 0.552633 | `datadog_monitoredresources_list` | ❌ | -| 5 | 0.549477 | `monitor_webtests_list` | ❌ | +| 1 | 0.823651 | `sql_db_delete` | ✅ **EXPECTED** | +| 2 | 0.819672 | `sql_server_delete` | ❌ | +| 3 | 0.778153 | `postgres_database_list` | ❌ | +| 4 | 0.766154 | `mysql_table_list` | ❌ | +| 5 | 0.756942 | `mysql_database_list` | ❌ | --- ## Test 377 -**Expected Tool:** `group_list` -**Prompt:** Show me my resource groups +**Expected Tool:** `sql_db_list` +**Prompt:** List all databases in the Azure SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.529504 | `group_list` | ✅ **EXPECTED** | -| 2 | 0.464690 | `redis_list` | ❌ | -| 3 | 0.463685 | `datadog_monitoredresources_list` | ❌ | -| 4 | 0.462391 | `mysql_server_list` | ❌ | -| 5 | 0.460280 | `loadtesting_testresource_list` | ❌ | +| 1 | 0.872949 | `mysql_database_list` | ❌ | +| 2 | 0.869679 | `postgres_database_list` | ❌ | +| 3 | 0.852748 | `kusto_database_list` | ❌ | +| 4 | 0.850883 | `mysql_table_list` | ❌ | +| 5 | 0.848527 | `kusto_table_list` | ❌ | --- ## Test 378 -**Expected Tool:** `group_list` -**Prompt:** Show me the resource groups in my subscription +**Expected Tool:** `sql_db_list` +**Prompt:** Show me all the databases configuration details in the Azure SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.665772 | `group_list` | ✅ **EXPECTED** | -| 2 | 0.532656 | `datadog_monitoredresources_list` | ❌ | -| 3 | 0.532505 | `redis_list` | ❌ | -| 4 | 0.532054 | `eventgrid_topic_list` | ❌ | -| 5 | 0.531920 | `resourcehealth_availability-status_list` | ❌ | +| 1 | 0.861464 | `sql_server_show` | ❌ | +| 2 | 0.853800 | `mysql_database_list` | ❌ | +| 3 | 0.853001 | `mysql_server_config_get` | ❌ | +| 4 | 0.845998 | `sql_db_list` | ✅ **EXPECTED** | +| 5 | 0.833247 | `postgres_database_list` | ❌ | --- ## Test 379 -**Expected Tool:** `resourcehealth_availability-status_get` -**Prompt:** Get the availability status for resource +**Expected Tool:** `sql_db_rename` +**Prompt:** Rename the SQL database on server to ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.556629 | `resourcehealth_availability-status_get` | ✅ **EXPECTED** | -| 2 | 0.538273 | `resourcehealth_availability-status_list` | ❌ | -| 3 | 0.377586 | `quota_usage_check` | ❌ | -| 4 | 0.373593 | `monitor_healthmodels_entity_get` | ❌ | -| 5 | 0.349981 | `datadog_monitoredresources_list` | ❌ | +| 1 | 0.820644 | `sql_db_rename` | ✅ **EXPECTED** | +| 2 | 0.769441 | `sql_server_delete` | ❌ | +| 3 | 0.766580 | `sql_db_delete` | ❌ | +| 4 | 0.765966 | `mysql_table_list` | ❌ | +| 5 | 0.765664 | `postgres_database_list` | ❌ | --- ## Test 380 -**Expected Tool:** `resourcehealth_availability-status_get` -**Prompt:** Show me the health status of the storage account +**Expected Tool:** `sql_db_rename` +**Prompt:** Rename my Azure SQL database to on server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.576591 | `storage_account_get` | ❌ | -| 2 | 0.564128 | `resourcehealth_availability-status_get` | ✅ **EXPECTED** | -| 3 | 0.556167 | `storage_blob_container_get` | ❌ | -| 4 | 0.487207 | `storage_blob_get` | ❌ | -| 5 | 0.466885 | `resourcehealth_availability-status_list` | ❌ | +| 1 | 0.867572 | `sql_db_rename` | ✅ **EXPECTED** | +| 2 | 0.818728 | `sql_db_delete` | ❌ | +| 3 | 0.816661 | `sql_server_delete` | ❌ | +| 4 | 0.808741 | `sql_db_create` | ❌ | +| 5 | 0.794511 | `mysql_table_list` | ❌ | --- ## Test 381 -**Expected Tool:** `resourcehealth_availability-status_get` -**Prompt:** What is the availability status of virtual machine in resource group ? +**Expected Tool:** `sql_db_show` +**Prompt:** Get the configuration details for the SQL database on server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.577398 | `resourcehealth_availability-status_list` | ❌ | -| 2 | 0.501568 | `resourcehealth_availability-status_get` | ✅ **EXPECTED** | -| 3 | 0.424939 | `mysql_server_list` | ❌ | -| 4 | 0.412025 | `loadtesting_testresource_list` | ❌ | -| 5 | 0.393479 | `managedlustre_fs_list` | ❌ | +| 1 | 0.841959 | `sql_server_show` | ❌ | +| 2 | 0.838014 | `postgres_server_config_get` | ❌ | +| 3 | 0.833105 | `mysql_server_config_get` | ❌ | +| 4 | 0.800194 | `sql_db_show` | ✅ **EXPECTED** | +| 5 | 0.798840 | `mysql_server_param_get` | ❌ | --- ## Test 382 -**Expected Tool:** `resourcehealth_availability-status_list` -**Prompt:** List availability status for all resources in my subscription +**Expected Tool:** `sql_db_show` +**Prompt:** Show me the details of SQL database in server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.737219 | `resourcehealth_availability-status_list` | ✅ **EXPECTED** | -| 2 | 0.585501 | `redis_list` | ❌ | -| 3 | 0.549914 | `loadtesting_testresource_list` | ❌ | -| 4 | 0.548549 | `grafana_list` | ❌ | -| 5 | 0.544501 | `subscription_list` | ❌ | +| 1 | 0.824731 | `postgres_database_list` | ❌ | +| 2 | 0.823927 | `sql_server_show` | ❌ | +| 3 | 0.818905 | `mysql_table_schema_get` | ❌ | +| 4 | 0.810948 | `mysql_table_list` | ❌ | +| 5 | 0.808168 | `postgres_table_list` | ❌ | --- ## Test 383 -**Expected Tool:** `resourcehealth_availability-status_list` -**Prompt:** Show me the health status of all my Azure resources +**Expected Tool:** `sql_db_update` +**Prompt:** Update the performance tier of SQL database on server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.644982 | `resourcehealth_availability-status_list` | ✅ **EXPECTED** | -| 2 | 0.545208 | `resourcehealth_availability-status_get` | ❌ | -| 3 | 0.509740 | `resourcehealth_health-events_list` | ❌ | -| 4 | 0.508252 | `quota_usage_check` | ❌ | -| 5 | 0.505776 | `redis_list` | ❌ | +| 1 | 0.818568 | `sql_db_update` | ✅ **EXPECTED** | +| 2 | 0.791576 | `sql_db_create` | ❌ | +| 3 | 0.767192 | `mysql_server_param_set` | ❌ | +| 4 | 0.753117 | `sql_db_rename` | ❌ | +| 5 | 0.749712 | `sql_server_show` | ❌ | --- ## Test 384 -**Expected Tool:** `resourcehealth_availability-status_list` -**Prompt:** What resources in resource group have health issues? +**Expected Tool:** `sql_db_update` +**Prompt:** Scale SQL database on server to use SKU ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.596890 | `resourcehealth_availability-status_list` | ✅ **EXPECTED** | -| 2 | 0.549900 | `resourcehealth_availability-status_get` | ❌ | -| 3 | 0.496640 | `resourcehealth_health-events_list` | ❌ | -| 4 | 0.441921 | `applens_resource_diagnose` | ❌ | -| 5 | 0.433614 | `loadtesting_testresource_list` | ❌ | +| 1 | 0.801331 | `sql_db_update` | ✅ **EXPECTED** | +| 2 | 0.759453 | `mysql_database_query` | ❌ | +| 3 | 0.757619 | `sql_server_delete` | ❌ | +| 4 | 0.756779 | `mysql_table_list` | ❌ | +| 5 | 0.753532 | `kusto_table_schema` | ❌ | --- ## Test 385 -**Expected Tool:** `resourcehealth_health-events_list` -**Prompt:** List all service health events in my subscription +**Expected Tool:** `sql_elastic-pool_list` +**Prompt:** List all elastic pools in SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.690720 | `resourcehealth_health-events_list` | ✅ **EXPECTED** | -| 2 | 0.554895 | `search_service_list` | ❌ | -| 3 | 0.534251 | `eventgrid_topic_list` | ❌ | -| 4 | 0.529761 | `eventgrid_subscription_list` | ❌ | -| 5 | 0.518372 | `resourcehealth_availability-status_list` | ❌ | +| 1 | 0.855533 | `sql_elastic-pool_list` | ✅ **EXPECTED** | +| 2 | 0.796709 | `aks_nodepool_get` | ❌ | +| 3 | 0.785831 | `kusto_cluster_list` | ❌ | +| 4 | 0.785251 | `mysql_database_list` | ❌ | +| 5 | 0.783243 | `mysql_table_list` | ❌ | --- ## Test 386 -**Expected Tool:** `resourcehealth_health-events_list` -**Prompt:** Show me Azure service health events for subscription +**Expected Tool:** `sql_elastic-pool_list` +**Prompt:** Show me the elastic pools configured for SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.686421 | `resourcehealth_health-events_list` | ✅ **EXPECTED** | -| 2 | 0.534614 | `eventgrid_subscription_list` | ❌ | -| 3 | 0.513821 | `search_service_list` | ❌ | -| 4 | 0.513254 | `eventgrid_topic_list` | ❌ | -| 5 | 0.501151 | `subscription_list` | ❌ | +| 1 | 0.829336 | `sql_elastic-pool_list` | ✅ **EXPECTED** | +| 2 | 0.780002 | `aks_nodepool_get` | ❌ | +| 3 | 0.773168 | `sql_server_show` | ❌ | +| 4 | 0.771375 | `mysql_server_config_get` | ❌ | +| 5 | 0.766098 | `mysql_database_list` | ❌ | --- ## Test 387 -**Expected Tool:** `resourcehealth_health-events_list` -**Prompt:** What service issues have occurred in the last 30 days? +**Expected Tool:** `sql_elastic-pool_list` +**Prompt:** What elastic pools are available in my SQL server ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.450841 | `resourcehealth_health-events_list` | ✅ **EXPECTED** | -| 2 | 0.267663 | `applens_resource_diagnose` | ❌ | -| 3 | 0.245720 | `cloudarchitect_design` | ❌ | -| 4 | 0.216847 | `resourcehealth_availability-status_list` | ❌ | -| 5 | 0.211842 | `search_service_list` | ❌ | +| 1 | 0.820544 | `sql_elastic-pool_list` | ✅ **EXPECTED** | +| 2 | 0.779960 | `mysql_server_list` | ❌ | +| 3 | 0.779487 | `aks_nodepool_get` | ❌ | +| 4 | 0.778977 | `mysql_database_list` | ❌ | +| 5 | 0.769456 | `mysql_table_list` | ❌ | --- ## Test 388 -**Expected Tool:** `resourcehealth_health-events_list` -**Prompt:** List active service health events in my subscription +**Expected Tool:** `sql_server_create` +**Prompt:** Create a new Azure SQL server named in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.685391 | `resourcehealth_health-events_list` | ✅ **EXPECTED** | -| 2 | 0.527905 | `eventgrid_subscription_list` | ❌ | -| 3 | 0.524063 | `eventgrid_topic_list` | ❌ | -| 4 | 0.520197 | `search_service_list` | ❌ | -| 5 | 0.502064 | `resourcehealth_availability-status_list` | ❌ | +| 1 | 0.864677 | `sql_server_create` | ✅ **EXPECTED** | +| 2 | 0.819256 | `redis_create` | ❌ | +| 3 | 0.811178 | `sql_db_create` | ❌ | +| 4 | 0.809783 | `mysql_server_list` | ❌ | +| 5 | 0.809756 | `storage_account_create` | ❌ | --- ## Test 389 -**Expected Tool:** `resourcehealth_health-events_list` -**Prompt:** Show me planned maintenance events for my Azure services +**Expected Tool:** `sql_server_create` +**Prompt:** Create an Azure SQL server with name in location with admin user ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.565851 | `resourcehealth_health-events_list` | ✅ **EXPECTED** | -| 2 | 0.437868 | `search_service_list` | ❌ | -| 3 | 0.403665 | `eventgrid_subscription_list` | ❌ | -| 4 | 0.402493 | `resourcehealth_availability-status_list` | ❌ | -| 5 | 0.397735 | `quota_usage_check` | ❌ | +| 1 | 0.862810 | `sql_server_create` | ✅ **EXPECTED** | +| 2 | 0.809713 | `sql_server_show` | ❌ | +| 3 | 0.799231 | `sql_db_create` | ❌ | +| 4 | 0.791963 | `storage_account_create` | ❌ | +| 5 | 0.786847 | `sql_server_delete` | ❌ | --- ## Test 390 -**Expected Tool:** `servicebus_queue_details` -**Prompt:** Show me the details of service bus queue +**Expected Tool:** `sql_server_create` +**Prompt:** Set up a new SQL server called in my resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.642997 | `servicebus_queue_details` | ✅ **EXPECTED** | -| 2 | 0.460919 | `servicebus_topic_subscription_details` | ❌ | -| 3 | 0.436910 | `servicebus_topic_details` | ❌ | -| 4 | 0.385762 | `search_knowledge_base_get` | ❌ | -| 5 | 0.384182 | `storage_account_get` | ❌ | +| 1 | 0.833383 | `sql_server_create` | ✅ **EXPECTED** | +| 2 | 0.795420 | `mysql_server_list` | ❌ | +| 3 | 0.791985 | `redis_create` | ❌ | +| 4 | 0.788993 | `sql_server_delete` | ❌ | +| 5 | 0.788158 | `sql_server_show` | ❌ | --- ## Test 391 -**Expected Tool:** `servicebus_topic_details` -**Prompt:** Show me the details of service bus topic +**Expected Tool:** `sql_server_delete` +**Prompt:** Delete the Azure SQL server from resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.642952 | `servicebus_topic_details` | ✅ **EXPECTED** | -| 2 | 0.571860 | `servicebus_topic_subscription_details` | ❌ | -| 3 | 0.483976 | `servicebus_queue_details` | ❌ | -| 4 | 0.482958 | `eventgrid_topic_list` | ❌ | -| 5 | 0.458712 | `eventgrid_subscription_list` | ❌ | +| 1 | 0.859275 | `sql_server_delete` | ✅ **EXPECTED** | +| 2 | 0.831949 | `sql_db_delete` | ❌ | +| 3 | 0.793028 | `sql_server_create` | ❌ | +| 4 | 0.784608 | `workbooks_delete` | ❌ | +| 5 | 0.782799 | `sql_server_show` | ❌ | --- ## Test 392 -**Expected Tool:** `servicebus_topic_subscription_details` -**Prompt:** Show me the details of service bus subscription +**Expected Tool:** `sql_server_delete` +**Prompt:** Remove the SQL server from my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.633187 | `servicebus_topic_subscription_details` | ✅ **EXPECTED** | -| 2 | 0.517623 | `servicebus_topic_details` | ❌ | -| 3 | 0.494515 | `servicebus_queue_details` | ❌ | -| 4 | 0.493853 | `eventgrid_topic_list` | ❌ | -| 5 | 0.472128 | `eventgrid_subscription_list` | ❌ | +| 1 | 0.852916 | `sql_server_delete` | ✅ **EXPECTED** | +| 2 | 0.783521 | `postgres_server_list` | ❌ | +| 3 | 0.777659 | `sql_db_delete` | ❌ | +| 4 | 0.752033 | `sql_server_show` | ❌ | +| 5 | 0.747077 | `kusto_cluster_list` | ❌ | --- ## Test 393 -**Expected Tool:** `signalr_runtime_get` -**Prompt:** Show me the details of SignalR +**Expected Tool:** `sql_server_delete` +**Prompt:** Delete SQL server permanently ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.532544 | `signalr_runtime_get` | ✅ **EXPECTED** | -| 2 | 0.355028 | `redis_list` | ❌ | -| 3 | 0.329804 | `foundry_resource_get` | ❌ | -| 4 | 0.319981 | `sql_server_show` | ❌ | -| 5 | 0.304420 | `servicebus_queue_details` | ❌ | +| 1 | 0.851952 | `sql_server_delete` | ✅ **EXPECTED** | +| 2 | 0.797903 | `sql_db_delete` | ❌ | +| 3 | 0.757697 | `sql_server_show` | ❌ | +| 4 | 0.750805 | `mysql_database_query` | ❌ | +| 5 | 0.737303 | `sql_server_firewall-rule_delete` | ❌ | --- ## Test 394 -**Expected Tool:** `signalr_runtime_get` -**Prompt:** Show me the network information of SignalR runtime +**Expected Tool:** `sql_server_entra-admin_list` +**Prompt:** List Microsoft Entra ID administrators for SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.573446 | `signalr_runtime_get` | ✅ **EXPECTED** | -| 2 | 0.337342 | `sql_server_show` | ❌ | -| 3 | 0.306559 | `foundry_resource_get` | ❌ | -| 4 | 0.305021 | `redis_list` | ❌ | -| 5 | 0.300956 | `servicebus_topic_details` | ❌ | +| 1 | 0.898488 | `sql_server_entra-admin_list` | ✅ **EXPECTED** | +| 2 | 0.778346 | `sql_server_show` | ❌ | +| 3 | 0.747307 | `foundry_threads_list` | ❌ | +| 4 | 0.735783 | `mysql_table_list` | ❌ | +| 5 | 0.735595 | `sql_server_list` | ❌ | --- ## Test 395 -**Expected Tool:** `signalr_runtime_get` -**Prompt:** Describe the SignalR runtime in resource group +**Expected Tool:** `sql_server_entra-admin_list` +**Prompt:** Show me the Entra ID administrators configured for SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.710353 | `signalr_runtime_get` | ✅ **EXPECTED** | -| 2 | 0.411396 | `loadtesting_testresource_list` | ❌ | -| 3 | 0.410606 | `foundry_resource_get` | ❌ | -| 4 | 0.399412 | `resourcehealth_availability-status_list` | ❌ | -| 5 | 0.382028 | `sql_server_list` | ❌ | +| 1 | 0.865818 | `sql_server_entra-admin_list` | ✅ **EXPECTED** | +| 2 | 0.776424 | `sql_server_show` | ❌ | +| 3 | 0.745369 | `postgres_server_config_get` | ❌ | +| 4 | 0.742073 | `mysql_server_config_get` | ❌ | +| 5 | 0.737477 | `mysql_table_list` | ❌ | --- ## Test 396 -**Expected Tool:** `signalr_runtime_get` -**Prompt:** Get information about my SignalR runtime in +**Expected Tool:** `sql_server_entra-admin_list` +**Prompt:** What Microsoft Entra ID administrators are set up for my SQL server ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.715974 | `signalr_runtime_get` | ✅ **EXPECTED** | -| 2 | 0.459046 | `foundry_resource_get` | ❌ | -| 3 | 0.430829 | `resourcehealth_availability-status_list` | ❌ | -| 4 | 0.430765 | `loadtesting_testresource_list` | ❌ | -| 5 | 0.417032 | `functionapp_get` | ❌ | +| 1 | 0.854495 | `sql_server_entra-admin_list` | ✅ **EXPECTED** | +| 2 | 0.797624 | `sql_server_show` | ❌ | +| 3 | 0.755084 | `mysql_table_schema_get` | ❌ | +| 4 | 0.752474 | `mysql_database_query` | ❌ | +| 5 | 0.750405 | `cloudarchitect_design` | ❌ | --- ## Test 397 -**Expected Tool:** `signalr_runtime_get` -**Prompt:** Show all the SignalRs information in +**Expected Tool:** `sql_server_firewall-rule_create` +**Prompt:** Create a firewall rule for my Azure SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.564071 | `signalr_runtime_get` | ✅ **EXPECTED** | -| 2 | 0.501077 | `redis_list` | ❌ | -| 3 | 0.494478 | `resourcehealth_availability-status_list` | ❌ | -| 4 | 0.481428 | `loadtesting_testresource_list` | ❌ | -| 5 | 0.462090 | `mysql_server_list` | ❌ | +| 1 | 0.831247 | `sql_server_firewall-rule_create` | ✅ **EXPECTED** | +| 2 | 0.815312 | `sql_server_firewall-rule_delete` | ❌ | +| 3 | 0.804633 | `sql_server_firewall-rule_list` | ❌ | +| 4 | 0.797793 | `sql_server_show` | ❌ | +| 5 | 0.795087 | `mysql_database_query` | ❌ | --- ## Test 398 -**Expected Tool:** `signalr_runtime_get` -**Prompt:** List all SignalRs in my subscription +**Expected Tool:** `sql_server_firewall-rule_create` +**Prompt:** Add a firewall rule to allow access from IP range to for SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.530646 | `signalr_runtime_get` | ✅ **EXPECTED** | -| 2 | 0.507654 | `postgres_server_list` | ❌ | -| 3 | 0.495157 | `redis_list` | ❌ | -| 4 | 0.494498 | `kusto_cluster_list` | ❌ | -| 5 | 0.487907 | `subscription_list` | ❌ | +| 1 | 0.876201 | `sql_server_firewall-rule_create` | ✅ **EXPECTED** | +| 2 | 0.802184 | `sql_server_firewall-rule_list` | ❌ | +| 3 | 0.792933 | `sql_server_firewall-rule_delete` | ❌ | +| 4 | 0.738478 | `mysql_database_query` | ❌ | +| 5 | 0.722482 | `postgres_database_query` | ❌ | --- ## Test 399 -**Expected Tool:** `sql_db_create` -**Prompt:** Create a new SQL database named in server +**Expected Tool:** `sql_server_firewall-rule_create` +**Prompt:** Create a new firewall rule named for SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.516780 | `sql_db_create` | ✅ **EXPECTED** | -| 2 | 0.470892 | `sql_server_create` | ❌ | -| 3 | 0.420504 | `sql_db_rename` | ❌ | -| 4 | 0.408515 | `sql_db_delete` | ❌ | -| 5 | 0.404860 | `sql_server_delete` | ❌ | +| 1 | 0.831918 | `sql_server_firewall-rule_create` | ✅ **EXPECTED** | +| 2 | 0.814691 | `sql_server_firewall-rule_delete` | ❌ | +| 3 | 0.806035 | `sql_server_firewall-rule_list` | ❌ | +| 4 | 0.756436 | `mysql_database_query` | ❌ | +| 5 | 0.746850 | `sql_server_create` | ❌ | --- ## Test 400 -**Expected Tool:** `sql_db_create` -**Prompt:** Create a SQL database with Basic tier in server +**Expected Tool:** `sql_server_firewall-rule_delete` +**Prompt:** Delete a firewall rule from my Azure SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.571760 | `sql_db_create` | ✅ **EXPECTED** | -| 2 | 0.459672 | `sql_server_create` | ❌ | -| 3 | 0.437525 | `sql_server_delete` | ❌ | -| 4 | 0.420843 | `sql_db_show` | ❌ | -| 5 | 0.417661 | `sql_db_delete` | ❌ | +| 1 | 0.876376 | `sql_server_firewall-rule_delete` | ✅ **EXPECTED** | +| 2 | 0.818141 | `sql_server_delete` | ❌ | +| 3 | 0.801269 | `sql_db_delete` | ❌ | +| 4 | 0.800536 | `sql_server_firewall-rule_list` | ❌ | +| 5 | 0.791225 | `sql_server_firewall-rule_create` | ❌ | --- ## Test 401 -**Expected Tool:** `sql_db_create` -**Prompt:** Create a new database called on SQL server in resource group +**Expected Tool:** `sql_server_firewall-rule_delete` +**Prompt:** Remove the firewall rule from SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.604362 | `sql_db_create` | ✅ **EXPECTED** | -| 2 | 0.545968 | `sql_server_create` | ❌ | -| 3 | 0.503884 | `sql_db_rename` | ❌ | -| 4 | 0.494323 | `sql_db_show` | ❌ | -| 5 | 0.473789 | `sql_db_list` | ❌ | +| 1 | 0.852353 | `sql_server_firewall-rule_delete` | ✅ **EXPECTED** | +| 2 | 0.784204 | `sql_server_firewall-rule_list` | ❌ | +| 3 | 0.782995 | `sql_server_delete` | ❌ | +| 4 | 0.777908 | `sql_server_firewall-rule_create` | ❌ | +| 5 | 0.748831 | `mysql_database_query` | ❌ | --- ## Test 402 -**Expected Tool:** `sql_db_delete` -**Prompt:** Delete the SQL database from server +**Expected Tool:** `sql_server_firewall-rule_delete` +**Prompt:** Delete firewall rule for SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.568196 | `sql_db_delete` | ✅ **EXPECTED** | -| 2 | 0.567412 | `sql_server_delete` | ❌ | -| 3 | 0.391509 | `sql_db_rename` | ❌ | -| 4 | 0.386564 | `sql_server_firewall-rule_delete` | ❌ | -| 5 | 0.364776 | `sql_db_show` | ❌ | +| 1 | 0.846622 | `sql_server_firewall-rule_delete` | ✅ **EXPECTED** | +| 2 | 0.787512 | `sql_server_delete` | ❌ | +| 3 | 0.786045 | `sql_server_firewall-rule_list` | ❌ | +| 4 | 0.771804 | `sql_server_firewall-rule_create` | ❌ | +| 5 | 0.761894 | `sql_db_delete` | ❌ | --- ## Test 403 -**Expected Tool:** `sql_db_delete` -**Prompt:** Remove database from SQL server in resource group +**Expected Tool:** `sql_server_firewall-rule_list` +**Prompt:** List all firewall rules for SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.567513 | `sql_server_delete` | ❌ | -| 2 | 0.543440 | `sql_db_delete` | ✅ **EXPECTED** | -| 3 | 0.500756 | `sql_db_show` | ❌ | -| 4 | 0.481083 | `sql_db_rename` | ❌ | -| 5 | 0.478729 | `sql_db_list` | ❌ | +| 1 | 0.872104 | `sql_server_firewall-rule_list` | ✅ **EXPECTED** | +| 2 | 0.819716 | `sql_server_firewall-rule_delete` | ❌ | +| 3 | 0.815955 | `sql_server_firewall-rule_create` | ❌ | +| 4 | 0.782384 | `mysql_table_list` | ❌ | +| 5 | 0.778905 | `sql_server_show` | ❌ | --- ## Test 404 -**Expected Tool:** `sql_db_delete` -**Prompt:** Delete the database called on server +**Expected Tool:** `sql_server_firewall-rule_list` +**Prompt:** Show me the firewall rules for SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.509916 | `sql_db_delete` | ✅ **EXPECTED** | -| 2 | 0.490893 | `sql_server_delete` | ❌ | -| 3 | 0.364494 | `postgres_database_list` | ❌ | -| 4 | 0.355416 | `mysql_database_list` | ❌ | -| 5 | 0.347837 | `sql_db_rename` | ❌ | +| 1 | 0.836402 | `sql_server_firewall-rule_list` | ✅ **EXPECTED** | +| 2 | 0.811356 | `sql_server_firewall-rule_create` | ❌ | +| 3 | 0.802883 | `sql_server_firewall-rule_delete` | ❌ | +| 4 | 0.787151 | `sql_server_show` | ❌ | +| 5 | 0.778071 | `mysql_database_query` | ❌ | --- ## Test 405 -**Expected Tool:** `sql_db_list` -**Prompt:** List all databases in the Azure SQL server +**Expected Tool:** `sql_server_firewall-rule_list` +**Prompt:** What firewall rules are configured for my SQL server ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.643186 | `sql_db_list` | ✅ **EXPECTED** | -| 2 | 0.639694 | `mysql_database_list` | ❌ | -| 3 | 0.609178 | `postgres_database_list` | ❌ | -| 4 | 0.602889 | `cosmos_database_list` | ❌ | -| 5 | 0.569950 | `kusto_database_list` | ❌ | +| 1 | 0.810427 | `sql_server_firewall-rule_list` | ✅ **EXPECTED** | +| 2 | 0.801505 | `sql_server_show` | ❌ | +| 3 | 0.796004 | `sql_server_firewall-rule_create` | ❌ | +| 4 | 0.783020 | `sql_server_firewall-rule_delete` | ❌ | +| 5 | 0.776939 | `mysql_database_query` | ❌ | --- ## Test 406 -**Expected Tool:** `sql_db_list` -**Prompt:** Show me all the databases configuration details in the Azure SQL server +**Expected Tool:** `sql_server_list` +**Prompt:** List all Azure SQL servers in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.617746 | `sql_server_show` | ❌ | -| 2 | 0.609322 | `sql_db_list` | ✅ **EXPECTED** | -| 3 | 0.557353 | `mysql_database_list` | ❌ | -| 4 | 0.553488 | `mysql_server_config_get` | ❌ | -| 5 | 0.524274 | `sql_db_show` | ❌ | +| 1 | 0.852366 | `sql_server_list` | ✅ **EXPECTED** | +| 2 | 0.849387 | `mysql_server_list` | ❌ | +| 3 | 0.828202 | `kusto_cluster_list` | ❌ | +| 4 | 0.826725 | `search_service_list` | ❌ | +| 5 | 0.811977 | `kusto_table_list` | ❌ | --- ## Test 407 -**Expected Tool:** `sql_db_rename` -**Prompt:** Rename the SQL database on server to +**Expected Tool:** `sql_server_list` +**Prompt:** Show me every SQL server available in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.593338 | `sql_db_rename` | ✅ **EXPECTED** | -| 2 | 0.425309 | `sql_server_delete` | ❌ | -| 3 | 0.416155 | `sql_db_delete` | ❌ | -| 4 | 0.396931 | `sql_db_create` | ❌ | -| 5 | 0.345953 | `sql_db_show` | ❌ | +| 1 | 0.833980 | `mysql_server_list` | ❌ | +| 2 | 0.809229 | `sql_server_list` | ✅ **EXPECTED** | +| 3 | 0.803620 | `kusto_cluster_list` | ❌ | +| 4 | 0.793686 | `kusto_database_list` | ❌ | +| 5 | 0.793317 | `redis_list` | ❌ | --- ## Test 408 -**Expected Tool:** `sql_db_rename` -**Prompt:** Rename my Azure SQL database to on server +**Expected Tool:** `sql_server_show` +**Prompt:** Show me the details of Azure SQL server in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.711063 | `sql_db_rename` | ✅ **EXPECTED** | -| 2 | 0.516485 | `sql_server_delete` | ❌ | -| 3 | 0.506499 | `sql_db_delete` | ❌ | -| 4 | 0.501476 | `sql_db_create` | ❌ | -| 5 | 0.433898 | `sql_server_show` | ❌ | +| 1 | 0.849971 | `sql_server_show` | ✅ **EXPECTED** | +| 2 | 0.833702 | `sql_db_show` | ❌ | +| 3 | 0.831025 | `mysql_server_list` | ❌ | +| 4 | 0.824924 | `sql_server_list` | ❌ | +| 5 | 0.817213 | `sql_server_create` | ❌ | --- ## Test 409 -**Expected Tool:** `sql_db_show` -**Prompt:** Get the configuration details for the SQL database on server +**Expected Tool:** `sql_server_show` +**Prompt:** Get the configuration details for SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.610991 | `sql_server_show` | ❌ | -| 2 | 0.593150 | `postgres_server_config_get` | ❌ | -| 3 | 0.530422 | `mysql_server_config_get` | ❌ | -| 4 | 0.528136 | `sql_db_show` | ✅ **EXPECTED** | -| 5 | 0.465693 | `sql_db_list` | ❌ | +| 1 | 0.864055 | `sql_server_show` | ✅ **EXPECTED** | +| 2 | 0.842651 | `postgres_server_config_get` | ❌ | +| 3 | 0.840563 | `mysql_server_config_get` | ❌ | +| 4 | 0.801763 | `mysql_server_param_get` | ❌ | +| 5 | 0.786885 | `sql_db_show` | ❌ | --- ## Test 410 -**Expected Tool:** `sql_db_show` -**Prompt:** Show me the details of SQL database in server +**Expected Tool:** `sql_server_show` +**Prompt:** Display the properties of SQL server ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.530095 | `sql_db_show` | ✅ **EXPECTED** | -| 2 | 0.503681 | `sql_server_show` | ❌ | -| 3 | 0.440073 | `sql_db_list` | ❌ | -| 4 | 0.438622 | `mysql_table_schema_get` | ❌ | -| 5 | 0.432919 | `mysql_database_list` | ❌ | +| 1 | 0.849149 | `sql_server_show` | ✅ **EXPECTED** | +| 2 | 0.798681 | `mysql_server_config_get` | ❌ | +| 3 | 0.789909 | `mysql_table_schema_get` | ❌ | +| 4 | 0.783363 | `postgres_database_list` | ❌ | +| 5 | 0.778605 | `mysql_table_list` | ❌ | --- ## Test 411 -**Expected Tool:** `sql_db_update` -**Prompt:** Update the performance tier of SQL database on server +**Expected Tool:** `storage_account_create` +**Prompt:** Create a new storage account called testaccount123 in East US region ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.603366 | `sql_db_update` | ✅ **EXPECTED** | -| 2 | 0.467571 | `sql_db_create` | ❌ | -| 3 | 0.440493 | `sql_db_rename` | ❌ | -| 4 | 0.427621 | `sql_db_show` | ❌ | -| 5 | 0.413941 | `sql_server_delete` | ❌ | +| 1 | 0.809947 | `storage_account_create` | ✅ **EXPECTED** | +| 2 | 0.803393 | `quota_usage_check` | ❌ | +| 3 | 0.798147 | `storage_blob_container_create` | ❌ | +| 4 | 0.793527 | `storage_account_get` | ❌ | +| 5 | 0.788024 | `redis_create` | ❌ | --- ## Test 412 -**Expected Tool:** `sql_db_update` -**Prompt:** Scale SQL database on server to use SKU +**Expected Tool:** `storage_account_create` +**Prompt:** Create a storage account with premium performance and LRS replication ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.550556 | `sql_db_update` | ✅ **EXPECTED** | -| 2 | 0.418358 | `sql_server_delete` | ❌ | -| 3 | 0.401817 | `sql_db_list` | ❌ | -| 4 | 0.395518 | `sql_db_rename` | ❌ | -| 5 | 0.394770 | `sql_db_show` | ❌ | +| 1 | 0.808966 | `redis_create` | ❌ | +| 2 | 0.800424 | `storage_blob_container_create` | ❌ | +| 3 | 0.795156 | `managedlustre_fs_list` | ❌ | +| 4 | 0.794542 | `managedlustre_fs_create` | ❌ | +| 5 | 0.794145 | `storage_account_get` | ❌ | --- ## Test 413 -**Expected Tool:** `sql_elastic-pool_list` -**Prompt:** List all elastic pools in SQL server +**Expected Tool:** `storage_account_create` +**Prompt:** Create a new storage account with Data Lake Storage Gen2 enabled ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.678124 | `sql_elastic-pool_list` | ✅ **EXPECTED** | -| 2 | 0.502376 | `sql_db_list` | ❌ | -| 3 | 0.498367 | `mysql_database_list` | ❌ | -| 4 | 0.485249 | `aks_nodepool_get` | ❌ | -| 5 | 0.479044 | `sql_server_show` | ❌ | +| 1 | 0.792633 | `storage_account_create` | ✅ **EXPECTED** | +| 2 | 0.784383 | `storage_account_get` | ❌ | +| 3 | 0.783429 | `storage_blob_container_create` | ❌ | +| 4 | 0.782223 | `redis_create` | ❌ | +| 5 | 0.771119 | `loadtesting_testresource_create` | ❌ | --- ## Test 414 -**Expected Tool:** `sql_elastic-pool_list` -**Prompt:** Show me the elastic pools configured for SQL server +**Expected Tool:** `storage_account_get` +**Prompt:** Show me the details for my storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.606425 | `sql_elastic-pool_list` | ✅ **EXPECTED** | -| 2 | 0.502877 | `sql_server_show` | ❌ | -| 3 | 0.457164 | `sql_db_list` | ❌ | -| 4 | 0.450743 | `aks_nodepool_get` | ❌ | -| 5 | 0.432816 | `mysql_database_list` | ❌ | +| 1 | 0.864705 | `storage_account_get` | ✅ **EXPECTED** | +| 2 | 0.852899 | `storage_blob_container_get` | ❌ | +| 3 | 0.833180 | `storage_blob_get` | ❌ | +| 4 | 0.809586 | `sql_server_show` | ❌ | +| 5 | 0.808785 | `quota_usage_check` | ❌ | --- ## Test 415 -**Expected Tool:** `sql_elastic-pool_list` -**Prompt:** What elastic pools are available in my SQL server ? +**Expected Tool:** `storage_account_get` +**Prompt:** Get details about the storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.592709 | `sql_elastic-pool_list` | ✅ **EXPECTED** | -| 2 | 0.420325 | `mysql_database_list` | ❌ | -| 3 | 0.407169 | `aks_nodepool_get` | ❌ | -| 4 | 0.402616 | `mysql_server_list` | ❌ | -| 5 | 0.397670 | `sql_db_list` | ❌ | +| 1 | 0.887388 | `storage_account_get` | ✅ **EXPECTED** | +| 2 | 0.848471 | `storage_blob_container_get` | ❌ | +| 3 | 0.839587 | `storage_account_create` | ❌ | +| 4 | 0.838349 | `storage_blob_get` | ❌ | +| 5 | 0.829422 | `sql_server_show` | ❌ | --- ## Test 416 -**Expected Tool:** `sql_server_create` -**Prompt:** Create a new Azure SQL server named in resource group +**Expected Tool:** `storage_account_get` +**Prompt:** List all storage accounts in my subscription including their location and SKU ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.682605 | `sql_server_create` | ✅ **EXPECTED** | -| 2 | 0.563707 | `sql_db_create` | ❌ | -| 3 | 0.529198 | `sql_server_list` | ❌ | -| 4 | 0.482102 | `storage_account_create` | ❌ | -| 5 | 0.474207 | `sql_db_rename` | ❌ | +| 1 | 0.883045 | `storage_account_get` | ✅ **EXPECTED** | +| 2 | 0.847833 | `kusto_cluster_list` | ❌ | +| 3 | 0.846985 | `subscription_list` | ❌ | +| 4 | 0.842619 | `search_service_list` | ❌ | +| 5 | 0.834655 | `managedlustre_fs_sku_get` | ❌ | --- ## Test 417 -**Expected Tool:** `sql_server_create` -**Prompt:** Create an Azure SQL server with name in location with admin user +**Expected Tool:** `storage_account_get` +**Prompt:** Show me my storage accounts with whether hierarchical namespace (HNS) is enabled ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.618309 | `sql_server_create` | ✅ **EXPECTED** | -| 2 | 0.510169 | `sql_db_create` | ❌ | -| 3 | 0.472463 | `sql_server_show` | ❌ | -| 4 | 0.441174 | `sql_server_delete` | ❌ | -| 5 | 0.400939 | `sql_db_rename` | ❌ | +| 1 | 0.824986 | `storage_account_get` | ✅ **EXPECTED** | +| 2 | 0.783882 | `storage_blob_container_get` | ❌ | +| 3 | 0.782774 | `eventhubs_namespace_get` | ❌ | +| 4 | 0.782649 | `eventhubs_eventhub_get` | ❌ | +| 5 | 0.770657 | `keyvault_admin_settings_get` | ❌ | --- ## Test 418 -**Expected Tool:** `sql_server_create` -**Prompt:** Set up a new SQL server called in my resource group +**Expected Tool:** `storage_account_get` +**Prompt:** Show me the storage accounts in my subscription and include HTTPS-only and public blob access settings ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.589818 | `sql_server_create` | ✅ **EXPECTED** | -| 2 | 0.501403 | `sql_db_create` | ❌ | -| 3 | 0.497890 | `sql_server_list` | ❌ | -| 4 | 0.461181 | `sql_db_rename` | ❌ | -| 5 | 0.442934 | `mysql_server_list` | ❌ | +| 1 | 0.865927 | `storage_account_get` | ✅ **EXPECTED** | +| 2 | 0.860765 | `storage_blob_container_get` | ❌ | +| 3 | 0.839171 | `storage_blob_get` | ❌ | +| 4 | 0.822019 | `subscription_list` | ❌ | +| 5 | 0.816924 | `storage_blob_container_create` | ❌ | --- ## Test 419 -**Expected Tool:** `sql_server_delete` -**Prompt:** Delete the Azure SQL server from resource group +**Expected Tool:** `storage_blob_container_create` +**Prompt:** Create the storage container mycontainer in storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.656593 | `sql_server_delete` | ✅ **EXPECTED** | -| 2 | 0.548064 | `sql_db_delete` | ❌ | -| 3 | 0.518037 | `sql_server_list` | ❌ | -| 4 | 0.495550 | `sql_server_create` | ❌ | -| 5 | 0.483132 | `workbooks_delete` | ❌ | +| 1 | 0.850639 | `storage_blob_container_create` | ✅ **EXPECTED** | +| 2 | 0.834053 | `storage_blob_container_get` | ❌ | +| 3 | 0.802218 | `storage_blob_get` | ❌ | +| 4 | 0.795088 | `storage_account_create` | ❌ | +| 5 | 0.791175 | `cosmos_database_container_list` | ❌ | --- ## Test 420 -**Expected Tool:** `sql_server_delete` -**Prompt:** Remove the SQL server from my subscription +**Expected Tool:** `storage_blob_container_create` +**Prompt:** Create the container using blob public access in storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.615073 | `sql_server_delete` | ✅ **EXPECTED** | -| 2 | 0.393885 | `postgres_server_list` | ❌ | -| 3 | 0.379760 | `sql_db_delete` | ❌ | -| 4 | 0.376660 | `sql_server_show` | ❌ | -| 5 | 0.350103 | `sql_server_list` | ❌ | +| 1 | 0.882125 | `storage_blob_container_create` | ✅ **EXPECTED** | +| 2 | 0.864153 | `storage_blob_container_get` | ❌ | +| 3 | 0.839007 | `storage_blob_get` | ❌ | +| 4 | 0.806190 | `storage_account_get` | ❌ | +| 5 | 0.796376 | `cosmos_database_container_item_query` | ❌ | --- ## Test 421 -**Expected Tool:** `sql_server_delete` -**Prompt:** Delete SQL server permanently +**Expected Tool:** `storage_blob_container_create` +**Prompt:** Create a new blob container named documents with container public access in storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.624310 | `sql_server_delete` | ✅ **EXPECTED** | -| 2 | 0.454892 | `sql_db_delete` | ❌ | -| 3 | 0.362389 | `sql_server_firewall-rule_delete` | ❌ | -| 4 | 0.341503 | `sql_server_show` | ❌ | -| 5 | 0.318758 | `eventhubs_eventhub_delete` | ❌ | +| 1 | 0.876258 | `storage_blob_container_create` | ✅ **EXPECTED** | +| 2 | 0.849822 | `storage_blob_container_get` | ❌ | +| 3 | 0.826988 | `storage_blob_get` | ❌ | +| 4 | 0.805862 | `storage_account_get` | ❌ | +| 5 | 0.800129 | `cosmos_database_container_item_query` | ❌ | --- ## Test 422 -**Expected Tool:** `sql_server_entra-admin_list` -**Prompt:** List Microsoft Entra ID administrators for SQL server +**Expected Tool:** `storage_blob_container_get` +**Prompt:** Show me the properties of the storage container in the storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.783479 | `sql_server_entra-admin_list` | ✅ **EXPECTED** | -| 2 | 0.456051 | `sql_server_show` | ❌ | -| 3 | 0.434868 | `sql_server_list` | ❌ | -| 4 | 0.401908 | `sql_server_firewall-rule_list` | ❌ | -| 5 | 0.376055 | `sql_db_list` | ❌ | +| 1 | 0.891728 | `storage_blob_container_get` | ✅ **EXPECTED** | +| 2 | 0.857128 | `storage_blob_get` | ❌ | +| 3 | 0.832367 | `storage_account_get` | ❌ | +| 4 | 0.829749 | `storage_blob_container_create` | ❌ | +| 5 | 0.814633 | `cosmos_database_container_list` | ❌ | --- ## Test 423 -**Expected Tool:** `sql_server_entra-admin_list` -**Prompt:** Show me the Entra ID administrators configured for SQL server +**Expected Tool:** `storage_blob_container_get` +**Prompt:** List all blob containers in the storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.713306 | `sql_server_entra-admin_list` | ✅ **EXPECTED** | -| 2 | 0.413144 | `sql_server_show` | ❌ | -| 3 | 0.368082 | `sql_server_list` | ❌ | -| 4 | 0.315966 | `sql_db_list` | ❌ | -| 5 | 0.311085 | `postgres_server_list` | ❌ | +| 1 | 0.892410 | `storage_blob_container_get` | ✅ **EXPECTED** | +| 2 | 0.879298 | `storage_blob_get` | ❌ | +| 3 | 0.854342 | `storage_blob_container_create` | ❌ | +| 4 | 0.844717 | `cosmos_database_container_list` | ❌ | +| 5 | 0.829854 | `storage_account_get` | ❌ | --- ## Test 424 -**Expected Tool:** `sql_server_entra-admin_list` -**Prompt:** What Microsoft Entra ID administrators are set up for my SQL server ? +**Expected Tool:** `storage_blob_container_get` +**Prompt:** Show me the containers in the storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.646419 | `sql_server_entra-admin_list` | ✅ **EXPECTED** | -| 2 | 0.356025 | `sql_server_show` | ❌ | -| 3 | 0.322155 | `sql_server_list` | ❌ | -| 4 | 0.307823 | `sql_server_create` | ❌ | -| 5 | 0.269788 | `sql_server_delete` | ❌ | +| 1 | 0.879623 | `storage_blob_container_get` | ✅ **EXPECTED** | +| 2 | 0.847528 | `cosmos_database_container_list` | ❌ | +| 3 | 0.833656 | `storage_blob_get` | ❌ | +| 4 | 0.825182 | `cosmos_database_container_item_query` | ❌ | +| 5 | 0.818749 | `storage_account_get` | ❌ | --- ## Test 425 -**Expected Tool:** `sql_server_firewall-rule_create` -**Prompt:** Create a firewall rule for my Azure SQL server +**Expected Tool:** `storage_blob_get` +**Prompt:** Show me the properties for blob in container in storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.635163 | `sql_server_firewall-rule_create` | ✅ **EXPECTED** | -| 2 | 0.532712 | `sql_server_firewall-rule_list` | ❌ | -| 3 | 0.522184 | `sql_server_firewall-rule_delete` | ❌ | -| 4 | 0.448822 | `sql_server_create` | ❌ | -| 5 | 0.440845 | `sql_server_delete` | ❌ | +| 1 | 0.894876 | `storage_blob_container_get` | ❌ | +| 2 | 0.891593 | `storage_blob_get` | ✅ **EXPECTED** | +| 3 | 0.837119 | `storage_blob_container_create` | ❌ | +| 4 | 0.829013 | `storage_account_get` | ❌ | +| 5 | 0.787205 | `cosmos_database_container_list` | ❌ | --- ## Test 426 -**Expected Tool:** `sql_server_firewall-rule_create` -**Prompt:** Add a firewall rule to allow access from IP range to for SQL server +**Expected Tool:** `storage_blob_get` +**Prompt:** Get the details about blob in the container in storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.670397 | `sql_server_firewall-rule_create` | ✅ **EXPECTED** | -| 2 | 0.533527 | `sql_server_firewall-rule_list` | ❌ | -| 3 | 0.503642 | `sql_server_firewall-rule_delete` | ❌ | -| 4 | 0.316557 | `sql_server_list` | ❌ | -| 5 | 0.302359 | `sql_server_delete` | ❌ | +| 1 | 0.893390 | `storage_blob_get` | ✅ **EXPECTED** | +| 2 | 0.883276 | `storage_blob_container_get` | ❌ | +| 3 | 0.861351 | `storage_account_get` | ❌ | +| 4 | 0.849169 | `storage_blob_container_create` | ❌ | +| 5 | 0.805667 | `cosmos_database_container_item_query` | ❌ | --- ## Test 427 -**Expected Tool:** `sql_server_firewall-rule_create` -**Prompt:** Create a new firewall rule named for SQL server +**Expected Tool:** `storage_blob_get` +**Prompt:** List all blobs in the blob container in the storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.684722 | `sql_server_firewall-rule_create` | ✅ **EXPECTED** | -| 2 | 0.574336 | `sql_server_firewall-rule_list` | ❌ | -| 3 | 0.539577 | `sql_server_firewall-rule_delete` | ❌ | -| 4 | 0.428920 | `sql_server_create` | ❌ | -| 5 | 0.395165 | `sql_db_create` | ❌ | +| 1 | 0.893094 | `storage_blob_container_get` | ❌ | +| 2 | 0.890815 | `storage_blob_get` | ✅ **EXPECTED** | +| 3 | 0.861403 | `storage_blob_container_create` | ❌ | +| 4 | 0.844959 | `cosmos_database_container_item_query` | ❌ | +| 5 | 0.841411 | `cosmos_database_container_list` | ❌ | --- ## Test 428 -**Expected Tool:** `sql_server_firewall-rule_delete` -**Prompt:** Delete a firewall rule from my Azure SQL server +**Expected Tool:** `storage_blob_get` +**Prompt:** Show me the blobs in the blob container in the storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.691421 | `sql_server_firewall-rule_delete` | ✅ **EXPECTED** | -| 2 | 0.584379 | `sql_server_delete` | ❌ | -| 3 | 0.543857 | `sql_server_firewall-rule_list` | ❌ | -| 4 | 0.540096 | `sql_server_firewall-rule_create` | ❌ | -| 5 | 0.498444 | `sql_db_delete` | ❌ | +| 1 | 0.895199 | `storage_blob_container_get` | ❌ | +| 2 | 0.884449 | `storage_blob_get` | ✅ **EXPECTED** | +| 3 | 0.847241 | `storage_blob_container_create` | ❌ | +| 4 | 0.827470 | `storage_account_get` | ❌ | +| 5 | 0.819109 | `cosmos_database_container_list` | ❌ | --- ## Test 429 -**Expected Tool:** `sql_server_firewall-rule_delete` -**Prompt:** Remove the firewall rule from SQL server +**Expected Tool:** `storage_blob_upload` +**Prompt:** Upload file to storage blob in container in storage account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.670179 | `sql_server_firewall-rule_delete` | ✅ **EXPECTED** | -| 2 | 0.574340 | `sql_server_firewall-rule_list` | ❌ | -| 3 | 0.530185 | `sql_server_firewall-rule_create` | ❌ | -| 4 | 0.488418 | `sql_server_delete` | ❌ | -| 5 | 0.360381 | `sql_db_delete` | ❌ | +| 1 | 0.837264 | `storage_blob_container_create` | ❌ | +| 2 | 0.834825 | `storage_blob_upload` | ✅ **EXPECTED** | +| 3 | 0.822190 | `storage_blob_get` | ❌ | +| 4 | 0.809272 | `storage_blob_container_get` | ❌ | +| 5 | 0.768544 | `storage_account_get` | ❌ | --- ## Test 430 -**Expected Tool:** `sql_server_firewall-rule_delete` -**Prompt:** Delete firewall rule for SQL server +**Expected Tool:** `subscription_list` +**Prompt:** List all subscriptions for my account ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.671211 | `sql_server_firewall-rule_delete` | ✅ **EXPECTED** | -| 2 | 0.601231 | `sql_server_firewall-rule_list` | ❌ | -| 3 | 0.576968 | `sql_server_firewall-rule_create` | ❌ | -| 4 | 0.499272 | `sql_server_delete` | ❌ | -| 5 | 0.378586 | `sql_db_delete` | ❌ | +| 1 | 0.856770 | `subscription_list` | ✅ **EXPECTED** | +| 2 | 0.842356 | `eventgrid_subscription_list` | ❌ | +| 3 | 0.831559 | `eventgrid_topic_list` | ❌ | +| 4 | 0.825601 | `cosmos_account_list` | ❌ | +| 5 | 0.825060 | `search_service_list` | ❌ | --- ## Test 431 -**Expected Tool:** `sql_server_firewall-rule_list` -**Prompt:** List all firewall rules for SQL server +**Expected Tool:** `subscription_list` +**Prompt:** Show me my subscriptions ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.729372 | `sql_server_firewall-rule_list` | ✅ **EXPECTED** | -| 2 | 0.549457 | `sql_server_firewall-rule_create` | ❌ | -| 3 | 0.513114 | `sql_server_firewall-rule_delete` | ❌ | -| 4 | 0.468812 | `sql_server_show` | ❌ | -| 5 | 0.418817 | `sql_server_list` | ❌ | +| 1 | 0.819054 | `eventgrid_subscription_list` | ❌ | +| 2 | 0.803164 | `redis_list` | ❌ | +| 3 | 0.800703 | `eventgrid_topic_list` | ❌ | +| 4 | 0.795662 | `subscription_list` | ✅ **EXPECTED** | +| 5 | 0.785142 | `search_service_list` | ❌ | --- ## Test 432 -**Expected Tool:** `sql_server_firewall-rule_list` -**Prompt:** Show me the firewall rules for SQL server +**Expected Tool:** `subscription_list` +**Prompt:** What is my current subscription? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.630731 | `sql_server_firewall-rule_list` | ✅ **EXPECTED** | -| 2 | 0.523916 | `sql_server_firewall-rule_create` | ❌ | -| 3 | 0.476757 | `sql_server_firewall-rule_delete` | ❌ | -| 4 | 0.410680 | `sql_server_show` | ❌ | -| 5 | 0.348100 | `sql_server_list` | ❌ | +| 1 | 0.791653 | `subscription_list` | ✅ **EXPECTED** | +| 2 | 0.784686 | `eventgrid_subscription_list` | ❌ | +| 3 | 0.777850 | `eventgrid_topic_list` | ❌ | +| 4 | 0.770929 | `redis_list` | ❌ | +| 5 | 0.768711 | `marketplace_product_list` | ❌ | --- ## Test 433 -**Expected Tool:** `sql_server_firewall-rule_list` -**Prompt:** What firewall rules are configured for my SQL server ? +**Expected Tool:** `subscription_list` +**Prompt:** What subscriptions do I have? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.630509 | `sql_server_firewall-rule_list` | ✅ **EXPECTED** | -| 2 | 0.532311 | `sql_server_firewall-rule_create` | ❌ | -| 3 | 0.473444 | `sql_server_firewall-rule_delete` | ❌ | -| 4 | 0.412843 | `sql_server_show` | ❌ | -| 5 | 0.350461 | `sql_server_list` | ❌ | +| 1 | 0.805756 | `eventgrid_subscription_list` | ❌ | +| 2 | 0.799051 | `subscription_list` | ✅ **EXPECTED** | +| 3 | 0.792453 | `redis_list` | ❌ | +| 4 | 0.792011 | `eventgrid_topic_list` | ❌ | +| 5 | 0.775511 | `search_service_list` | ❌ | --- ## Test 434 -**Expected Tool:** `sql_server_list` -**Prompt:** List all Azure SQL servers in resource group +**Expected Tool:** `azureterraformbestpractices_get` +**Prompt:** Fetch the Azure Terraform best practices ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.694404 | `sql_server_list` | ✅ **EXPECTED** | -| 2 | 0.596686 | `mysql_server_list` | ❌ | -| 3 | 0.578238 | `sql_db_list` | ❌ | -| 4 | 0.515851 | `sql_elastic-pool_list` | ❌ | -| 5 | 0.509789 | `sql_db_show` | ❌ | +| 1 | 0.879282 | `azureterraformbestpractices_get` | ✅ **EXPECTED** | +| 2 | 0.861340 | `get_bestpractices_get` | ❌ | +| 3 | 0.838960 | `azureaibestpractices_get` | ❌ | +| 4 | 0.814913 | `deploy_iac_rules_get` | ❌ | +| 5 | 0.804409 | `cloudarchitect_design` | ❌ | --- ## Test 435 -**Expected Tool:** `sql_server_list` -**Prompt:** Show me every SQL server available in resource group +**Expected Tool:** `azureterraformbestpractices_get` +**Prompt:** Show me the Azure Terraform best practices and generate code sample to get a secret from Azure Key Vault ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.618178 | `sql_server_list` | ✅ **EXPECTED** | -| 2 | 0.593840 | `mysql_server_list` | ❌ | -| 3 | 0.542335 | `sql_db_list` | ❌ | -| 4 | 0.507480 | `resourcehealth_availability-status_list` | ❌ | -| 5 | 0.496297 | `group_list` | ❌ | +| 1 | 0.821345 | `keyvault_secret_get` | ❌ | +| 2 | 0.812174 | `azureterraformbestpractices_get` | ✅ **EXPECTED** | +| 3 | 0.808736 | `keyvault_secret_create` | ❌ | +| 4 | 0.807316 | `get_bestpractices_get` | ❌ | +| 5 | 0.801636 | `foundry_agents_get-sdk-sample` | ❌ | --- ## Test 436 -**Expected Tool:** `sql_server_show` -**Prompt:** Show me the details of Azure SQL server in resource group +**Expected Tool:** `virtualdesktop_hostpool_list` +**Prompt:** List all host pools in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.629672 | `sql_db_show` | ❌ | -| 2 | 0.595184 | `sql_server_show` | ✅ **EXPECTED** | -| 3 | 0.587728 | `sql_server_list` | ❌ | -| 4 | 0.559893 | `mysql_server_list` | ❌ | -| 5 | 0.540218 | `sql_db_list` | ❌ | +| 1 | 0.875486 | `virtualdesktop_hostpool_list` | ✅ **EXPECTED** | +| 2 | 0.849043 | `virtualdesktop_hostpool_host_list` | ❌ | +| 3 | 0.811700 | `kusto_cluster_list` | ❌ | +| 4 | 0.804164 | `postgres_server_list` | ❌ | +| 5 | 0.802633 | `virtualdesktop_hostpool_host_user-list` | ❌ | --- ## Test 437 -**Expected Tool:** `sql_server_show` -**Prompt:** Get the configuration details for SQL server +**Expected Tool:** `virtualdesktop_hostpool_host_list` +**Prompt:** List all session hosts in host pool ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.658817 | `sql_server_show` | ✅ **EXPECTED** | -| 2 | 0.610507 | `postgres_server_config_get` | ❌ | -| 3 | 0.538034 | `mysql_server_config_get` | ❌ | -| 4 | 0.471541 | `sql_db_show` | ❌ | -| 5 | 0.445430 | `postgres_server_param_get` | ❌ | +| 1 | 0.877241 | `virtualdesktop_hostpool_host_list` | ✅ **EXPECTED** | +| 2 | 0.872758 | `virtualdesktop_hostpool_host_user-list` | ❌ | +| 3 | 0.814023 | `virtualdesktop_hostpool_list` | ❌ | +| 4 | 0.768581 | `aks_nodepool_get` | ❌ | +| 5 | 0.745038 | `postgres_server_list` | ❌ | --- ## Test 438 -**Expected Tool:** `sql_server_show` -**Prompt:** Display the properties of SQL server +**Expected Tool:** `virtualdesktop_hostpool_host_user-list` +**Prompt:** List all user sessions on session host in host pool ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.563143 | `sql_server_show` | ✅ **EXPECTED** | -| 2 | 0.392532 | `postgres_server_config_get` | ❌ | -| 3 | 0.380021 | `postgres_server_param_get` | ❌ | -| 4 | 0.372194 | `sql_server_firewall-rule_list` | ❌ | -| 5 | 0.370539 | `sql_db_show` | ❌ | +| 1 | 0.903083 | `virtualdesktop_hostpool_host_user-list` | ✅ **EXPECTED** | +| 2 | 0.852811 | `virtualdesktop_hostpool_host_list` | ❌ | +| 3 | 0.792047 | `virtualdesktop_hostpool_list` | ❌ | +| 4 | 0.754118 | `aks_nodepool_get` | ❌ | +| 5 | 0.742301 | `foundry_threads_list` | ❌ | --- ## Test 439 -**Expected Tool:** `storage_account_create` -**Prompt:** Create a new storage account called testaccount123 in East US region +**Expected Tool:** `workbooks_create` +**Prompt:** Create a new workbook named ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.533552 | `storage_account_create` | ✅ **EXPECTED** | -| 2 | 0.438046 | `storage_blob_container_create` | ❌ | -| 3 | 0.418191 | `storage_account_get` | ❌ | -| 4 | 0.414518 | `storage_blob_container_get` | ❌ | -| 5 | 0.370957 | `managedlustre_fs_create` | ❌ | +| 1 | 0.814658 | `workbooks_create` | ✅ **EXPECTED** | +| 2 | 0.749806 | `workbooks_update` | ❌ | +| 3 | 0.746902 | `workbooks_delete` | ❌ | +| 4 | 0.741031 | `workbooks_list` | ❌ | +| 5 | 0.734155 | `workbooks_show` | ❌ | --- ## Test 440 -**Expected Tool:** `storage_account_create` -**Prompt:** Create a storage account with premium performance and LRS replication +**Expected Tool:** `workbooks_delete` +**Prompt:** Delete the workbook with resource ID ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.500638 | `storage_account_create` | ✅ **EXPECTED** | -| 2 | 0.483202 | `managedlustre_fs_create` | ❌ | -| 3 | 0.407222 | `storage_account_get` | ❌ | -| 4 | 0.406804 | `storage_blob_container_create` | ❌ | -| 5 | 0.400151 | `managedlustre_fs_sku_get` | ❌ | +| 1 | 0.870999 | `workbooks_delete` | ✅ **EXPECTED** | +| 2 | 0.800042 | `workbooks_show` | ❌ | +| 3 | 0.775413 | `workbooks_list` | ❌ | +| 4 | 0.767718 | `workbooks_create` | ❌ | +| 5 | 0.760256 | `workbooks_update` | ❌ | --- ## Test 441 -**Expected Tool:** `storage_account_create` -**Prompt:** Create a new storage account with Data Lake Storage Gen2 enabled +**Expected Tool:** `workbooks_list` +**Prompt:** List all workbooks in my resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.589002 | `storage_account_create` | ✅ **EXPECTED** | -| 2 | 0.535501 | `managedlustre_fs_create` | ❌ | -| 3 | 0.509731 | `storage_blob_container_create` | ❌ | -| 4 | 0.462519 | `storage_account_get` | ❌ | -| 5 | 0.447156 | `sql_db_create` | ❌ | +| 1 | 0.882445 | `workbooks_list` | ✅ **EXPECTED** | +| 2 | 0.795826 | `workbooks_create` | ❌ | +| 3 | 0.794634 | `group_list` | ❌ | +| 4 | 0.785093 | `workbooks_delete` | ❌ | +| 5 | 0.784826 | `loadtesting_testresource_list` | ❌ | --- ## Test 442 -**Expected Tool:** `storage_account_get` -**Prompt:** Show me the details for my storage account +**Expected Tool:** `workbooks_list` +**Prompt:** What workbooks do I have in resource group ? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.673750 | `storage_account_get` | ✅ **EXPECTED** | -| 2 | 0.608256 | `storage_blob_container_get` | ❌ | -| 3 | 0.556457 | `storage_blob_get` | ❌ | -| 4 | 0.483435 | `storage_account_create` | ❌ | -| 5 | 0.439236 | `cosmos_account_list` | ❌ | +| 1 | 0.831874 | `workbooks_list` | ✅ **EXPECTED** | +| 2 | 0.786614 | `workbooks_create` | ❌ | +| 3 | 0.777631 | `workbooks_delete` | ❌ | +| 4 | 0.770929 | `redis_list` | ❌ | +| 5 | 0.770043 | `resourcehealth_availability-status_list` | ❌ | --- ## Test 443 -**Expected Tool:** `storage_account_get` -**Prompt:** Get details about the storage account +**Expected Tool:** `workbooks_show` +**Prompt:** Get information about the workbook with resource ID ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.692687 | `storage_account_get` | ✅ **EXPECTED** | -| 2 | 0.577547 | `storage_blob_container_get` | ❌ | -| 3 | 0.529205 | `storage_blob_get` | ❌ | -| 4 | 0.518215 | `storage_account_create` | ❌ | -| 5 | 0.448506 | `storage_blob_container_create` | ❌ | +| 1 | 0.885743 | `workbooks_show` | ✅ **EXPECTED** | +| 2 | 0.822054 | `workbooks_create` | ❌ | +| 3 | 0.802642 | `workbooks_list` | ❌ | +| 4 | 0.799931 | `workbooks_delete` | ❌ | +| 5 | 0.792484 | `workbooks_update` | ❌ | --- ## Test 444 -**Expected Tool:** `storage_account_get` -**Prompt:** List all storage accounts in my subscription including their location and SKU +**Expected Tool:** `workbooks_show` +**Prompt:** Show me the workbook with resource ID ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.649215 | `storage_account_get` | ✅ **EXPECTED** | -| 2 | 0.557015 | `managedlustre_fs_sku_get` | ❌ | -| 3 | 0.550148 | `storage_blob_container_get` | ❌ | -| 4 | 0.547610 | `subscription_list` | ❌ | -| 5 | 0.536909 | `cosmos_account_list` | ❌ | +| 1 | 0.825558 | `workbooks_show` | ✅ **EXPECTED** | +| 2 | 0.812208 | `workbooks_list` | ❌ | +| 3 | 0.809751 | `workbooks_delete` | ❌ | +| 4 | 0.801816 | `workbooks_create` | ❌ | +| 5 | 0.779042 | `workbooks_update` | ❌ | --- ## Test 445 -**Expected Tool:** `storage_account_get` -**Prompt:** Show me my storage accounts with whether hierarchical namespace (HNS) is enabled +**Expected Tool:** `workbooks_update` +**Prompt:** Update the workbook with a new text step ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.556860 | `storage_account_get` | ✅ **EXPECTED** | -| 2 | 0.482418 | `storage_blob_container_get` | ❌ | -| 3 | 0.461284 | `managedlustre_fs_list` | ❌ | -| 4 | 0.421642 | `cosmos_account_list` | ❌ | -| 5 | 0.410587 | `storage_blob_get` | ❌ | +| 1 | 0.818557 | `workbooks_update` | ✅ **EXPECTED** | +| 2 | 0.730313 | `workbooks_delete` | ❌ | +| 3 | 0.725172 | `workbooks_create` | ❌ | +| 4 | 0.724545 | `loadtesting_testrun_update` | ❌ | +| 5 | 0.719208 | `workbooks_show` | ❌ | --- ## Test 446 -**Expected Tool:** `storage_account_get` -**Prompt:** Show me the storage accounts in my subscription and include HTTPS-only and public blob access settings +**Expected Tool:** `bicepschema_get` +**Prompt:** How can I use Bicep to create an Azure OpenAI service? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.619462 | `storage_account_get` | ✅ **EXPECTED** | -| 2 | 0.556436 | `storage_blob_container_get` | ❌ | -| 3 | 0.518229 | `storage_blob_get` | ❌ | -| 4 | 0.473598 | `cosmos_account_list` | ❌ | -| 5 | 0.465544 | `subscription_list` | ❌ | +| 1 | 0.829298 | `bicepschema_get` | ✅ **EXPECTED** | +| 2 | 0.811738 | `foundry_models_deploy` | ❌ | +| 3 | 0.809275 | `azureaibestpractices_get` | ❌ | +| 4 | 0.805211 | `foundry_openai_embeddings-create` | ❌ | +| 5 | 0.803405 | `foundry_openai_create-completion` | ❌ | --- ## Test 447 -**Expected Tool:** `storage_blob_container_create` -**Prompt:** Create the storage container mycontainer in storage account +**Expected Tool:** `cloudarchitect_design` +**Prompt:** Please help me design an architecture for a large-scale file upload, storage, and retrieval service ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.649793 | `storage_blob_container_create` | ✅ **EXPECTED** | -| 2 | 0.583896 | `storage_blob_container_get` | ❌ | -| 3 | 0.524779 | `storage_account_create` | ❌ | -| 4 | 0.496679 | `storage_blob_get` | ❌ | -| 5 | 0.447784 | `cosmos_database_container_list` | ❌ | +| 1 | 0.839665 | `cloudarchitect_design` | ✅ **EXPECTED** | +| 2 | 0.781938 | `deploy_architecture_diagram_generate` | ❌ | +| 3 | 0.777407 | `deploy_iac_rules_get` | ❌ | +| 4 | 0.768999 | `deploy_pipeline_guidance_get` | ❌ | +| 5 | 0.753698 | `deploy_plan_get` | ❌ | --- ## Test 448 -**Expected Tool:** `storage_blob_container_create` -**Prompt:** Create the container using blob public access in storage account +**Expected Tool:** `cloudarchitect_design` +**Prompt:** Help me design an Azure cloud service that will serve as an ATM for users ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.682161 | `storage_blob_container_create` | ✅ **EXPECTED** | -| 2 | 0.590160 | `storage_blob_container_get` | ❌ | -| 3 | 0.559264 | `storage_blob_get` | ❌ | -| 4 | 0.500625 | `storage_account_create` | ❌ | -| 5 | 0.420514 | `storage_account_get` | ❌ | +| 1 | 0.831567 | `cloudarchitect_design` | ✅ **EXPECTED** | +| 2 | 0.800906 | `deploy_iac_rules_get` | ❌ | +| 3 | 0.797772 | `deploy_architecture_diagram_generate` | ❌ | +| 4 | 0.794105 | `deploy_plan_get` | ❌ | +| 5 | 0.790057 | `quota_usage_check` | ❌ | --- ## Test 449 -**Expected Tool:** `storage_blob_container_create` -**Prompt:** Create a new blob container named documents with container public access in storage account +**Expected Tool:** `cloudarchitect_design` +**Prompt:** I want to design a cloud app for ordering groceries ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.625397 | `storage_blob_container_create` | ✅ **EXPECTED** | -| 2 | 0.543503 | `storage_blob_container_get` | ❌ | -| 3 | 0.497804 | `storage_blob_get` | ❌ | -| 4 | 0.463198 | `storage_account_create` | ❌ | -| 5 | 0.435099 | `cosmos_database_container_list` | ❌ | +| 1 | 0.805150 | `cloudarchitect_design` | ✅ **EXPECTED** | +| 2 | 0.775187 | `deploy_pipeline_guidance_get` | ❌ | +| 3 | 0.773639 | `deploy_iac_rules_get` | ❌ | +| 4 | 0.768590 | `deploy_app_logs_get` | ❌ | +| 5 | 0.764537 | `deploy_architecture_diagram_generate` | ❌ | --- ## Test 450 -**Expected Tool:** `storage_blob_container_get` -**Prompt:** Show me the properties of the storage container in the storage account +**Expected Tool:** `cloudarchitect_design` +**Prompt:** How can I design a cloud service in Azure that will store and present videos for users? ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.701642 | `storage_blob_container_get` | ✅ **EXPECTED** | -| 2 | 0.623681 | `storage_blob_get` | ❌ | -| 3 | 0.577921 | `storage_account_get` | ❌ | -| 4 | 0.549804 | `storage_blob_container_create` | ❌ | -| 5 | 0.523289 | `cosmos_database_container_list` | ❌ | +| 1 | 0.833132 | `cloudarchitect_design` | ✅ **EXPECTED** | +| 2 | 0.796075 | `deploy_pipeline_guidance_get` | ❌ | +| 3 | 0.795825 | `deploy_iac_rules_get` | ❌ | +| 4 | 0.780347 | `speech_stt_recognize` | ❌ | +| 5 | 0.777490 | `quota_usage_check` | ❌ | --- ## Test 451 -**Expected Tool:** `storage_blob_container_get` -**Prompt:** List all blob containers in the storage account +**Expected Tool:** `foundry_agents_connect` +**Prompt:** Query an agent in my Microsoft Foundry resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.712037 | `storage_blob_container_get` | ✅ **EXPECTED** | -| 2 | 0.680802 | `storage_blob_get` | ❌ | -| 3 | 0.613933 | `cosmos_database_container_list` | ❌ | -| 4 | 0.556319 | `storage_blob_container_create` | ❌ | -| 5 | 0.518266 | `storage_account_get` | ❌ | +| 1 | 0.867235 | `foundry_agents_connect` | ✅ **EXPECTED** | +| 2 | 0.858626 | `foundry_threads_list` | ❌ | +| 3 | 0.855151 | `foundry_threads_get-messages` | ❌ | +| 4 | 0.848660 | `foundry_agents_list` | ❌ | +| 5 | 0.841828 | `foundry_agents_get-sdk-sample` | ❌ | --- ## Test 452 -**Expected Tool:** `storage_blob_container_get` -**Prompt:** Show me the containers in the storage account +**Expected Tool:** `foundry_agents_create` +**Prompt:** Create a new Microsoft Foundry agent using instructions in the active editor ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.713527 | `storage_blob_container_get` | ✅ **EXPECTED** | -| 2 | 0.592373 | `cosmos_database_container_list` | ❌ | -| 3 | 0.586169 | `storage_blob_get` | ❌ | -| 4 | 0.523322 | `storage_account_get` | ❌ | -| 5 | 0.487520 | `storage_blob_container_create` | ❌ | +| 1 | 0.877886 | `foundry_agents_get-sdk-sample` | ❌ | +| 2 | 0.875213 | `foundry_agents_create` | ✅ **EXPECTED** | +| 3 | 0.847476 | `foundry_threads_list` | ❌ | +| 4 | 0.837091 | `foundry_threads_create` | ❌ | +| 5 | 0.835023 | `foundry_threads_get-messages` | ❌ | --- ## Test 453 -**Expected Tool:** `storage_blob_get` -**Prompt:** Show me the properties for blob in container in storage account +**Expected Tool:** `foundry_agents_evaluate` +**Prompt:** Evaluate the full query and response I got from my agent for task_adherence ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.700979 | `storage_blob_get` | ✅ **EXPECTED** | -| 2 | 0.646957 | `storage_blob_container_get` | ❌ | -| 3 | 0.540996 | `storage_blob_container_create` | ❌ | -| 4 | 0.527333 | `storage_account_get` | ❌ | -| 5 | 0.477922 | `cosmos_database_container_list` | ❌ | +| 1 | 0.816665 | `foundry_agents_query-and-evaluate` | ❌ | +| 2 | 0.805375 | `foundry_agents_connect` | ❌ | +| 3 | 0.774396 | `foundry_agents_evaluate` | ✅ **EXPECTED** | +| 4 | 0.741379 | `resourcehealth_health-events_list` | ❌ | +| 5 | 0.731201 | `search_index_query` | ❌ | --- ## Test 454 -**Expected Tool:** `storage_blob_get` -**Prompt:** Get the details about blob in the container in storage account +**Expected Tool:** `foundry_agents_get-sdk-sample` +**Prompt:** Create a CLI app that can talk to a Microsoft Foundry Agent using Python SDK ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.694956 | `storage_blob_get` | ✅ **EXPECTED** | -| 2 | 0.631148 | `storage_blob_container_get` | ❌ | -| 3 | 0.589155 | `storage_blob_container_create` | ❌ | -| 4 | 0.580204 | `storage_account_get` | ❌ | -| 5 | 0.457001 | `storage_account_create` | ❌ | +| 1 | 0.865434 | `foundry_agents_get-sdk-sample` | ✅ **EXPECTED** | +| 2 | 0.828147 | `foundry_agents_create` | ❌ | +| 3 | 0.810601 | `foundry_threads_get-messages` | ❌ | +| 4 | 0.806691 | `foundry_threads_list` | ❌ | +| 5 | 0.803575 | `foundry_agents_connect` | ❌ | --- ## Test 455 -**Expected Tool:** `storage_blob_get` -**Prompt:** List all blobs in the blob container in the storage account +**Expected Tool:** `foundry_agents_list` +**Prompt:** List all agents in my Microsoft Foundry resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.733586 | `storage_blob_get` | ✅ **EXPECTED** | -| 2 | 0.700891 | `storage_blob_container_get` | ❌ | -| 3 | 0.605993 | `storage_blob_container_create` | ❌ | -| 4 | 0.579070 | `cosmos_database_container_list` | ❌ | -| 5 | 0.506660 | `cosmos_database_container_item_query` | ❌ | +| 1 | 0.874889 | `foundry_agents_list` | ✅ **EXPECTED** | +| 2 | 0.868465 | `foundry_threads_list` | ❌ | +| 3 | 0.850070 | `foundry_threads_get-messages` | ❌ | +| 4 | 0.831211 | `foundry_openai_models-list` | ❌ | +| 5 | 0.826676 | `foundry_agents_get-sdk-sample` | ❌ | --- ## Test 456 -**Expected Tool:** `storage_blob_get` -**Prompt:** Show me the blobs in the blob container in the storage account +**Expected Tool:** `foundry_agents_list` +**Prompt:** Show me the available agents in my Microsoft Foundry resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.704426 | `storage_blob_get` | ✅ **EXPECTED** | -| 2 | 0.664940 | `storage_blob_container_get` | ❌ | -| 3 | 0.561557 | `storage_blob_container_create` | ❌ | -| 4 | 0.533515 | `cosmos_database_container_list` | ❌ | -| 5 | 0.484018 | `storage_account_get` | ❌ | +| 1 | 0.868599 | `foundry_agents_list` | ✅ **EXPECTED** | +| 2 | 0.865799 | `foundry_threads_list` | ❌ | +| 3 | 0.857189 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.848802 | `foundry_threads_get-messages` | ❌ | +| 5 | 0.837519 | `foundry_openai_models-list` | ❌ | --- ## Test 457 -**Expected Tool:** `storage_blob_upload` -**Prompt:** Upload file to storage blob in container in storage account +**Expected Tool:** `foundry_agents_query-and-evaluate` +**Prompt:** Query and evaluate an agent in my Microsoft Foundry resource for task_adherence ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.566599 | `storage_blob_upload` | ✅ **EXPECTED** | -| 2 | 0.525386 | `storage_blob_container_create` | ❌ | -| 3 | 0.517956 | `storage_blob_get` | ❌ | -| 4 | 0.474296 | `storage_blob_container_get` | ❌ | -| 5 | 0.384098 | `storage_account_create` | ❌ | +| 1 | 0.837471 | `foundry_agents_connect` | ❌ | +| 2 | 0.805013 | `foundry_threads_list` | ❌ | +| 3 | 0.802627 | `foundry_agents_create` | ❌ | +| 4 | 0.796484 | `foundry_threads_get-messages` | ❌ | +| 5 | 0.796023 | `foundry_agents_list` | ❌ | --- ## Test 458 -**Expected Tool:** `subscription_list` -**Prompt:** List all subscriptions for my account +**Expected Tool:** `foundry_knowledge_index_list` +**Prompt:** List all knowledge indexes in my Microsoft Foundry project ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.654154 | `subscription_list` | ✅ **EXPECTED** | -| 2 | 0.512964 | `cosmos_account_list` | ❌ | -| 3 | 0.471653 | `postgres_server_list` | ❌ | -| 4 | 0.469023 | `kusto_cluster_list` | ❌ | -| 5 | 0.461078 | `redis_list` | ❌ | +| 1 | 0.882646 | `foundry_knowledge_index_list` | ✅ **EXPECTED** | +| 2 | 0.825618 | `foundry_agents_list` | ❌ | +| 3 | 0.818500 | `foundry_models_deployments_list` | ❌ | +| 4 | 0.817764 | `foundry_agents_get-sdk-sample` | ❌ | +| 5 | 0.817055 | `foundry_resource_get` | ❌ | --- ## Test 459 -**Expected Tool:** `subscription_list` -**Prompt:** Show me my subscriptions +**Expected Tool:** `foundry_knowledge_index_list` +**Prompt:** Show me the knowledge indexes in my Microsoft Foundry project ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.458887 | `subscription_list` | ✅ **EXPECTED** | -| 2 | 0.407471 | `eventgrid_subscription_list` | ❌ | -| 3 | 0.393695 | `eventgrid_topic_list` | ❌ | -| 4 | 0.391555 | `redis_list` | ❌ | -| 5 | 0.381238 | `postgres_server_list` | ❌ | +| 1 | 0.845938 | `foundry_knowledge_index_list` | ✅ **EXPECTED** | +| 2 | 0.838416 | `foundry_agents_get-sdk-sample` | ❌ | +| 3 | 0.815401 | `foundry_knowledge_index_schema` | ❌ | +| 4 | 0.814724 | `foundry_resource_get` | ❌ | +| 5 | 0.812493 | `foundry_models_deployments_list` | ❌ | --- ## Test 460 -**Expected Tool:** `subscription_list` -**Prompt:** What is my current subscription? +**Expected Tool:** `foundry_knowledge_index_schema` +**Prompt:** Show me the schema for knowledge index in my Microsoft Foundry resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.433214 | `subscription_list` | ✅ **EXPECTED** | -| 2 | 0.319579 | `marketplace_product_list` | ❌ | -| 3 | 0.315547 | `marketplace_product_get` | ❌ | -| 4 | 0.293772 | `eventgrid_subscription_list` | ❌ | -| 5 | 0.289334 | `eventgrid_topic_list` | ❌ | +| 1 | 0.867230 | `foundry_knowledge_index_schema` | ✅ **EXPECTED** | +| 2 | 0.823041 | `foundry_resource_get` | ❌ | +| 3 | 0.807527 | `foundry_knowledge_index_list` | ❌ | +| 4 | 0.804298 | `kusto_table_schema` | ❌ | +| 5 | 0.800012 | `foundry_agents_get-sdk-sample` | ❌ | --- ## Test 461 -**Expected Tool:** `subscription_list` -**Prompt:** What subscriptions do I have? +**Expected Tool:** `foundry_knowledge_index_schema` +**Prompt:** Get the schema configuration for knowledge index ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.477681 | `subscription_list` | ✅ **EXPECTED** | -| 2 | 0.357625 | `eventgrid_subscription_list` | ❌ | -| 3 | 0.354286 | `marketplace_product_list` | ❌ | -| 4 | 0.344549 | `redis_list` | ❌ | -| 5 | 0.340836 | `eventgrid_topic_list` | ❌ | +| 1 | 0.852073 | `foundry_knowledge_index_schema` | ✅ **EXPECTED** | +| 2 | 0.784873 | `kusto_table_schema` | ❌ | +| 3 | 0.784505 | `postgres_table_schema_get` | ❌ | +| 4 | 0.778622 | `postgres_server_config_get` | ❌ | +| 5 | 0.757097 | `search_index_get` | ❌ | --- ## Test 462 -**Expected Tool:** `azureterraformbestpractices_get` -**Prompt:** Fetch the Azure Terraform best practices +**Expected Tool:** `foundry_models_deploy` +**Prompt:** Deploy a GPT4o instance on my resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.686886 | `azureterraformbestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.625270 | `deploy_iac_rules_get` | ❌ | -| 3 | 0.605048 | `get_bestpractices_get` | ❌ | -| 4 | 0.482936 | `deploy_pipeline_guidance_get` | ❌ | -| 5 | 0.474128 | `azureaibestpractices_get` | ❌ | +| 1 | 0.845092 | `foundry_models_deploy` | ✅ **EXPECTED** | +| 2 | 0.752056 | `foundry_openai_embeddings-create` | ❌ | +| 3 | 0.749333 | `redis_create` | ❌ | +| 4 | 0.744135 | `loadtesting_testrun_create` | ❌ | +| 5 | 0.744059 | `loadtesting_testresource_create` | ❌ | --- ## Test 463 -**Expected Tool:** `azureterraformbestpractices_get` -**Prompt:** Show me the Azure Terraform best practices and generate code sample to get a secret from Azure Key Vault +**Expected Tool:** `foundry_models_deployments_list` +**Prompt:** List all Microsoft Foundry model deployments ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.581316 | `azureterraformbestpractices_get` | ✅ **EXPECTED** | -| 2 | 0.512141 | `get_bestpractices_get` | ❌ | -| 3 | 0.510005 | `deploy_iac_rules_get` | ❌ | -| 4 | 0.473597 | `keyvault_secret_get` | ❌ | -| 5 | 0.451283 | `azureaibestpractices_get` | ❌ | +| 1 | 0.871219 | `foundry_models_deployments_list` | ✅ **EXPECTED** | +| 2 | 0.867883 | `foundry_openai_models-list` | ❌ | +| 3 | 0.844224 | `foundry_resource_get` | ❌ | +| 4 | 0.842227 | `foundry_models_deploy` | ❌ | +| 5 | 0.836863 | `foundry_agents_create` | ❌ | --- ## Test 464 -**Expected Tool:** `virtualdesktop_hostpool_list` -**Prompt:** List all host pools in my subscription +**Expected Tool:** `foundry_models_deployments_list` +**Prompt:** Show me all Microsoft Foundry model deployments ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.711969 | `virtualdesktop_hostpool_list` | ✅ **EXPECTED** | -| 2 | 0.659763 | `virtualdesktop_hostpool_host_list` | ❌ | -| 3 | 0.620665 | `kusto_cluster_list` | ❌ | -| 4 | 0.548888 | `search_service_list` | ❌ | -| 5 | 0.535739 | `virtualdesktop_hostpool_host_user-list` | ❌ | +| 1 | 0.865766 | `foundry_models_deployments_list` | ✅ **EXPECTED** | +| 2 | 0.857345 | `foundry_openai_models-list` | ❌ | +| 3 | 0.839798 | `foundry_resource_get` | ❌ | +| 4 | 0.838284 | `foundry_models_deploy` | ❌ | +| 5 | 0.832294 | `foundry_agents_get-sdk-sample` | ❌ | --- ## Test 465 -**Expected Tool:** `virtualdesktop_hostpool_host_list` -**Prompt:** List all session hosts in host pool +**Expected Tool:** `foundry_models_list` +**Prompt:** List all Microsoft Foundry models ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.727054 | `virtualdesktop_hostpool_host_list` | ✅ **EXPECTED** | -| 2 | 0.714468 | `virtualdesktop_hostpool_host_user-list` | ❌ | -| 3 | 0.573352 | `virtualdesktop_hostpool_list` | ❌ | -| 4 | 0.438659 | `aks_nodepool_get` | ❌ | -| 5 | 0.393721 | `sql_elastic-pool_list` | ❌ | +| 1 | 0.841117 | `foundry_threads_list` | ❌ | +| 2 | 0.840300 | `foundry_openai_models-list` | ❌ | +| 3 | 0.839827 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.839404 | `foundry_models_list` | ✅ **EXPECTED** | +| 5 | 0.839232 | `foundry_resource_get` | ❌ | --- ## Test 466 -**Expected Tool:** `virtualdesktop_hostpool_host_user-list` -**Prompt:** List all user sessions on session host in host pool +**Expected Tool:** `foundry_models_list` +**Prompt:** Show me the available Microsoft Foundry models ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.812659 | `virtualdesktop_hostpool_host_user-list` | ✅ **EXPECTED** | -| 2 | 0.659213 | `virtualdesktop_hostpool_host_list` | ❌ | -| 3 | 0.501168 | `virtualdesktop_hostpool_list` | ❌ | -| 4 | 0.357561 | `aks_nodepool_get` | ❌ | -| 5 | 0.336385 | `monitor_workspace_list` | ❌ | +| 1 | 0.856844 | `foundry_agents_get-sdk-sample` | ❌ | +| 2 | 0.835485 | `foundry_models_list` | ✅ **EXPECTED** | +| 3 | 0.833273 | `foundry_resource_get` | ❌ | +| 4 | 0.829116 | `foundry_threads_list` | ❌ | +| 5 | 0.828666 | `foundry_openai_models-list` | ❌ | --- ## Test 467 -**Expected Tool:** `workbooks_create` -**Prompt:** Create a new workbook named +**Expected Tool:** `foundry_openai_chat-completions-create` +**Prompt:** Create a chat completion with the message "Hello, how are you today?" using my Microsoft Foundry resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.552261 | `workbooks_create` | ✅ **EXPECTED** | -| 2 | 0.417999 | `workbooks_update` | ❌ | -| 3 | 0.361380 | `workbooks_delete` | ❌ | -| 4 | 0.329149 | `workbooks_show` | ❌ | -| 5 | 0.328127 | `workbooks_list` | ❌ | +| 1 | 0.879420 | `foundry_openai_chat-completions-create` | ✅ **EXPECTED** | +| 2 | 0.852761 | `foundry_openai_create-completion` | ❌ | +| 3 | 0.814358 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.796340 | `foundry_threads_get-messages` | ❌ | +| 5 | 0.794456 | `foundry_openai_embeddings-create` | ❌ | --- ## Test 468 -**Expected Tool:** `workbooks_delete` -**Prompt:** Delete the workbook with resource ID +**Expected Tool:** `foundry_openai_create-completion` +**Prompt:** Create a completion with the prompt "What is Azure?" using my Microsoft Foundry resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.621310 | `workbooks_delete` | ✅ **EXPECTED** | -| 2 | 0.498518 | `workbooks_show` | ❌ | -| 3 | 0.432454 | `workbooks_create` | ❌ | -| 4 | 0.425569 | `workbooks_list` | ❌ | -| 5 | 0.421897 | `workbooks_update` | ❌ | +| 1 | 0.894548 | `foundry_openai_create-completion` | ✅ **EXPECTED** | +| 2 | 0.857348 | `foundry_openai_chat-completions-create` | ❌ | +| 3 | 0.816217 | `foundry_openai_embeddings-create` | ❌ | +| 4 | 0.802719 | `foundry_agents_get-sdk-sample` | ❌ | +| 5 | 0.788438 | `foundry_resource_get` | ❌ | --- ## Test 469 -**Expected Tool:** `workbooks_list` -**Prompt:** List all workbooks in my resource group +**Expected Tool:** `foundry_openai_embeddings-create` +**Prompt:** Generate embeddings for the text "Azure OpenAI Service" using my Microsoft Foundry resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.772430 | `workbooks_list` | ✅ **EXPECTED** | -| 2 | 0.562485 | `workbooks_create` | ❌ | -| 3 | 0.516739 | `grafana_list` | ❌ | -| 4 | 0.494073 | `workbooks_show` | ❌ | -| 5 | 0.488599 | `group_list` | ❌ | +| 1 | 0.929721 | `foundry_openai_embeddings-create` | ✅ **EXPECTED** | +| 2 | 0.868312 | `foundry_openai_create-completion` | ❌ | +| 3 | 0.846850 | `foundry_openai_chat-completions-create` | ❌ | +| 4 | 0.844258 | `foundry_models_deploy` | ❌ | +| 5 | 0.838538 | `foundry_openai_models-list` | ❌ | --- ## Test 470 -**Expected Tool:** `workbooks_list` -**Prompt:** What workbooks do I have in resource group ? +**Expected Tool:** `foundry_openai_embeddings-create` +**Prompt:** Create vector embeddings for my text using my Microsoft Foundry resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.708612 | `workbooks_list` | ✅ **EXPECTED** | -| 2 | 0.570260 | `workbooks_create` | ❌ | -| 3 | 0.499716 | `workbooks_show` | ❌ | -| 4 | 0.485504 | `workbooks_delete` | ❌ | -| 5 | 0.472378 | `grafana_list` | ❌ | +| 1 | 0.911299 | `foundry_openai_embeddings-create` | ✅ **EXPECTED** | +| 2 | 0.825539 | `foundry_openai_create-completion` | ❌ | +| 3 | 0.808836 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.807193 | `foundry_models_deploy` | ❌ | +| 5 | 0.803516 | `foundry_openai_chat-completions-create` | ❌ | --- ## Test 471 -**Expected Tool:** `workbooks_show` -**Prompt:** Get information about the workbook with resource ID +**Expected Tool:** `foundry_openai_models-list` +**Prompt:** List all available OpenAI models in my Microsoft Foundry resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.686087 | `workbooks_show` | ✅ **EXPECTED** | -| 2 | 0.498390 | `workbooks_create` | ❌ | -| 3 | 0.494708 | `workbooks_list` | ❌ | -| 4 | 0.463156 | `workbooks_update` | ❌ | -| 5 | 0.452348 | `workbooks_delete` | ❌ | +| 1 | 0.910699 | `foundry_openai_models-list` | ✅ **EXPECTED** | +| 2 | 0.859678 | `foundry_models_deploy` | ❌ | +| 3 | 0.855422 | `foundry_resource_get` | ❌ | +| 4 | 0.854975 | `foundry_models_deployments_list` | ❌ | +| 5 | 0.854210 | `foundry_openai_embeddings-create` | ❌ | --- ## Test 472 -**Expected Tool:** `workbooks_show` -**Prompt:** Show me the workbook with resource ID +**Expected Tool:** `foundry_openai_models-list` +**Prompt:** Show me the OpenAI model deployments in my Microsoft Foundry resource ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.583023 | `workbooks_show` | ✅ **EXPECTED** | -| 2 | 0.501484 | `workbooks_list` | ❌ | -| 3 | 0.469602 | `workbooks_create` | ❌ | -| 4 | 0.465829 | `workbooks_update` | ❌ | -| 5 | 0.455452 | `workbooks_delete` | ❌ | +| 1 | 0.907173 | `foundry_openai_models-list` | ✅ **EXPECTED** | +| 2 | 0.883055 | `foundry_models_deploy` | ❌ | +| 3 | 0.877723 | `foundry_models_deployments_list` | ❌ | +| 4 | 0.872315 | `foundry_openai_embeddings-create` | ❌ | +| 5 | 0.860925 | `foundry_openai_create-completion` | ❌ | --- ## Test 473 -**Expected Tool:** `workbooks_update` -**Prompt:** Update the workbook with a new text step +**Expected Tool:** `foundry_resource_get` +**Prompt:** List all Microsoft Foundry resources in my subscription ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.586347 | `workbooks_update` | ✅ **EXPECTED** | -| 2 | 0.382651 | `workbooks_create` | ❌ | -| 3 | 0.349689 | `workbooks_delete` | ❌ | -| 4 | 0.347944 | `workbooks_show` | ❌ | -| 5 | 0.292904 | `loadtesting_testrun_update` | ❌ | +| 1 | 0.864306 | `foundry_resource_get` | ✅ **EXPECTED** | +| 2 | 0.842775 | `foundry_openai_models-list` | ❌ | +| 3 | 0.823936 | `redis_list` | ❌ | +| 4 | 0.823880 | `foundry_models_deployments_list` | ❌ | +| 5 | 0.823588 | `foundry_threads_list` | ❌ | --- ## Test 474 -**Expected Tool:** `bicepschema_get` -**Prompt:** How can I use Bicep to create an Azure OpenAI service? +**Expected Tool:** `foundry_resource_get` +**Prompt:** Show me the Microsoft Foundry resources in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.543154 | `bicepschema_get` | ✅ **EXPECTED** | -| 2 | 0.485970 | `foundry_models_deploy` | ❌ | -| 3 | 0.485889 | `deploy_iac_rules_get` | ❌ | -| 4 | 0.477726 | `azureaibestpractices_get` | ❌ | -| 5 | 0.453282 | `foundry_openai_embeddings-create` | ❌ | +| 1 | 0.837913 | `foundry_resource_get` | ✅ **EXPECTED** | +| 2 | 0.826237 | `foundry_openai_models-list` | ❌ | +| 3 | 0.817387 | `foundry_agents_get-sdk-sample` | ❌ | +| 4 | 0.815888 | `foundry_openai_embeddings-create` | ❌ | +| 5 | 0.815298 | `foundry_models_deploy` | ❌ | --- ## Test 475 -**Expected Tool:** `cloudarchitect_design` -**Prompt:** Please help me design an architecture for a large-scale file upload, storage, and retrieval service +**Expected Tool:** `foundry_resource_get` +**Prompt:** Get details for Microsoft Foundry resource in resource group ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.502125 | `cloudarchitect_design` | ✅ **EXPECTED** | -| 2 | 0.290902 | `storage_blob_upload` | ❌ | -| 3 | 0.259162 | `managedlustre_fs_create` | ❌ | -| 4 | 0.254991 | `deploy_architecture_diagram_generate` | ❌ | -| 5 | 0.245034 | `managedlustre_fs_subnetsize_validate` | ❌ | +| 1 | 0.857772 | `foundry_resource_get` | ✅ **EXPECTED** | +| 2 | 0.833423 | `foundry_openai_models-list` | ❌ | +| 3 | 0.801578 | `foundry_openai_create-completion` | ❌ | +| 4 | 0.800082 | `foundry_openai_embeddings-create` | ❌ | +| 5 | 0.796833 | `foundry_agents_list` | ❌ | --- ## Test 476 -**Expected Tool:** `cloudarchitect_design` -**Prompt:** Help me design an Azure cloud service that will serve as an ATM for users +**Expected Tool:** `foundry_threads_create` +**Prompt:** Create a Microsoft Foundry thread to hold the conversation ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.508153 | `cloudarchitect_design` | ✅ **EXPECTED** | -| 2 | 0.377941 | `deploy_architecture_diagram_generate` | ❌ | -| 3 | 0.342110 | `azureaibestpractices_get` | ❌ | -| 4 | 0.341462 | `deploy_pipeline_guidance_get` | ❌ | -| 5 | 0.328747 | `get_bestpractices_get` | ❌ | +| 1 | 0.892131 | `foundry_threads_create` | ✅ **EXPECTED** | +| 2 | 0.887435 | `foundry_threads_get-messages` | ❌ | +| 3 | 0.878656 | `foundry_threads_list` | ❌ | +| 4 | 0.822267 | `foundry_agents_get-sdk-sample` | ❌ | +| 5 | 0.815628 | `foundry_agents_create` | ❌ | --- ## Test 477 -**Expected Tool:** `cloudarchitect_design` -**Prompt:** I want to design a cloud app for ordering groceries +**Expected Tool:** `foundry_threads_get-messages` +**Prompt:** Show me the messages in the Microsoft Foundry thread with id ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.423577 | `cloudarchitect_design` | ✅ **EXPECTED** | -| 2 | 0.271943 | `deploy_pipeline_guidance_get` | ❌ | -| 3 | 0.265972 | `deploy_architecture_diagram_generate` | ❌ | -| 4 | 0.249131 | `azureaibestpractices_get` | ❌ | -| 5 | 0.242581 | `deploy_plan_get` | ❌ | +| 1 | 0.905501 | `foundry_threads_get-messages` | ✅ **EXPECTED** | +| 2 | 0.867787 | `foundry_threads_list` | ❌ | +| 3 | 0.839016 | `foundry_threads_create` | ❌ | +| 4 | 0.810627 | `foundry_agents_get-sdk-sample` | ❌ | +| 5 | 0.796806 | `foundry_agents_connect` | ❌ | --- ## Test 478 -**Expected Tool:** `cloudarchitect_design` -**Prompt:** How can I design a cloud service in Azure that will store and present videos for users? +**Expected Tool:** `foundry_threads_list` +**Prompt:** List my Microsoft Foundry threads ### Results | Rank | Score | Tool | Status | |------|-------|------|--------| -| 1 | 0.534690 | `cloudarchitect_design` | ✅ **EXPECTED** | -| 2 | 0.369969 | `deploy_pipeline_guidance_get` | ❌ | -| 3 | 0.356331 | `managedlustre_fs_create` | ❌ | -| 4 | 0.352797 | `deploy_architecture_diagram_generate` | ❌ | -| 5 | 0.329229 | `azureaibestpractices_get` | ❌ | +| 1 | 0.920093 | `foundry_threads_list` | ✅ **EXPECTED** | +| 2 | 0.888431 | `foundry_threads_get-messages` | ❌ | +| 3 | 0.834885 | `foundry_threads_create` | ❌ | +| 4 | 0.830549 | `foundry_agents_get-sdk-sample` | ❌ | +| 5 | 0.795629 | `foundry_agents_connect` | ❌ | --- ## Summary **Total Prompts Tested:** 478 -**Analysis Execution Time:** 78.2111860s +**Analysis Execution Time:** 418.3437261s ### Success Rate Metrics -**Top Choice Success:** 91.4% (437/478 tests) +**Top Choice Success:** 83.5% (399/478 tests) #### Confidence Level Distribution -**💪 Very High Confidence (≥0.8):** 3.1% (15/478 tests) -**🎯 High Confidence (≥0.7):** 22.2% (106/478 tests) -**✅ Good Confidence (≥0.6):** 60.5% (289/478 tests) -**👍 Fair Confidence (≥0.5):** 91.0% (435/478 tests) -**👌 Acceptable Confidence (≥0.4):** 98.7% (472/478 tests) -**❌ Low Confidence (<0.4):** 1.3% (6/478 tests) +**💪 Very High Confidence (≥0.8):** 92.3% (441/478 tests) +**🎯 High Confidence (≥0.7):** 99.6% (476/478 tests) +**✅ Good Confidence (≥0.6):** 99.6% (476/478 tests) +**👍 Fair Confidence (≥0.5):** 99.6% (476/478 tests) +**👌 Acceptable Confidence (≥0.4):** 99.6% (476/478 tests) +**❌ Low Confidence (<0.4):** 0.4% (2/478 tests) #### Top Choice + Confidence Combinations -**💪 Top Choice + Very High Confidence (≥0.8):** 3.1% (15/478 tests) -**🎯 Top Choice + High Confidence (≥0.7):** 22.2% (106/478 tests) -**✅ Top Choice + Good Confidence (≥0.6):** 58.6% (280/478 tests) -**👍 Top Choice + Fair Confidence (≥0.5):** 85.6% (409/478 tests) -**👌 Top Choice + Acceptable Confidence (≥0.4):** 91.4% (437/478 tests) +**💪 Top Choice + Very High Confidence (≥0.8):** 78.9% (377/478 tests) +**🎯 Top Choice + High Confidence (≥0.7):** 83.5% (399/478 tests) +**✅ Top Choice + Good Confidence (≥0.6):** 83.5% (399/478 tests) +**👍 Top Choice + Fair Confidence (≥0.5):** 83.5% (399/478 tests) +**👌 Top Choice + Acceptable Confidence (≥0.4):** 83.5% (399/478 tests) ### Success Rate Analysis -🟢 **Excellent** - The tool selection system is performing exceptionally well. +🟡 **Good** - The tool selection system is performing well. -⚠️ **Recommendation:** Tool descriptions need improvement to better match user intent (targets: ≥0.6 good, ≥0.7 high). +🎯 **Recommendation:** Tool descriptions are highly optimized for user intent matching. diff --git a/eng/tools/ToolDescriptionEvaluator/tools.json b/eng/tools/ToolDescriptionEvaluator/tools.json index ac8d43a258..9249ce4ca0 100644 --- a/eng/tools/ToolDescriptionEvaluator/tools.json +++ b/eng/tools/ToolDescriptionEvaluator/tools.json @@ -1713,7 +1713,7 @@ }, { "name": "get", - "description": "Shows application logs specifically for Azure Developer CLI (azd) deployed applications from their associated Log Analytics workspace for Container Apps, App Services, and Function Apps. Designed exclusively for applications deployed via 'azd up' command and automatically discovers the correct workspace and resources based on the azd environment configuration. Use this tool to check deployment status or troubleshoot post-deployment issues.", + "description": "Shows application logs from Log Analytics workspace for Container Apps, App Services, and Function Apps. Designed to discover the correct workspace and resources for deployed applications based on the provided resource group name. Use this tool to check deployment status or troubleshoot post-deployment issues.", "command": "deploy app logs get", "option": [ { @@ -1765,14 +1765,8 @@ "required": null }, { - "name": "--workspace-folder", - "description": "The full path of the workspace folder.", - "type": "string", - "required": true - }, - { - "name": "--azd-env-name", - "description": "The name of the environment created by azd (AZURE_ENV_NAME) during `azd init` or `azd up`. If not provided in context, try to find it in the .azure directory in the workspace or use 'azd env list'.", + "name": "--resource-group", + "description": "The name of the resource group containing the application resources.", "type": "string", "required": true }, @@ -1799,21 +1793,9 @@ }, { "name": "get", - "description": "This tool offers guidelines for creating Bicep/Terraform files to deploy applications on Azure. The guidelines outline rules to improve the quality of Infrastructure as Code files, ensuring they are compatible with the azd tool and adhere to best practices.", + "description": "This tool offers guidelines for creating Infrastructure as Code (IaC) files to deploy applications on Azure using Azure CLI. The guidelines outline rules to improve the quality of IaC files, ensuring they are compatible with Azure CLI deployment workflows.", "command": "deploy iac rules get", "option": [ - { - "name": "--deployment-tool", - "description": "The deployment tool to use. Valid values: AZD, AzCli", - "type": "string", - "required": true - }, - { - "name": "--iac-type", - "description": "The Infrastructure as Code type. Valid values: bicep, terraform. Leave empty if deployment-tool is AzCli.", - "type": "string", - "required": null - }, { "name": "--resource-types", "description": "Specifies the Azure resource types to retrieve IaC rules for. It should be comma-separated. Supported values are: 'appservice', 'containerapp', 'function', 'aks', 'storage'. If none of these services are used, this parameter can be left empty.", @@ -1875,12 +1857,6 @@ "type": "string", "required": null }, - { - "name": "--use-azd-pipeline-config", - "description": "Whether to use azd tool to set up the deployment pipeline. Set to true ONLY if azure.yaml is provided or the context suggests AZD tools.", - "type": "string", - "required": null - }, { "name": "--organization-name", "description": "The name of the organization or the user account name of the current Github repository. DO NOT fill this in if you're not sure.", @@ -1903,7 +1879,7 @@ }, { "name": "get", - "description": "Generates a deployment plan to construct the infrastructure and deploy the application on Azure. Agent should read its output and generate a deploy plan in '.azure/plan.copilotmd' for execution steps, recommended azure services based on the information agent detected from project. Before calling this tool, please scan this workspace to detect the services to deploy and their dependent services.", + "description": "Generates a deployment plan to construct the infrastructure and deploy the application on Azure. Agent should read its output and generate a deploy plan in 'docs/plan.copilotmd' for execution steps, recommended azure services based on the information agent detected from project. Before calling this tool, please scan this workspace to detect the services to deploy and their dependent services.", "command": "deploy plan get", "option": [ { @@ -1923,18 +1899,6 @@ "description": "The Azure service to deploy the application. Valid values: ContainerApp, WebApp, FunctionApp, AKS. Recommend one based on user application.", "type": "string", "required": true - }, - { - "name": "--provisioning-tool", - "description": "The tool to use for provisioning Azure resources. Valid values: AZD, AzCli. Use AzCli if TargetAppService is AKS.", - "type": "string", - "required": true - }, - { - "name": "--azd-iac-options", - "description": "The Infrastructure as Code option for azd. Valid values: bicep, terraform. Leave empty if Deployment tool is AzCli.", - "type": "string", - "required": null } ] }, @@ -3065,7 +3029,7 @@ }, { "name": "connect", - "description": "Query an Microsoft Foundry agent and get the response as is (no query and evaluate). Use for one-off interaction or to capture run/thread IDs before calling evaluation tools. Do not use this tool for combined answer-plus-score workflows - instead, use agents_query-and-evaluate or agents_evaluate.", + "description": "Query a Microsoft Foundry agent and get the response as is (no query and evaluate). Use for one-off interaction or to capture run/thread IDs before calling evaluation tools. Do not use this tool for combined answer-plus-score workflows - instead, use agents_query-and-evaluate or agents_evaluate.", "command": "foundry agents connect", "option": [ { @@ -3132,7 +3096,7 @@ }, { "name": "create", - "description": " Creates an Microsoft Foundry Agent that processes messages according to a given system instruction using an existing Microsoft Foundry model deployment.", + "description": " Creates a Microsoft Foundry Agent that processes messages according to a given system instruction using an existing Microsoft Foundry model deployment.", "command": "foundry agents create", "option": [ { @@ -3303,7 +3267,7 @@ }, { "name": "list", - "description": "List all Azure AI Agents in an Microsoft Foundry project. Shows agents that can be used for AI workflows, \r\nevaluations, and interactive tasks. Requires the project endpoint URL (format: https://.services.ai.azure.com/api/projects/).", + "description": "List all Azure AI Agents in a Microsoft Foundry project. Shows agents that can be used for AI workflows, \r\nevaluations, and interactive tasks. Requires the project endpoint URL (format: https://.services.ai.azure.com/api/projects/).", "command": "foundry agents list", "option": [ { @@ -3680,7 +3644,7 @@ }, { "name": "list", - "description": "List model deployments in an Microsoft Foundry (Cognitive Services) project. Shows currently deployed AI models at the project level. \r\nUse this to audit what models are deployed before invoking or creating new deployments. Requires the project \r\nendpoint URL (format: https://.services.ai.azure.com/api/projects/). Note: This lists \r\ndeployed models only - use models_list to discover available catalog/base models.", + "description": "List model deployments in a Microsoft Foundry (Cognitive Services) project. Shows currently deployed AI models at the project level. \r\nUse this to audit what models are deployed before invoking or creating new deployments. Requires the project \r\nendpoint URL (format: https://.services.ai.azure.com/api/projects/). Note: This lists \r\ndeployed models only - use models_list to discover available catalog/base models.", "command": "foundry models deployments list", "option": [ { @@ -4129,7 +4093,7 @@ }, { "name": "models-list", - "description": "List all available Azure OpenAI models and deployments in an Microsoft Foundry resource. This tool retrieves information \r\nabout Azure OpenAI models deployed in your Microsoft Foundry resource including model names, versions, capabilities, \r\nand deployment status. Use this when you need to see what OpenAI models are available, check model deployments, \r\nor list Azure OpenAI models in your foundry resource. Returns model information as JSON array. Requires resource-name.", + "description": "List all available Azure OpenAI models and deployments in a Microsoft Foundry resource. This tool retrieves information \r\nabout Azure OpenAI models deployed in your Microsoft Foundry resource including model names, versions, capabilities, \r\nand deployment status. Use this when you need to see what OpenAI models are available, check model deployments, \r\nor list Azure OpenAI models in your foundry resource. Returns model information as JSON array. Requires resource-name.", "command": "foundry openai models-list", "option": [ { @@ -4263,7 +4227,7 @@ }, { "name": "create", - "description": " Creates an Microsoft Foundry Agent Thread that holds the messages between the Agent and the user.", + "description": " Creates a Microsoft Foundry Agent Thread that holds the messages between the Agent and the user.", "command": "foundry threads create", "option": [ { @@ -4324,7 +4288,7 @@ }, { "name": "get-messages", - "description": " Get messages in an Microsoft Foundry Agent Thread.", + "description": " Get messages in a Microsoft Foundry Agent Thread.", "command": "foundry threads get-messages", "option": [ { @@ -9961,6 +9925,97 @@ } ] }, + { + "name": "create", + "description": "Create a new Azure Managed Redis resource in Azure. Use this command to provision a new Redis resource in your subscription.", + "command": "redis create", + "option": [ + { + "name": "--tenant", + "description": "The Microsoft Entra ID tenant ID or name. This can be either the GUID identifier or the display name of your Entra ID tenant.", + "type": "string", + "required": null + }, + { + "name": "--auth-method", + "description": "Authentication method to use. Options: 'credential' (Azure CLI/managed identity), 'key' (access key), or 'connectionString'.", + "type": "string", + "required": null + }, + { + "name": "--retry-delay", + "description": "Initial delay in seconds between retry attempts. For exponential backoff, this value is used as the base.", + "type": "string", + "required": null + }, + { + "name": "--retry-max-delay", + "description": "Maximum delay in seconds between retries, regardless of the retry strategy.", + "type": "string", + "required": null + }, + { + "name": "--retry-max-retries", + "description": "Maximum number of retry attempts for failed operations before giving up.", + "type": "string", + "required": null + }, + { + "name": "--retry-mode", + "description": "Retry strategy to use. 'fixed' uses consistent delays, 'exponential' increases delay between attempts.", + "type": "string", + "required": null + }, + { + "name": "--retry-network-timeout", + "description": "Network operation timeout in seconds. Operations taking longer than this will be cancelled.", + "type": "string", + "required": null + }, + { + "name": "--subscription", + "description": "Specifies the Azure subscription to use. Accepts either a subscription ID (GUID) or display name. If not specified, the AZURE_SUBSCRIPTION_ID environment variable will be used instead.", + "type": "string", + "required": null + }, + { + "name": "--resource", + "description": "The name of the Redis resource (e.g., my-redis).", + "type": "string", + "required": true + }, + { + "name": "--resource-group", + "description": "The name of the Azure resource group. This is a logical container for Azure resources.", + "type": "string", + "required": true + }, + { + "name": "--sku", + "description": "The SKU for the Redis resource. (Default: Balanced_B0)", + "type": "string", + "required": null + }, + { + "name": "--location", + "description": "The location for the Redis resource (e.g. eastus).", + "type": "string", + "required": true + }, + { + "name": "--access-keys-authentication", + "description": "Whether to enable access keys for authentication for the Redis resource. (Default: false)", + "type": "string", + "required": null + }, + { + "name": "--modules", + "description": "A list of modules to enable on the Azure Managed Redis resource (e.g., RedisBloom, RedisJSON).", + "type": "string", + "required": null + } + ] + }, { "name": "list", "description": "List/show all Redis resources in a subscription. Returns details of all Azure Managed Redis, Azure Cache for Redis, and Azure Redis Enterprise resources. Use this command to explore and view which Redis resources are available in your subscription.", @@ -13316,5 +13371,5 @@ } ], "consolidated_tools": null, - "duration": 60 + "duration": 51 } \ No newline at end of file diff --git a/servers/Azure.Mcp.Server/CHANGELOG.md b/servers/Azure.Mcp.Server/CHANGELOG.md index 20d9a2a6a1..97697dc921 100644 --- a/servers/Azure.Mcp.Server/CHANGELOG.md +++ b/servers/Azure.Mcp.Server/CHANGELOG.md @@ -12,6 +12,7 @@ The Azure MCP Server updates automatically by default whenever a new release com - Removed usage of `writeIndented = true` (pretty printing) from `JsonSourceGenerationOptions` to reduce token usage. [[#1226](https://github.com/microsoft/mcp/pull/1226)] - Updated .NET tooling to stable version: `10.0.100-preview.7.25380.108` → `10.0.100`. [[#1243](https://github.com/microsoft/mcp/pull/1243)] +- Migrate azd tool out of C2C Deploy. [[#1108](https://github.com/microsoft/mcp/pull/1108)] ## 2.0.0-beta.6 (2025-11-20) @@ -194,6 +195,7 @@ For a complete history of pre-release changes, see versions [0.9.9](#099-2025-10 - Set telemetry fields for `ToolArea` and `ToolName` when "single" mode is used. [[#952](https://github.com/microsoft/mcp/pull/952)] - Added instructions on when to not use azd init [[#942](https://github.com/microsoft/mcp/pull/942)] + ## 0.9.8 (2025-10-23) ### Features Added diff --git a/servers/Azure.Mcp.Server/docs/azmcp-commands.md b/servers/Azure.Mcp.Server/docs/azmcp-commands.md index 4c7c897553..cc1ae984ed 100644 --- a/servers/Azure.Mcp.Server/docs/azmcp-commands.md +++ b/servers/Azure.Mcp.Server/docs/azmcp-commands.md @@ -897,10 +897,9 @@ azmcp postgres server param set --subscription \ ### Azure Deploy Operations ```bash -# Get the application service log for a specific azd environment +# Get the application service log for a specific resource group # ❌ Destructive | ✅ Idempotent | ❌ OpenWorld | ✅ ReadOnly | ❌ Secret | ❌ LocalRequired -azmcp deploy app logs get --workspace-folder \ - --azd-env-name \ +azmcp deploy app logs get --resource-group \ [--limit ] # Generate a mermaid architecture diagram for the application topology follow the schema defined in [deploy-app-topology-schema.json](../areas/deploy/src/AzureMcp.Deploy/Schemas/deploy-app-topology-schema.json) @@ -915,8 +914,7 @@ azmcp deploy iac rules get --deployment-tool \ # Get the ci/cd pipeline guidance # ❌ Destructive | ✅ Idempotent | ❌ OpenWorld | ✅ ReadOnly | ❌ Secret | ❌ LocalRequired -azmcp deploy pipeline guidance get [--use-azd-pipeline-config ] \ - [--organization-name ] \ +azmcp deploy pipeline guidance get [--organization-name ] \ [--repository-name ] \ [--github-environment-name ] @@ -924,9 +922,7 @@ azmcp deploy pipeline guidance get [--use-azd-pipeline-config \ --project-name \ - --target-app-service \ - --provisioning-tool \ - [--azd-iac-options ] + --target-app-service ``` ### Azure Event Grid Operations diff --git a/servers/Azure.Mcp.Server/docs/e2eTestPrompts.md b/servers/Azure.Mcp.Server/docs/e2eTestPrompts.md index e25f7ee219..da393e86f0 100644 --- a/servers/Azure.Mcp.Server/docs/e2eTestPrompts.md +++ b/servers/Azure.Mcp.Server/docs/e2eTestPrompts.md @@ -242,9 +242,9 @@ This file contains prompts used for end-to-end testing to ensure each tool is in | Tool Name | Test Prompt | |:----------|:----------| -| deploy_app_logs_get | Show me the log of the application deployed by azd | +| deploy_app_logs_get | Show me the log of the application in resource group | | deploy_architecture_diagram_generate | Generate the azure architecture diagram for this application | -| deploy_iac_rules_get | Show me the rules to generate bicep scripts | +| deploy_iac_rules_get | Show me the rules to generate scripts | | deploy_pipeline_guidance_get | How can I create a CI/CD pipeline to deploy this app to Azure? | | deploy_plan_get | Create a plan to deploy this application to azure | diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Commands/App/LogsGetCommand.cs b/tools/Azure.Mcp.Tools.Deploy/src/Commands/App/LogsGetCommand.cs index 809c0bc258..261a5fafa8 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Commands/App/LogsGetCommand.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Commands/App/LogsGetCommand.cs @@ -13,7 +13,7 @@ namespace Azure.Mcp.Tools.Deploy.Commands.App; public sealed class LogsGetCommand(ILogger logger) : SubscriptionCommand() { - private const string CommandTitle = "Get AZD deployed App Logs"; + private const string CommandTitle = "Get Application Logs"; private readonly ILogger _logger = logger; public override string Id => "ce9d648d-7c76-48a0-8cba-b9b57c6fd00b"; @@ -31,23 +31,21 @@ public sealed class LogsGetCommand(ILogger logger) : Subscriptio public override string Description => """ - Shows application logs specifically for Azure Developer CLI (azd) deployed applications from their associated Log Analytics workspace for Container Apps, App Services, and Function Apps. Designed exclusively for applications deployed via 'azd up' command and automatically discovers the correct workspace and resources based on the azd environment configuration. Use this tool to check deployment status or troubleshoot post-deployment issues. + Shows application logs from Log Analytics workspace for Container Apps, App Services, and Function Apps. Designed to discover the correct workspace and resources for deployed applications based on the provided resource group name. Use this tool to check deployment status or troubleshoot post-deployment issues. """; protected override void RegisterOptions(Command command) { base.RegisterOptions(command); - command.Options.Add(DeployOptionDefinitions.AzdAppLogOptions.WorkspaceFolder); - command.Options.Add(DeployOptionDefinitions.AzdAppLogOptions.AzdEnvName); - command.Options.Add(DeployOptionDefinitions.AzdAppLogOptions.Limit); + command.Options.Add(DeployOptionDefinitions.AppLogOptions.ResourceGroupName); + command.Options.Add(DeployOptionDefinitions.AppLogOptions.Limit); } protected override LogsGetOptions BindOptions(ParseResult parseResult) { var options = base.BindOptions(parseResult); - options.WorkspaceFolder = parseResult.GetValueOrDefault(DeployOptionDefinitions.AzdAppLogOptions.WorkspaceFolder.Name)!; - options.AzdEnvName = parseResult.GetValueOrDefault(DeployOptionDefinitions.AzdAppLogOptions.AzdEnvName.Name)!; - options.Limit = parseResult.GetValueOrDefault(DeployOptionDefinitions.AzdAppLogOptions.Limit.Name); + options.ResourceGroup = parseResult.GetValueOrDefault(DeployOptionDefinitions.AppLogOptions.ResourceGroupName.Name); + options.Limit = parseResult.GetValueOrDefault(DeployOptionDefinitions.AppLogOptions.Limit.Name); return options; } @@ -65,17 +63,16 @@ public override async Task ExecuteAsync(CommandContext context, var deployService = context.GetService(); - string result = await deployService.GetAzdResourceLogsAsync( - options.WorkspaceFolder!, - options.AzdEnvName!, + string result = await deployService.GetResourceLogsAsync( options.Subscription!, + options.ResourceGroup!, options.Limit, cancellationToken); context.Response.Message = result; } catch (Exception ex) { - _logger.LogError(ex, "An exception occurred getting azd app logs."); + _logger.LogError(ex, "An exception occurred getting app logs."); HandleException(context, ex); } diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Commands/Infrastructure/RulesGetCommand.cs b/tools/Azure.Mcp.Tools.Deploy/src/Commands/Infrastructure/RulesGetCommand.cs index 83d1d0bd39..2f73bbcb22 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Commands/Infrastructure/RulesGetCommand.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Commands/Infrastructure/RulesGetCommand.cs @@ -6,6 +6,7 @@ using Azure.Mcp.Tools.Deploy.Models; using Azure.Mcp.Tools.Deploy.Options; using Azure.Mcp.Tools.Deploy.Options.Infrastructure; +using Azure.Mcp.Tools.Deploy.Services.Templates; using Azure.Mcp.Tools.Deploy.Services.Util; using Microsoft.Extensions.Logging; @@ -32,22 +33,18 @@ public sealed class RulesGetCommand(ILogger logger) public override string Description => """ - This tool offers guidelines for creating Bicep/Terraform files to deploy applications on Azure. The guidelines outline rules to improve the quality of Infrastructure as Code files, ensuring they are compatible with the azd tool and adhere to best practices. + This tool offers guidelines for creating Infrastructure as Code (IaC) files to deploy applications on Azure using Azure CLI. The guidelines outline rules to improve the quality of IaC files, ensuring they are compatible with Azure CLI deployment workflows. """; protected override void RegisterOptions(Command command) { base.RegisterOptions(command); - command.Options.Add(DeployOptionDefinitions.IaCRules.DeploymentTool); - command.Options.Add(DeployOptionDefinitions.IaCRules.IacType); command.Options.Add(DeployOptionDefinitions.IaCRules.ResourceTypes); } protected override RulesGetOptions BindOptions(ParseResult parseResult) { var options = new RulesGetOptions(); - options.DeploymentTool = parseResult.GetValueOrDefault(DeployOptionDefinitions.IaCRules.DeploymentTool.Name) ?? string.Empty; - options.IacType = parseResult.GetValueOrDefault(DeployOptionDefinitions.IaCRules.IacType.Name) ?? string.Empty; options.ResourceTypes = parseResult.GetValueOrDefault(DeployOptionDefinitions.IaCRules.ResourceTypes.Name) ?? string.Empty; return options; } @@ -64,19 +61,9 @@ public override Task ExecuteAsync(CommandContext context, Parse try { context.Activity? - .AddTag(DeployTelemetryTags.DeploymentTool, options.DeploymentTool) - .AddTag(DeployTelemetryTags.IacType, options.IacType) .AddTag(DeployTelemetryTags.ComputeHostResources, options.ResourceTypes); - var resourceTypes = options.ResourceTypes.Split(',') - .Select(rt => rt.Trim()) - .Where(rt => !string.IsNullOrWhiteSpace(rt)) - .ToArray(); - - string iacRules = IaCRulesTemplateUtil.GetIaCRules( - options.DeploymentTool, - options.IacType, - resourceTypes); + string iacRules = TemplateService.LoadTemplate("IaCRules/azcli-rules"); context.Response.Message = iacRules; } diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Commands/Pipeline/GuidanceGetCommand.cs b/tools/Azure.Mcp.Tools.Deploy/src/Commands/Pipeline/GuidanceGetCommand.cs index 6931757309..b3567f33d5 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Commands/Pipeline/GuidanceGetCommand.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Commands/Pipeline/GuidanceGetCommand.cs @@ -40,7 +40,6 @@ public sealed class GuidanceGetCommand(ILogger logger) protected override void RegisterOptions(Command command) { base.RegisterOptions(command); - command.Options.Add(DeployOptionDefinitions.PipelineGenerateOptions.UseAZDPipelineConfig); command.Options.Add(DeployOptionDefinitions.PipelineGenerateOptions.OrganizationName); command.Options.Add(DeployOptionDefinitions.PipelineGenerateOptions.RepositoryName); command.Options.Add(DeployOptionDefinitions.PipelineGenerateOptions.GithubEnvironmentName); @@ -49,7 +48,6 @@ protected override void RegisterOptions(Command command) protected override GuidanceGetOptions BindOptions(ParseResult parseResult) { var options = base.BindOptions(parseResult); - options.UseAZDPipelineConfig = parseResult.GetValueOrDefault(DeployOptionDefinitions.PipelineGenerateOptions.UseAZDPipelineConfig.Name); options.OrganizationName = parseResult.GetValueOrDefault(DeployOptionDefinitions.PipelineGenerateOptions.OrganizationName.Name); options.RepositoryName = parseResult.GetValueOrDefault(DeployOptionDefinitions.PipelineGenerateOptions.RepositoryName.Name); options.GithubEnvironmentName = parseResult.GetValueOrDefault(DeployOptionDefinitions.PipelineGenerateOptions.GithubEnvironmentName.Name); diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Commands/Plan/GetCommand.cs b/tools/Azure.Mcp.Tools.Deploy/src/Commands/Plan/GetCommand.cs index e822324345..304c1cdd64 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Commands/Plan/GetCommand.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Commands/Plan/GetCommand.cs @@ -25,7 +25,7 @@ public sealed class GetCommand(ILogger logger) public override string Description => """ - Generates a deployment plan to construct the infrastructure and deploy the application on Azure. Agent should read its output and generate a deploy plan in '.azure/plan.copilotmd' for execution steps, recommended azure services based on the information agent detected from project. Before calling this tool, please scan this workspace to detect the services to deploy and their dependent services. + Generates a deployment plan to construct the infrastructure and deploy the application on Azure. Agent should read its output and generate a deploy plan in 'docs/plan.copilotmd' for execution steps, recommended azure services based on the information agent detected from project. Before calling this tool, please scan this workspace to detect the services to deploy and their dependent services. """; public override string Title => CommandTitle; @@ -45,8 +45,6 @@ protected override void RegisterOptions(Command command) command.Options.Add(DeployOptionDefinitions.PlanGet.WorkspaceFolder); command.Options.Add(DeployOptionDefinitions.PlanGet.ProjectName); command.Options.Add(DeployOptionDefinitions.PlanGet.TargetAppService); - command.Options.Add(DeployOptionDefinitions.PlanGet.ProvisioningTool); - command.Options.Add(DeployOptionDefinitions.PlanGet.AzdIacOptions); } protected override GetOptions BindOptions(ParseResult parseResult) @@ -55,9 +53,7 @@ protected override GetOptions BindOptions(ParseResult parseResult) { WorkspaceFolder = parseResult.GetValueOrDefault(DeployOptionDefinitions.PlanGet.WorkspaceFolder.Name) ?? string.Empty, ProjectName = parseResult.GetValueOrDefault(DeployOptionDefinitions.PlanGet.ProjectName.Name) ?? string.Empty, - TargetAppService = parseResult.GetValueOrDefault(DeployOptionDefinitions.PlanGet.TargetAppService.Name) ?? string.Empty, - ProvisioningTool = parseResult.GetValueOrDefault(DeployOptionDefinitions.PlanGet.ProvisioningTool.Name) ?? string.Empty, - AzdIacOptions = parseResult.GetValueOrDefault(DeployOptionDefinitions.PlanGet.AzdIacOptions.Name) ?? string.Empty + TargetAppService = parseResult.GetValueOrDefault(DeployOptionDefinitions.PlanGet.TargetAppService.Name) ?? string.Empty }; } @@ -78,11 +74,9 @@ public override Task ExecuteAsync(CommandContext context, Parse context.Activity?.AddTag(DeployTelemetryTags.ProjectName, BitConverter.ToString(bytes).Replace("-", "").ToLowerInvariant()); } context.Activity? - .AddTag(DeployTelemetryTags.ComputeHostResources, options.TargetAppService) - .AddTag(DeployTelemetryTags.DeploymentTool, options.ProvisioningTool) - .AddTag(DeployTelemetryTags.IacType, options.AzdIacOptions ?? string.Empty); + .AddTag(DeployTelemetryTags.ComputeHostResources, options.TargetAppService); - var planTemplate = DeploymentPlanTemplateUtil.GetPlanTemplate(options.ProjectName, options.TargetAppService, options.ProvisioningTool, options.AzdIacOptions); + var planTemplate = DeploymentPlanTemplateUtil.GetPlanTemplate(options.ProjectName, options.TargetAppService); context.Response.Message = planTemplate; context.Response.Status = HttpStatusCode.OK; diff --git a/tools/Azure.Mcp.Tools.Deploy/src/DeploySetup.cs b/tools/Azure.Mcp.Tools.Deploy/src/DeploySetup.cs index 8ec59ae69e..eef1adf6aa 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/DeploySetup.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/DeploySetup.cs @@ -33,14 +33,13 @@ public void ConfigureServices(IServiceCollection services) public CommandGroup RegisterCommands(IServiceProvider serviceProvider) { var deploy = new CommandGroup(Name, "Deploy commands for deploying applications to Azure, including sub commands: " - + "- plan get: generates a deployment plan to construct the infrastructure and deploy the application on Azure. Agent should read its output and generate a deploy plan in '.azure/plan.copilotmd' for execution steps, recommended azure services based on the information agent detected from project. Before calling this tool, please scan this workspace to detect the services to deploy and their dependent services; " - + "- iac rules get: offers guidelines for creating Bicep/Terraform files to deploy applications on Azure; " - + "- app logs get: fetch logs from log analytics workspace for Container Apps, App Services, function apps that were deployed through azd; " + + "- plan get: generates a deployment plan to construct the infrastructure and deploy the application on Azure. Agent should read its output and generate a deploy plan in 'docs/plan.copilotmd' for execution steps, recommended azure services based on the information agent detected from project. Before calling this tool, please scan this workspace to detect the services to deploy and their dependent services; " + + "- iac rules get: offers guidelines for authoring infrastructure as code files to deploy applications on Azure; " + + "- app logs get: fetch logs from log analytics workspace for Container Apps, App Services, function apps; " + "- pipeline guidance get: guidance to create a CI/CD pipeline which provision Azure resources and build and deploy applications to Azure; " + "- architecture diagram generate: generates an azure service architecture diagram for the application based on the provided app topology; ", Title); // Application-specific commands - // This command will be deprecated when 'azd cli' supports the same functionality var appGroup = new CommandGroup("app", "Application-specific deployment tools"); var logsGroup = new CommandGroup("logs", "Application logs management"); var logsGet = serviceProvider.GetRequiredService(); diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Models/IaCRulesParameters.cs b/tools/Azure.Mcp.Tools.Deploy/src/Models/IaCRulesParameters.cs index a44221ea4d..a550c2f20a 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Models/IaCRulesParameters.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Models/IaCRulesParameters.cs @@ -3,7 +3,6 @@ namespace Azure.Mcp.Tools.Deploy.Models; public static class DeploymentTool { - public const string Azd = "AZD"; public const string AzCli = "AzCli"; } diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Models/Templates/DeploymentPlanTemplateParameters.cs b/tools/Azure.Mcp.Tools.Deploy/src/Models/Templates/DeploymentPlanTemplateParameters.cs index 323cb04694..8991bbdf40 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Models/Templates/DeploymentPlanTemplateParameters.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Models/Templates/DeploymentPlanTemplateParameters.cs @@ -23,16 +23,6 @@ public sealed class DeploymentPlanTemplateParameters /// public string TargetAppService { get; set; } = string.Empty; - /// - /// The provisioning tool (AZD, AzCli). - /// - public string ProvisioningTool { get; set; } = string.Empty; - - /// - /// The Infrastructure as Code type (bicep, terraform). - /// - public string IacType { get; set; } = string.Empty; - /// /// The Azure compute host display name. /// @@ -54,8 +44,6 @@ public Dictionary ToDictionary() { nameof(Title), Title }, { nameof(ProjectName), ProjectName }, { nameof(TargetAppService), TargetAppService }, - { nameof(ProvisioningTool), ProvisioningTool }, - { nameof(IacType), IacType }, { nameof(AzureComputeHost), AzureComputeHost }, { nameof(ExecutionSteps), ExecutionSteps }, }; diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Options/App/LogsGetOptions.cs b/tools/Azure.Mcp.Tools.Deploy/src/Options/App/LogsGetOptions.cs index f0aa5d30a2..1d31e7beb8 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Options/App/LogsGetOptions.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Options/App/LogsGetOptions.cs @@ -8,12 +8,6 @@ namespace Azure.Mcp.Tools.Deploy.Options.App; public class LogsGetOptions : SubscriptionOptions { - [JsonPropertyName("workspaceFolder")] - public string WorkspaceFolder { get; set; } = string.Empty; - - [JsonPropertyName("azdEnvName")] - public string AzdEnvName { get; set; } = string.Empty; - [JsonPropertyName("limit")] public int? Limit { get; set; } } diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Options/DeployOptionDefinitions.cs b/tools/Azure.Mcp.Tools.Deploy/src/Options/DeployOptionDefinitions.cs index df7c550a41..92b62e6180 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Options/DeployOptionDefinitions.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Options/DeployOptionDefinitions.cs @@ -22,25 +22,16 @@ public static class RawMcpToolInput }; } - public class AzdAppLogOptions : SubscriptionOptions + public class AppLogOptions : SubscriptionOptions { - public const string WorkspaceFolderName = "workspace-folder"; - public const string AzdEnvNameName = "azd-env-name"; + public const string ResourceGroupsName = "resource-group"; public const string LimitName = "limit"; - public static readonly Option WorkspaceFolder = new( - $"--{WorkspaceFolderName}" - ) - { - Description = "The full path of the workspace folder.", - Required = true - }; - - public static readonly Option AzdEnvName = new( - $"--{AzdEnvNameName}" + public static readonly Option ResourceGroupName = new( + $"--{ResourceGroupsName}" ) { - Description = "The name of the environment created by azd (AZURE_ENV_NAME) during `azd init` or `azd up`. If not provided in context, try to find it in the .azure directory in the workspace or use 'azd env list'.", + Description = "The name of the resource group containing the application resources.", Required = true }; @@ -56,20 +47,10 @@ public class AzdAppLogOptions : SubscriptionOptions public class PipelineGenerateOptions : SubscriptionOptions { - public const string UseAZDPipelineConfigName = "use-azd-pipeline-config"; public const string OrganizationNameName = "organization-name"; public const string RepositoryNameName = "repository-name"; public const string GithubEnvironmentNameName = "github-environment-name"; - public static readonly Option UseAZDPipelineConfig = new( - $"--{UseAZDPipelineConfigName}" - ) - { - Description = "Whether to use azd tool to set up the deployment pipeline. Set to true ONLY if azure.yaml is provided or the context suggests AZD tools.", - DefaultValueFactory = _ => false, - Required = false - }; - public static readonly Option OrganizationName = new( $"--{OrganizationNameName}" ) @@ -101,8 +82,6 @@ public static class PlanGet public const string WorkspaceFolderName = "workspace-folder"; public const string ProjectNameName = "project-name"; public const string TargetAppServiceName = "target-app-service"; - public const string ProvisioningToolName = "provisioning-tool"; - public const string AzdIacOptionsName = "azd-iac-options"; public static readonly Option WorkspaceFolder = new( $"--{WorkspaceFolderName}" @@ -127,42 +106,10 @@ public static class PlanGet Description = "The Azure service to deploy the application. Valid values: ContainerApp, WebApp, FunctionApp, AKS. Recommend one based on user application.", Required = true }; - - public static readonly Option ProvisioningTool = new( - $"--{ProvisioningToolName}" - ) - { - Description = "The tool to use for provisioning Azure resources. Valid values: AZD, AzCli. Use AzCli if TargetAppService is AKS.", - Required = true - }; - - public static readonly Option AzdIacOptions = new( - $"--{AzdIacOptionsName}" - ) - { - Description = "The Infrastructure as Code option for azd. Valid values: bicep, terraform. Leave empty if Deployment tool is AzCli.", - Required = false - }; } public static class IaCRules { - public static readonly Option DeploymentTool = new( - "--deployment-tool") - { - Description = "The deployment tool to use. Valid values: AZD, AzCli", - Required = true - }; - - public static readonly Option IacType = new( - "--iac-type" - ) - { - Description = "The Infrastructure as Code type. Valid values: bicep, terraform. Leave empty if deployment-tool is AzCli.", - Required = false - - - }; public static readonly Option ResourceTypes = new( "--resource-types") diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Options/Infrastructure/RulesGetOptions.cs b/tools/Azure.Mcp.Tools.Deploy/src/Options/Infrastructure/RulesGetOptions.cs index 8ccecd7924..3a83fbd347 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Options/Infrastructure/RulesGetOptions.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Options/Infrastructure/RulesGetOptions.cs @@ -5,7 +5,5 @@ namespace Azure.Mcp.Tools.Deploy.Options.Infrastructure; public sealed class RulesGetOptions { - public string DeploymentTool { get; set; } = string.Empty; - public string IacType { get; set; } = string.Empty; public string ResourceTypes { get; set; } = string.Empty; } diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Options/Pipeline/GuidanceGetOptions.cs b/tools/Azure.Mcp.Tools.Deploy/src/Options/Pipeline/GuidanceGetOptions.cs index 01767dd4da..1c3164c0f0 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Options/Pipeline/GuidanceGetOptions.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Options/Pipeline/GuidanceGetOptions.cs @@ -8,9 +8,6 @@ namespace Azure.Mcp.Tools.Deploy.Options.Pipeline; public class GuidanceGetOptions : SubscriptionOptions { - [JsonPropertyName("useAZDPipelineConfig")] - public bool UseAZDPipelineConfig { get; set; } - [JsonPropertyName("organizationName")] public string? OrganizationName { get; set; } diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Options/Plan/GetOptions.cs b/tools/Azure.Mcp.Tools.Deploy/src/Options/Plan/GetOptions.cs index ec2c7c2c97..838d488b86 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Options/Plan/GetOptions.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Options/Plan/GetOptions.cs @@ -15,10 +15,4 @@ public sealed class GetOptions [JsonPropertyName("targetAppService")] public string TargetAppService { get; set; } = string.Empty; - - [JsonPropertyName("provisioningTool")] - public string ProvisioningTool { get; set; } = string.Empty; - - [JsonPropertyName("azdIacOptions")] - public string? AzdIacOptions { get; set; } = string.Empty; } diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Services/DeployService.cs b/tools/Azure.Mcp.Tools.Deploy/src/Services/DeployService.cs index dc89e69a22..f745334a3c 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Services/DeployService.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Services/DeployService.cs @@ -10,19 +10,17 @@ namespace Azure.Mcp.Tools.Deploy.Services; public class DeployService(ITenantService tenantService) : BaseAzureService(tenantService), IDeployService { - public async Task GetAzdResourceLogsAsync( - string workspaceFolder, - string azdEnvName, + public async Task GetResourceLogsAsync( string subscriptionId, + string resourceGroupName, int? limit = null, CancellationToken cancellationToken = default) { TokenCredential credential = await GetCredential(cancellationToken); - string result = await AzdResourceLogService.GetAzdResourceLogsAsync( + string result = await ResourceLogService.GetResourceLogsAsync( credential, - workspaceFolder, - azdEnvName, subscriptionId, + resourceGroupName, limit); return result; } diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Services/IDeployService.cs b/tools/Azure.Mcp.Tools.Deploy/src/Services/IDeployService.cs index e3b05b6389..774d18a511 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Services/IDeployService.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Services/IDeployService.cs @@ -5,9 +5,8 @@ namespace Azure.Mcp.Tools.Deploy.Services; public interface IDeployService { - Task GetAzdResourceLogsAsync( - string workspaceFolder, - string azdEnvName, + Task GetResourceLogsAsync( string subscriptionId, + string resourceGroupName, int? limit = null, CancellationToken cancellationToken = default); } diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/AzdAppLogRetriever.cs b/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/AppLogRetriever.cs similarity index 75% rename from tools/Azure.Mcp.Tools.Deploy/src/Services/Util/AzdAppLogRetriever.cs rename to tools/Azure.Mcp.Tools.Deploy/src/Services/Util/AppLogRetriever.cs index 685dcaf7bf..2074834e4f 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/AzdAppLogRetriever.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/AppLogRetriever.cs @@ -8,14 +8,13 @@ namespace Azure.Mcp.Tools.Deploy.Services.Util; -public class AzdAppLogRetriever(TokenCredential credential, string subscriptionId, string azdEnvName) +public class AppLogRetriever(TokenCredential credential, string subscriptionId, string resourceGroupName) { private readonly string _subscriptionId = subscriptionId; - private readonly string _azdEnvName = azdEnvName; + private readonly string _resourceGroupName = resourceGroupName; private readonly Dictionary _apps = new(); private readonly Dictionary _logs = new(); private readonly List _logAnalyticsWorkspaceIds = new(); - private string _resourceGroupName = string.Empty; private ArmClient? _armClient; private LogsQueryClient? _queryClient; @@ -25,10 +24,12 @@ public async Task InitializeAsync() _armClient = new ArmClient(credential, _subscriptionId); _queryClient = new LogsQueryClient(credential); - _resourceGroupName = await GetResourceGroupNameAsync(); - if (string.IsNullOrEmpty(_resourceGroupName)) + // Verify resource group exists + var subscription = _armClient!.GetSubscriptionResource(new($"/subscriptions/{_subscriptionId}")); + var resourceGroupResponse = await subscription.GetResourceGroupAsync(_resourceGroupName); + if (resourceGroupResponse?.Value == null) { - throw new InvalidOperationException($"No resource group with tag {{\"azd-env-name\": {_azdEnvName}}} found."); + throw new InvalidOperationException($"Resource group '{_resourceGroupName}' not found in subscription '{_subscriptionId}'."); } } @@ -50,33 +51,62 @@ public async Task GetLogAnalyticsWorkspacesInfoAsync() } } - public async Task RegisterAppAsync(ResourceType resourceType, string serviceName) + private static bool IsResourceTypeMatch(GenericResource resource, ResourceType resourceType) + { + var resourceTypeString = resourceType.GetResourceTypeString(); + var parts = resourceTypeString.Split('|'); + var expectedType = parts[0]; + var expectedKind = parts.Length > 1 ? parts[1] : null; + + return resource.Data.ResourceType.ToString() == expectedType && + (expectedKind == null || resource.Data.Kind?.StartsWith(expectedKind) == true); + } + + public async Task> GetAllSupportedResourcesAsync() { var subscription = _armClient!.GetSubscriptionResource(new($"/subscriptions/{_subscriptionId}")); var resourceGroup = await subscription.GetResourceGroupAsync(_resourceGroupName); - var filter = $"tagName eq 'azd-service-name' and tagValue eq '{serviceName}'"; - var apps = new List(); + var resources = new List<(string name, ResourceType type)>(); - await foreach (var resource in resourceGroup.Value.GetGenericResourcesAsync(filter: filter)) + await foreach (var resource in resourceGroup.Value.GetGenericResourcesAsync()) { - var resourceTypeString = resourceType.GetResourceTypeString(); - var parts = resourceTypeString.Split('|'); - var type = parts[0]; - var kind = parts.Length > 1 ? parts[1] : null; + foreach (ResourceType resourceType in Enum.GetValues()) + { + if (IsResourceTypeMatch(resource, resourceType)) + { + resources.Add((resource.Data.Name, resourceType)); + break; + } + } + } - if (resource.Data.ResourceType.ToString() == type && - (kind == null || resource.Data.Kind?.StartsWith(kind) == true)) + return resources; + } + + public async Task RegisterAppAsync(ResourceType resourceType, string rgName) + { + var subscription = _armClient!.GetSubscriptionResource(new($"/subscriptions/{_subscriptionId}")); + var resourceGroup = await subscription.GetResourceGroupAsync(_resourceGroupName); + + var apps = new List(); + + await foreach (var resource in resourceGroup.Value.GetGenericResourcesAsync()) + { + if (resource.Data.Name.Equals(rgName, StringComparison.OrdinalIgnoreCase)) { - _logs[resource.Id.ToString()] = string.Empty; - apps.Add(resource); + if (IsResourceTypeMatch(resource, resourceType)) + { + _logs[resource.Id.ToString()] = string.Empty; + apps.Add(resource); + } } } return apps.Count switch { - 0 => throw new InvalidOperationException($"No resources found for resource type {resourceType} with tag azd-service-name={serviceName}"), - > 1 => throw new InvalidOperationException($"Multiple resources found for resource type {resourceType} with tag azd-service-name={serviceName}"), + 0 => throw new InvalidOperationException($"No resources found for resource type {resourceType} in resource group {rgName}"), + > 1 => throw new InvalidOperationException($"Multiple resources found for resource type {resourceType} in resource group {rgName}"), _ => apps[0] }; } @@ -90,9 +120,9 @@ private static string GetAppServiceLogsQuery(string appServiceResourceId, int li private static string GetFunctionAppLogsQuery(string functionAppName, int limit) => $"AppTraces | where AppRoleName == '{functionAppName}' | order by TimeGenerated desc | project TimeGenerated, Message | take {limit}"; - public async Task QueryAppLogsAsync(ResourceType resourceType, string serviceName, int? limit = null) + public async Task QueryAppLogsAsync(ResourceType resourceType, string resourceName, int? limit = null) { - var app = await RegisterAppAsync(resourceType, serviceName); + var app = await RegisterAppAsync(resourceType, resourceName); var getLogErrors = new List(); var getLogSuccess = false; var logSearchQuery = string.Empty; @@ -182,24 +212,8 @@ public async Task QueryAppLogsAsync(ResourceType resourceType, string se throw new InvalidOperationException($"Errors: {string.Join(", ", getLogErrors)}"); } - return $"Console Logs for {serviceName} with resource ID {app.Id} between {startTime} and {endTime}:\n{_logs[app.Id.ToString()]}"; + return $"Console Logs for {resourceName} with resource ID {app.Id} between {startTime} and {endTime}:\n{_logs[app.Id.ToString()]}"; } - - private async Task GetResourceGroupNameAsync() - { - var subscription = _armClient!.GetSubscriptionResource(new($"/subscriptions/{_subscriptionId}")); - - await foreach (var resourceGroup in subscription.GetResourceGroups()) - { - if (resourceGroup.Data.Tags.TryGetValue("azd-env-name", out var envName) && envName == _azdEnvName) - { - return resourceGroup.Data.Name; - } - } - - return string.Empty; - } - } public enum ResourceType diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/AzdResourceLogService.cs b/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/AzdResourceLogService.cs deleted file mode 100644 index 8d8637e4fb..0000000000 --- a/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/AzdResourceLogService.cs +++ /dev/null @@ -1,192 +0,0 @@ -using Azure.Core; -using YamlDotNet.Core; -using YamlDotNet.Core.Events; - -namespace Azure.Mcp.Tools.Deploy.Services.Util; - -public static class AzdResourceLogService -{ - private const string AzureYamlFileName = "azure.yaml"; - - public static async Task GetAzdResourceLogsAsync( - TokenCredential credential, - string workspaceFolder, - string azdEnvName, - string subscriptionId, - int? limit = null) - { - var toolErrorLogs = new List(); - var appLogs = new List(); - - try - { - var azdAppLogRetriever = new AzdAppLogRetriever(credential, subscriptionId, azdEnvName); - await azdAppLogRetriever.InitializeAsync(); - await azdAppLogRetriever.GetLogAnalyticsWorkspacesInfoAsync(); - - var services = GetServicesFromAzureYaml(workspaceFolder); - - foreach (var (serviceName, service) in services) - { - try - { - if (service.Host != null) - { - var resourceType = ResourceTypeExtensions.GetResourceTypeFromHost(service.Host); - var logs = await azdAppLogRetriever.QueryAppLogsAsync(resourceType, serviceName, limit); - appLogs.Add(logs); - } - } - catch (Exception ex) - { - toolErrorLogs.Add($"Error finding app logs for service {serviceName}: {ex.Message}"); - } - } - } - catch (Exception ex) - { - toolErrorLogs.Add(ex.Message); - } - - if (appLogs.Count > 0) - { - return $"App logs retrieved:\n{string.Join("\n\n", appLogs)}"; - } - - if (toolErrorLogs.Count > 0) - { - return $"Error during retrieval of app logs of azd project:\n{string.Join("\n", toolErrorLogs)}"; - } - - return "No logs found."; - } - - private static Dictionary GetServicesFromAzureYaml(string workspaceFolder) - { - var azureYamlPath = Path.Combine(workspaceFolder, AzureYamlFileName); - - if (!File.Exists(azureYamlPath)) - { - throw new FileNotFoundException($"Azure YAML file not found at {azureYamlPath}"); - } - - var yamlContent = File.ReadAllText(azureYamlPath); - - using var stringReader = new StringReader(yamlContent); - var parser = new Parser(stringReader); - - return ParseAzureYamlServices(parser); - } - - private static Dictionary ParseAzureYamlServices(Parser parser) - { - var result = new Dictionary(); - - parser.Consume(); - - parser.Consume(); - - parser.Consume(); - - while (parser.Accept(out _) == false) - { - var key = parser.Consume().Value; - - if (key == "services") - { - parser.Consume(); - - while (parser.Accept(out _) == false) - { - var serviceName = parser.Consume().Value; - - parser.Consume(); - - string? host = null; - string? project = null; - string? language = null; - - while (parser.Accept(out _) == false) - { - var propertyKey = parser.Consume().Value; - // Only accept properties host, project, and language which are scalars - if (parser.Accept(out _)) - { - var propertyValue = parser.Consume().Value; - switch (propertyKey) - { - case "host": - host = propertyValue; - break; - case "project": - project = propertyValue; - break; - case "language": - language = propertyValue; - break; - } - } - else - { - SkipValue(parser); - } - } - - parser.Consume(); - - result[serviceName] = new Service( - Host: host, - Project: project, - Language: language - ); - } - - parser.Consume(); - } - else - { - SkipValue(parser); - } - } - - if (result.Count == 0) - { - throw new InvalidOperationException("No services section found in azure.yaml"); - } - - return result; - } - - private static void SkipValue(Parser parser) - { - if (parser.Accept(out _)) - { - parser.Consume(); - } - else if (parser.Accept(out _)) - { - parser.Consume(); - while (!parser.Accept(out _)) - { - SkipValue(parser); - SkipValue(parser); - } - parser.Consume(); - } - else if (parser.Accept(out _)) - { - parser.Consume(); - while (!parser.Accept(out _)) - { - SkipValue(parser); - } - parser.Consume(); - } - } -} - -public record Service( - string? Host = null, - string? Project = null, - string? Language = null -); diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/DeploymentPlanTemplateUtil.cs b/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/DeploymentPlanTemplateUtil.cs index 3de086d7a2..e0e9a9d086 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/DeploymentPlanTemplateUtil.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/DeploymentPlanTemplateUtil.cs @@ -17,18 +17,10 @@ public static class DeploymentPlanTemplateUtil /// /// The name of the project. Can be null or empty. /// The target Azure service. - /// The provisioning tool. - /// The Infrastructure as Code options for AZD. /// A formatted deployment plan template string. - public static string GetPlanTemplate(string projectName, string targetAppService, string provisioningTool, string? azdIacOptions = "") + public static string GetPlanTemplate(string projectName, string targetAppService) { - // Default values for optional parameters - if (provisioningTool == "azd" && string.IsNullOrWhiteSpace(azdIacOptions)) - { - azdIacOptions = "bicep"; - } - - DeploymentPlanTemplateParameters parameters = CreateTemplateParameters(projectName, targetAppService, provisioningTool, azdIacOptions); + DeploymentPlanTemplateParameters parameters = CreateTemplateParameters(projectName, targetAppService); var executionSteps = GenerateExecutionSteps(parameters); parameters.ExecutionSteps = executionSteps; @@ -41,9 +33,7 @@ public static string GetPlanTemplate(string projectName, string targetAppService /// private static DeploymentPlanTemplateParameters CreateTemplateParameters( string projectName, - string targetAppService, - string provisioningTool, - string? azdIacOptions) + string targetAppService) { var azureComputeHost = GetAzureComputeHost(targetAppService); var title = string.IsNullOrWhiteSpace(projectName) @@ -55,8 +45,6 @@ private static DeploymentPlanTemplateParameters CreateTemplateParameters( Title = title, ProjectName = projectName, TargetAppService = targetAppService, - ProvisioningTool = provisioningTool, - IacType = azdIacOptions ?? "bicep", AzureComputeHost = azureComputeHost, }; } @@ -84,50 +72,11 @@ private static string GenerateExecutionSteps(DeploymentPlanTemplateParameters pa var steps = new List(); var isAks = parameters.TargetAppService.ToLowerInvariant() == "aks"; - if (parameters.ProvisioningTool.ToLowerInvariant() == "azd") - { - steps.AddRange(GenerateAzdSteps(parameters, isAks)); - } - else if (parameters.ProvisioningTool.Equals(DeploymentTool.AzCli, StringComparison.OrdinalIgnoreCase)) - { - steps.AddRange(GenerateAzCliSteps(parameters, isAks)); - } + steps.AddRange(GenerateAzCliSteps(parameters, isAks)); return string.Join(Environment.NewLine, steps); } - /// - /// Generates AZD-specific execution steps. - /// - private static List GenerateAzdSteps(DeploymentPlanTemplateParameters parameters, bool isAks) - { - var steps = new List(); - - var deployTitle = isAks ? "" : " And Deploy the Application"; - var checkLog = isAks ? "" : "6. Check the application log with tool `azd-app-log-get` to ensure the services are running."; - - var azdStepReplacements = new Dictionary - { - { "DeployTitle", deployTitle }, - { "IacType", parameters.IacType }, - { "CheckLog", checkLog } - }; - - var azdSteps = TemplateService.ProcessTemplate("Plan/azd-steps", azdStepReplacements); - steps.Add(azdSteps); - - if (isAks) - { - steps.Add(TemplateService.LoadTemplate("Plan/aks-steps")); - steps.Add(TemplateService.ProcessTemplate("Plan/summary-steps", new Dictionary { { "StepNumber", "4" } })); - } - else - { - steps.Add(TemplateService.ProcessTemplate("Plan/summary-steps", new Dictionary { { "StepNumber", "2" } })); - } - - return steps; - } /// /// Generates Azure CLI-specific execution steps. diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/IaCRulesTemplateUtil.cs b/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/IaCRulesTemplateUtil.cs index 995f617756..ada04c3a65 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/IaCRulesTemplateUtil.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/IaCRulesTemplateUtil.cs @@ -12,40 +12,6 @@ namespace Azure.Mcp.Tools.Deploy.Services.Util; /// public static class IaCRulesTemplateUtil { - /// - /// Generates IaC rules using embedded templates. - /// - /// The deployment tool (azd, azcli). - /// The IaC type (bicep, terraform). - /// Array of resource types. - /// A formatted IaC rules string. - public static string GetIaCRules(string deploymentTool, string iacType, string[] resourceTypes) - { - var parameters = CreateTemplateParameters(deploymentTool, iacType, resourceTypes); - var deploymentToolRules = GenerateDeploymentToolRules(parameters); - if (deploymentTool.Equals(DeploymentTool.AzCli, StringComparison.OrdinalIgnoreCase)) - { - return TemplateService.LoadTemplate("IaCRules/azcli-rules"); - } - // Default values for optional parameters - if (string.IsNullOrWhiteSpace(iacType)) - { - iacType = "bicep"; - } - var iacTypeRules = GenerateIaCTypeRules(parameters); - var resourceSpecificRules = GenerateResourceSpecificRules(parameters); - var finalInstructions = GenerateFinalInstructions(parameters); - - parameters.DeploymentToolRules = deploymentToolRules; - parameters.IacTypeRules = iacTypeRules; - parameters.ResourceSpecificRules = resourceSpecificRules; - parameters.FinalInstructions = finalInstructions; - parameters.RequiredTools = BuildRequiredTools(deploymentTool, resourceTypes); - parameters.AdditionalNotes = BuildAdditionalNotes(deploymentTool, iacType); - - return TemplateService.ProcessTemplate("IaCRules/base-iac-rules", parameters.ToDictionary()); - } - /// /// Creates template parameters from the provided inputs. /// @@ -104,116 +70,4 @@ private static void SetIaCTypeSpecificParameters(IaCRulesTemplateParameters para parameters.LogAnalyticsConfiguration = "- Container App Environment must be connected to Log Analytics Workspace. Use logs_destination=\"log-analytics\" azurerm_container_app_environment.log_analytics_workspace_id = azurerm_log_analytics_workspace..id."; } } - - /// - /// Generates deployment tool specific rules. - /// - private static string GenerateDeploymentToolRules(IaCRulesTemplateParameters parameters) - { - if (parameters.DeploymentTool.Equals(DeploymentTool.Azd, StringComparison.OrdinalIgnoreCase)) - { - var containerRegistryOutput = parameters.ResourceTypes.Contains(AzureServiceNames.AzureContainerApp) - ? "\n- Expected output in " + parameters.OutputFileName + ": AZURE_CONTAINER_REGISTRY_ENDPOINT representing the URI of the container registry endpoint." - : string.Empty; - - var azdReplacements = new Dictionary - { - { "IacType", parameters.IacType }, - { "OutputFileName", parameters.OutputFileName }, - { "ContainerRegistryOutput", containerRegistryOutput } - }; - - return TemplateService.ProcessTemplate("IaCRules/azd-rules", azdReplacements); - } - else if (parameters.DeploymentTool.Equals(DeploymentTool.AzCli, StringComparison.OrdinalIgnoreCase)) - { - return TemplateService.LoadTemplate("IaCRules/azcli-rules"); - } - - return string.Empty; - } - - /// - /// Generates IaC type specific rules. - /// - private static string GenerateIaCTypeRules(IaCRulesTemplateParameters parameters) - { - return parameters.IacType switch - { - IacType.Bicep => TemplateService.LoadTemplate("IaCRules/bicep-rules"), - IacType.Terraform => TemplateService.LoadTemplate("IaCRules/terraform-rules"), - _ => string.Empty - }; - } - - /// - /// Generates resource specific rules. - /// - private static string GenerateResourceSpecificRules(IaCRulesTemplateParameters parameters) - { - var rules = new List(); - - if (parameters.ResourceTypes.Contains(AzureServiceNames.AzureContainerApp)) - { - rules.Add(TemplateService.ProcessTemplate("IaCRules/containerapp-rules", parameters.ToDictionary())); - } - - if (parameters.ResourceTypes.Contains(AzureServiceNames.AzureAppService)) - { - rules.Add(TemplateService.ProcessTemplate("IaCRules/appservice-rules", parameters.ToDictionary())); - } - - if (parameters.ResourceTypes.Contains(AzureServiceNames.AzureFunctionApp)) - { - rules.Add(TemplateService.ProcessTemplate("IaCRules/functionapp-rules", parameters.ToDictionary())); - } - - if (parameters.ResourceTypes.Contains(AzureServiceNames.AzureStorage)) - { - rules.Add(TemplateService.ProcessTemplate("IaCRules/storage-rules", parameters.ToDictionary())); - } - - return string.Join(Environment.NewLine, rules); - } - - /// - /// Generates final instructions for the IaC rules. - /// - private static string GenerateFinalInstructions(IaCRulesTemplateParameters parameters) - { - return TemplateService.ProcessTemplate("IaCRules/final-instructions", parameters.ToDictionary()); - } - - /// - /// Builds the required tools list based on deployment tool and resource types. - /// - private static string BuildRequiredTools(string deploymentTool, string[] resourceTypes) - { - var tools = new List { "az cli (az --version)" }; - - if (deploymentTool == DeploymentTool.Azd) - { - tools.Add("azd (azd version)"); - } - - if (resourceTypes.Contains(AzureServiceNames.AzureContainerApp)) - { - tools.Add("docker (docker --version)"); - } - - return string.Join(", ", tools) + "."; - } - - /// - /// Builds additional notes based on deployment tool and IaC type. - /// - private static string BuildAdditionalNotes(string deploymentTool, string iacType) - { - if (iacType == IacType.Terraform && deploymentTool == DeploymentTool.Azd) - { - return "Note: Do not use Terraform CLI."; - } - - return string.Empty; - } } diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/PipelineGenerationUtil.cs b/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/PipelineGenerationUtil.cs index ccb0d87cd7..6736c636fc 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/PipelineGenerationUtil.cs +++ b/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/PipelineGenerationUtil.cs @@ -19,15 +19,8 @@ public static class PipelineGenerationUtil /// A formatted pipeline guidelines string. public static string GeneratePipelineGuidelines(GuidanceGetOptions options) { - if (options.UseAZDPipelineConfig) - { - return TemplateService.LoadTemplate("Pipeline/azd-pipeline"); - } - else - { - var parameters = CreatePipelineParameters(options); - return TemplateService.ProcessTemplate("Pipeline/azcli-pipeline", parameters.ToDictionary()); - } + var parameters = CreatePipelineParameters(options); + return TemplateService.ProcessTemplate("Pipeline/azcli-pipeline", parameters.ToDictionary()); } /// diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/ResourceLogService.cs b/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/ResourceLogService.cs new file mode 100644 index 0000000000..5c024095bd --- /dev/null +++ b/tools/Azure.Mcp.Tools.Deploy/src/Services/Util/ResourceLogService.cs @@ -0,0 +1,61 @@ +using Azure.Core; + +namespace Azure.Mcp.Tools.Deploy.Services.Util; + +public static class ResourceLogService +{ + public static async Task GetResourceLogsAsync( + TokenCredential credential, + string subscriptionId, + string resourceGroupName, + int? limit = null) + { + var toolErrorLogs = new List(); + var appLogs = new List(); + + try + { + var appLogRetriever = new AppLogRetriever(credential, subscriptionId, resourceGroupName); + await appLogRetriever.InitializeAsync(); + await appLogRetriever.GetLogAnalyticsWorkspacesInfoAsync(); + + // Auto-discover all supported resources in the resource group + var resources = await appLogRetriever.GetAllSupportedResourcesAsync(); + + if (resources.Count == 0) + { + return "No supported resources found in resource group."; + } + + foreach (var (resourceName, resourceType) in resources) + { + try + { + var logs = await appLogRetriever.QueryAppLogsAsync(resourceType, resourceName, limit); + appLogs.Add(logs); + } + catch (Exception ex) + { + toolErrorLogs.Add($"Error finding app logs for resource {resourceName}: {ex.Message}"); + } + } + } + catch (Exception ex) + { + toolErrorLogs.Add(ex.Message); + } + + if (appLogs.Count > 0) + { + return $"App logs retrieved:\n{string.Join("\n\n", appLogs)}"; + } + + if (toolErrorLogs.Count > 0) + { + return $"Error during retrieval of app logs:\n{string.Join("\n", toolErrorLogs)}"; + } + + return "No logs found."; + } + +} diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Templates/Architecture/architecture-diagram.md b/tools/Azure.Mcp.Tools.Deploy/src/Templates/Architecture/architecture-diagram.md index 2d41d3d12c..4d06b1579c 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Templates/Architecture/architecture-diagram.md +++ b/tools/Azure.Mcp.Tools.Deploy/src/Templates/Architecture/architecture-diagram.md @@ -1,5 +1,5 @@ Here is the user's mermaid diagram. Write a reminder to the user to install a Mermaid preview extension to be able to render the diagram. -Please write this into .azure/architecture.copilotmd WITHOUT additional explanations on the deployment. Explain only the architecture and data flow. +Please write this into docs/architecture.copilotmd WITHOUT additional explanations on the deployment. Explain only the architecture and data flow. Make changes if these do not fulfill requirements (do not use
in strings when generating the diagram): ```mermaid\n{{chart}}\n```. Ask user if the topology is expected, if not, you should directly update the generated diagram with the user's updated instructions. diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Templates/IaCRules/azd-rules.md b/tools/Azure.Mcp.Tools.Deploy/src/Templates/IaCRules/azd-rules.md deleted file mode 100644 index 1484236200..0000000000 --- a/tools/Azure.Mcp.Tools.Deploy/src/Templates/IaCRules/azd-rules.md +++ /dev/null @@ -1,5 +0,0 @@ -- Ensure an User-Assigned Managed Identity exists. -- Resource Group resource (if exists) must have tag "azd-env-name" = environmentName. Apply this tag to resource group resource ONLY. -- Expected parameters in {{IacType}} parameters: environmentName='${{AZURE_ENV_NAME}}', location='${{AZURE_LOCATION}}'. resourceGroupName='rg-${{AZURE_ENV_NAME}}' is required if scope is subscription. -- All container apps, app services, function apps, static web apps (and nothing else) must have tag "azd-service-name" matching the service name in azure.yaml. -- Expected output in {{OutputFileName}}: RESOURCE_GROUP_ID.{{ContainerRegistryOutput}} diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Templates/IaCRules/base-iac-rules.md b/tools/Azure.Mcp.Tools.Deploy/src/Templates/IaCRules/base-iac-rules.md deleted file mode 100644 index 4ea1663a73..0000000000 --- a/tools/Azure.Mcp.Tools.Deploy/src/Templates/IaCRules/base-iac-rules.md +++ /dev/null @@ -1,16 +0,0 @@ -Mandatory rules for deployment. You must implement every rule exactly as stated, with no exceptions or omissions, even if it is not a common pattern or seems redundant. Do not use your own judgment to simplify, skip, or modify any rule. If a rule is present, it must be enforced in the code, regardless of context. Adjust {{IacType}} files to align with these rules. - -## Deployment Tool {{DeploymentTool}} rules: -{{DeploymentToolRules}} - -## IaC Type: {{IacType}} rules: -{{IacTypeRules}} - -## Resources: {{ResourceTypesDisplay}} -{{ResourceSpecificRules}} - -{{FinalInstructions}} - -## Tools needed: {{RequiredTools}} - -{{AdditionalNotes}} diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Templates/IaCRules/bicep-rules.md b/tools/Azure.Mcp.Tools.Deploy/src/Templates/IaCRules/bicep-rules.md deleted file mode 100644 index 5480681753..0000000000 --- a/tools/Azure.Mcp.Tools.Deploy/src/Templates/IaCRules/bicep-rules.md +++ /dev/null @@ -1,3 +0,0 @@ -- Expected files: main.bicep, main.parameters.json (with parameters from main.bicep). -- Resource token format: 'uniqueString(subscription().id, resourceGroup().id, location, environmentName)' (scope = resourceGroup) or 'uniqueString(subscription().id, location, environmentName)' (scope = subscription). -- All resources should be named like az{resourcePrefix}{resourceToken}, where resourcePrefix is a prefix for the resource (ex. 'kv' for key vault) and <= 3 characters. Alphanumeric only. Entire resource name should be 32 characters maximum. ResourceToken is the string generated by uniqueString as per earlier. \ No newline at end of file diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Templates/IaCRules/terraform-rules.md b/tools/Azure.Mcp.Tools.Deploy/src/Templates/IaCRules/terraform-rules.md deleted file mode 100644 index 25eeae4676..0000000000 --- a/tools/Azure.Mcp.Tools.Deploy/src/Templates/IaCRules/terraform-rules.md +++ /dev/null @@ -1,2 +0,0 @@ -- Expected files: main.tf, main.tfvars.json (with the minimally required parameters), outputs.tf. -- Resource names should use Azure CAF naming convention. This is required for deployments. Add aztfmod/azurecaf in the required provider. DO NOT use random_length. NO suffixes needed. diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Templates/Pipeline/azd-pipeline.md b/tools/Azure.Mcp.Tools.Deploy/src/Templates/Pipeline/azd-pipeline.md deleted file mode 100644 index 630fb0a501..0000000000 --- a/tools/Azure.Mcp.Tools.Deploy/src/Templates/Pipeline/azd-pipeline.md +++ /dev/null @@ -1 +0,0 @@ -Run "azd pipeline config" to help the user create a deployment pipeline. diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Templates/Plan/azd-steps.md b/tools/Azure.Mcp.Tools.Deploy/src/Templates/Plan/azd-steps.md deleted file mode 100644 index a646fde8b7..0000000000 --- a/tools/Azure.Mcp.Tools.Deploy/src/Templates/Plan/azd-steps.md +++ /dev/null @@ -1,7 +0,0 @@ -1. Provision Azure Infrastructure{{DeployTitle}}: - 1. Based on following required Azure resources in plan, get the IaC rules from the tool `iac-rules-get` - 2. Generate IaC ({{IacType}} files) for required azure resources based on the plan. - 3. Pre-check: use `get_errors` tool to check generated Bicep grammar errors. Fix the errors if exist. - 4. Run the AZD command `azd up` to provision the resources and confirm each resource is created or already exists. - 5. Check the deployment output to ensure the resources are provisioned successfully. - {{CheckLog}} diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Templates/Plan/deployment-plan-base.md b/tools/Azure.Mcp.Tools.Deploy/src/Templates/Plan/deployment-plan-base.md index 786bb780a2..e2760bbec0 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Templates/Plan/deployment-plan-base.md +++ b/tools/Azure.Mcp.Tools.Deploy/src/Templates/Plan/deployment-plan-base.md @@ -1,9 +1,9 @@ # {{Title}} -{Agent should fill in and polish the markdown template below to generate a deployment plan for the project. Then save it to '.azure/plan.copilotmd' file. Don't add cost estimation! Don't add extra validation steps unless it is required! Don't change the tool name!} +{Agent should fill in and polish the markdown template below to generate a deployment plan for the project. Then save it to 'docs/plan.copilotmd' file. Don't add cost estimation! Don't add extra validation steps unless it is required! Don't change the tool name!} ## **Goal** -Based on the project to provide a plan to deploy the project to {{AzureComputeHost}} using {{ProvisioningTool}}. +Based on the project to provide a plan to deploy the project to {{AzureComputeHost}} using Azure CLI. ## **Project Information** { diff --git a/tools/Azure.Mcp.Tools.Deploy/src/Templates/Plan/summary-steps.md b/tools/Azure.Mcp.Tools.Deploy/src/Templates/Plan/summary-steps.md index 5ab29e67f9..25515b2d26 100644 --- a/tools/Azure.Mcp.Tools.Deploy/src/Templates/Plan/summary-steps.md +++ b/tools/Azure.Mcp.Tools.Deploy/src/Templates/Plan/summary-steps.md @@ -1,2 +1,2 @@ {{StepNumber}}: Summary: - 1. Summarize the deployment result and save to '.azure/summary.copilotmd'. It should list all changes deployment files and brief description of each file. Then have a diagram showing the provisioned azure resource. + 1. Summarize the deployment result and save to 'docs/summary.copilotmd'. It should list all changes deployment files and brief description of each file. Then have a diagram showing the provisioned azure resource. diff --git a/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.LiveTests/DeployCommandTests.cs b/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.LiveTests/DeployCommandTests.cs index ffa431692c..c47ad1134a 100644 --- a/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.LiveTests/DeployCommandTests.cs +++ b/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.LiveTests/DeployCommandTests.cs @@ -30,9 +30,7 @@ public async Task Should_get_plan() { { "workspace-folder", "C:/" }, { "project-name", "django" }, - { "target-app-service", "ContainerApp" }, - { "provisioning-tool", "AZD" }, - { "azd-iac-options", "bicep" } + { "target-app-service", "ContainerApp" } }); // assert Assert.StartsWith("# Azure Deployment Plan for django Project", result); @@ -46,45 +44,10 @@ public async Task Should_get_infrastructure_code_rules() "deploy_iac_rules_get", new() { - { "deployment-tool", "azd" }, - { "iac-type", "bicep" }, { "resource-types", "appservice, azurestorage" } }); - Assert.Contains("Deployment Tool azd rules", result ?? string.Empty, StringComparison.OrdinalIgnoreCase); - } - - [Fact] - public async Task Should_get_infrastructure_rules_for_terraform() - { - // act - var result = await CallToolMessageAsync( - "deploy_iac_rules_get", - new() - { - { "deployment-tool", "azd" }, - { "iac-type", "terraform" }, - { "resource-types", "containerapp, azurecosmosdb" } - }); - - // assert - Assert.Contains("IaC Type: terraform rules", result ?? string.Empty, StringComparison.OrdinalIgnoreCase); - } - - [Fact] - public async Task Should_generate_pipeline() - { - // act - var result = await CallToolMessageAsync( - "deploy_pipeline_guidance_get", - new() - { - { "subscription", _subscriptionId }, - { "use-azd-pipeline-config", true } - }); - - // assert - Assert.Contains("Run \"azd pipeline config\" to help the user create a deployment pipeline.", result); + Assert.Contains("If creating AzCli script, the script should stop if any command fails", result ?? string.Empty, StringComparison.OrdinalIgnoreCase); } [Fact] @@ -96,7 +59,6 @@ public async Task Should_generate_pipeline_with_github_details() new() { { "subscription", _subscriptionId }, - { "use-azd-pipeline-config", false }, { "organization-name", "test-org" }, { "repository-name", "test-repo" }, { "github-environment-name", "production" } diff --git a/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/App/LogsGetCommandTests.cs b/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/App/LogsGetCommandTests.cs index 03f3c6a516..bbb7dbc13d 100644 --- a/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/App/LogsGetCommandTests.cs +++ b/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/App/LogsGetCommandTests.cs @@ -38,12 +38,11 @@ public LogsGetCommandTests() } [Fact] - public async Task Should_get_azd_app_logs() + public async Task Should_get_app_logs() { // arrange var expectedLogs = "App logs retrieved:\n[2024-01-01 10:00:00] Application started\n[2024-01-01 10:01:00] Processing request"; - _deployService.GetAzdResourceLogsAsync( - Arg.Any(), + _deployService.GetResourceLogsAsync( Arg.Any(), Arg.Any(), Arg.Any(), @@ -52,8 +51,7 @@ public async Task Should_get_azd_app_logs() var args = _commandDefinition.Parse([ "--subscription", "test-subscription-id", - "--workspace-folder", "C:/Users/", - "--azd-env-name", "dotnet-demo", + "--resource-group", "rg-test", "--limit", "10" ]); @@ -70,12 +68,11 @@ public async Task Should_get_azd_app_logs() } [Fact] - public async Task Should_get_azd_app_logs_with_default_limit() + public async Task Should_get_app_logs_with_default_limit() { // arrange var expectedLogs = "App logs retrieved:\nSample log entry"; - _deployService.GetAzdResourceLogsAsync( - Arg.Any(), + _deployService.GetResourceLogsAsync( Arg.Any(), Arg.Any(), Arg.Any(), @@ -84,8 +81,7 @@ public async Task Should_get_azd_app_logs_with_default_limit() var args = _commandDefinition.Parse([ "--subscription", "test-subscription-id", - "--workspace-folder", "C:/project", - "--azd-env-name", "my-env" + "--resource-group", "rg-project" // No limit specified - should use default ]); @@ -103,8 +99,7 @@ public async Task Should_get_azd_app_logs_with_default_limit() public async Task Should_handle_no_logs_found() { // arrange - _deployService.GetAzdResourceLogsAsync( - Arg.Any(), + _deployService.GetResourceLogsAsync( Arg.Any(), Arg.Any(), Arg.Any(), @@ -113,8 +108,7 @@ public async Task Should_handle_no_logs_found() var args = _commandDefinition.Parse([ "--subscription", "test-subscription-id", - "--workspace-folder", "C:/empty-project", - "--azd-env-name", "empty-env", + "--resource-group", "rg-empty", "--limit", "50" ]); @@ -131,9 +125,8 @@ public async Task Should_handle_no_logs_found() public async Task Should_handle_error_during_log_retrieval() { // arrange - var errorMessage = "Error during retrieval of app logs of azd project:\nNo resource group with tag {\"azd-env-name\": test-env} found."; - _deployService.GetAzdResourceLogsAsync( - Arg.Any(), + var errorMessage = "Error during retrieval of app logs:\nNo resources found in resource group rg-test."; + _deployService.GetResourceLogsAsync( Arg.Any(), Arg.Any(), Arg.Any(), @@ -142,8 +135,7 @@ public async Task Should_handle_error_during_log_retrieval() var args = _commandDefinition.Parse([ "--subscription", "test-subscription-id", - "--workspace-folder", "C:/invalid-project", - "--azd-env-name", "test-env" + "--resource-group", "rg-test" ]); // act @@ -154,15 +146,14 @@ public async Task Should_handle_error_during_log_retrieval() Assert.Equal(HttpStatusCode.OK, result.Status); Assert.NotNull(result.Message); Assert.StartsWith("Error during retrieval of app logs", result.Message); - Assert.Contains("test-env", result.Message); + Assert.Contains("rg-test", result.Message); } [Fact] public async Task Should_handle_service_exception() { // arrange - _deployService.GetAzdResourceLogsAsync( - Arg.Any(), + _deployService.GetResourceLogsAsync( Arg.Any(), Arg.Any(), Arg.Any(), @@ -171,8 +162,7 @@ public async Task Should_handle_service_exception() var args = _commandDefinition.Parse([ "--subscription", "test-subscription-id", - "--workspace-folder", "C:/project", - "--azd-env-name", "test-env" + "--resource-group", "rg-project" ]); // act @@ -186,13 +176,12 @@ public async Task Should_handle_service_exception() } [Fact] - public async Task Should_validate_required_parameters() + public async Task Should_validate_required_resource_group_parameter() { - // arrange - missing required workspace-folder parameter + // arrange - missing required resource-group parameter var args = _commandDefinition.Parse([ - "--subscription", "test-subscription-id", - "--azd-env-name", "test-env" - // Missing workspace-folder + "--subscription", "test-subscription-id" + // Missing resource-group (required) ]); // act @@ -203,5 +192,4 @@ public async Task Should_validate_required_parameters() Assert.NotEqual(HttpStatusCode.OK, result.Status); // Should fail validation } - } diff --git a/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/Infrastructure/RulesGetCommandTests.cs b/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/Infrastructure/RulesGetCommandTests.cs index e5d17ae215..de62ac91b8 100644 --- a/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/Infrastructure/RulesGetCommandTests.cs +++ b/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/Infrastructure/RulesGetCommandTests.cs @@ -37,8 +37,6 @@ public async Task Should_get_infrastructure_code_rules() { // arrange var args = _commandDefinition.Parse([ - "--deployment-tool", "azd", - "--iac-type", "bicep", "--resource-types", "appservice, azurestorage" ]); @@ -49,16 +47,13 @@ public async Task Should_get_infrastructure_code_rules() Assert.NotNull(result); Assert.Equal(HttpStatusCode.OK, result.Status); Assert.NotNull(result.Message); - Assert.Contains("Deployment Tool azd rules", result.Message, StringComparison.OrdinalIgnoreCase); } [Fact] - public async Task Should_get_infrastructure_rules_for_terraform() + public async Task Should_get_infrastructure_rules_for_azurecosmosdb() { // arrange var args = _commandDefinition.Parse([ - "--deployment-tool", "azd", - "--iac-type", "terraform", "--resource-types", "containerapp, azurecosmosdb" ]); @@ -69,18 +64,14 @@ public async Task Should_get_infrastructure_rules_for_terraform() Assert.NotNull(result); Assert.Equal(HttpStatusCode.OK, result.Status); Assert.NotNull(result.Message); - Assert.Contains("Expected parameters in terraform parameters", result.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("If creating AzCli script, the script should stop if any command fails", result.Message, StringComparison.OrdinalIgnoreCase); } [Fact] - public async Task Should_get_infrastructure_rules_for_function_app() + public async Task Should_get_infrastructure_rules_without_resource_types() { // arrange - var args = _commandDefinition.Parse([ - "--deployment-tool", "azd", - "--iac-type", "bicep", - "--resource-types", "function" - ]); + var args = _commandDefinition.Parse([]); // act var result = await _command.ExecuteAsync(_context, args, TestContext.Current.CancellationToken); @@ -89,18 +80,14 @@ public async Task Should_get_infrastructure_rules_for_function_app() Assert.NotNull(result); Assert.Equal(HttpStatusCode.OK, result.Status); Assert.NotNull(result.Message); - Assert.Contains("Additional requirements for Function Apps", result.Message, StringComparison.OrdinalIgnoreCase); - Assert.Contains("Storage Blob Data Owner", result.Message, StringComparison.OrdinalIgnoreCase); } [Fact] - public async Task Should_get_infrastructure_rules_for_container_app() + public async Task Should_get_infrastructure_rules_for_function_app() { // arrange var args = _commandDefinition.Parse([ - "--deployment-tool", "azd", - "--iac-type", "bicep", - "--resource-types", "containerapp" + "--resource-types", "function" ]); // act @@ -110,18 +97,15 @@ public async Task Should_get_infrastructure_rules_for_container_app() Assert.NotNull(result); Assert.Equal(HttpStatusCode.OK, result.Status); Assert.NotNull(result.Message); - Assert.Contains("Additional requirements for Container Apps", result.Message, StringComparison.OrdinalIgnoreCase); - Assert.Contains("mcr.microsoft.com/azuredocs/containerapps-helloworld:latest", result.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("If creating AzCli script, the script should stop if any command fails", result.Message, StringComparison.OrdinalIgnoreCase); } [Fact] - public async Task Should_get_infrastructure_rules_for_azcli_deployment_tool() + public async Task Should_get_infrastructure_rules_for_container_app() { // arrange var args = _commandDefinition.Parse([ - "--deployment-tool", "AzCli", - "--iac-type", "", - "--resource-types", "aks" + "--resource-types", "containerapp" ]); // act @@ -131,17 +115,15 @@ public async Task Should_get_infrastructure_rules_for_azcli_deployment_tool() Assert.NotNull(result); Assert.Equal(HttpStatusCode.OK, result.Status); Assert.NotNull(result.Message); - Assert.Contains("If creating AzCli script, the script should stop if any command fails.", result.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("If creating AzCli script, the script should stop if any command fails", result.Message, StringComparison.OrdinalIgnoreCase); } [Fact] - public async Task Should_include_necessary_tools_in_response() + public async Task Should_get_infrastructure_rules_for_azcli() { // arrange var args = _commandDefinition.Parse([ - "--deployment-tool", "azd", - "--iac-type", "terraform", - "--resource-types", "containerapp" + "--resource-types", "aks" ]); // act @@ -151,10 +133,7 @@ public async Task Should_include_necessary_tools_in_response() Assert.NotNull(result); Assert.Equal(HttpStatusCode.OK, result.Status); Assert.NotNull(result.Message); - Assert.Contains("Tools needed:", result.Message, StringComparison.OrdinalIgnoreCase); - Assert.Contains("az cli", result.Message, StringComparison.OrdinalIgnoreCase); - Assert.Contains("azd", result.Message, StringComparison.OrdinalIgnoreCase); - Assert.Contains("docker", result.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("If creating AzCli script, the script should stop if any command fails.", result.Message, StringComparison.OrdinalIgnoreCase); } [Fact] @@ -162,8 +141,6 @@ public async Task Should_handle_multiple_resource_types() { // arrange var args = _commandDefinition.Parse([ - "--deployment-tool", "azd", - "--iac-type", "bicep", "--resource-types", "appservice,containerapp,function" ]); @@ -174,9 +151,6 @@ public async Task Should_handle_multiple_resource_types() Assert.NotNull(result); Assert.Equal(HttpStatusCode.OK, result.Status); Assert.NotNull(result.Message); - Assert.Contains("Resources: appservice, containerapp, function", result.Message, StringComparison.OrdinalIgnoreCase); - Assert.Contains("App Service Rules", result.Message, StringComparison.OrdinalIgnoreCase); - Assert.Contains("Additional requirements for Container Apps", result.Message, StringComparison.OrdinalIgnoreCase); - Assert.Contains("Additional requirements for Function Apps", result.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("If creating AzCli script, the script should stop if any command fails", result.Message, StringComparison.OrdinalIgnoreCase); } } diff --git a/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/Pipeline/GuidanceGetCommandTests.cs b/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/Pipeline/GuidanceGetCommandTests.cs index 0839ccc3e3..73b5e32f2d 100644 --- a/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/Pipeline/GuidanceGetCommandTests.cs +++ b/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/Pipeline/GuidanceGetCommandTests.cs @@ -32,32 +32,12 @@ public GuidanceGetCommandTests() _commandDefinition = _command.GetCommand(); } - [Fact] - public async Task Should_generate_pipeline() - { - // arrange - var args = _commandDefinition.Parse([ - "--subscription", "test-subscription-id", - "--use-azd-pipeline-config", "true" - ]); - - // act - var result = await _command.ExecuteAsync(_context, args, TestContext.Current.CancellationToken); - - // assert - Assert.NotNull(result); - Assert.Equal(HttpStatusCode.OK, result.Status); - Assert.NotNull(result.Message); - Assert.Contains("Run \"azd pipeline config\" to help the user create a deployment pipeline.", result.Message); - } - [Fact] public async Task Should_generate_pipeline_with_github_details() { // arrange var args = _commandDefinition.Parse([ "--subscription", "test-subscription-id", - "--use-azd-pipeline-config", "false", "--organization-name", "test-org", "--repository-name", "test-repo", "--github-environment-name", "production" @@ -77,9 +57,8 @@ public async Task Should_generate_pipeline_with_github_details() } [Fact] - public async Task Should_generate_pipeline_with_default_azd_pipeline_config() + public async Task Should_generate_pipeline_with_default_config() { - // arrange - not providing use-azd-pipeline-config should default to false var args = _commandDefinition.Parse([ "--subscription", "test-subscription-id" ]); @@ -100,8 +79,7 @@ public async Task Should_generate_pipeline_with_minimal_github_info() { // arrange var args = _commandDefinition.Parse([ - "--subscription", "test-subscription-id", - "--use-azd-pipeline-config", "false" + "--subscription", "test-subscription-id" ]); // act @@ -123,8 +101,7 @@ public async Task Should_handle_guid_subscription_id() // arrange var guidSubscriptionId = "12345678-1234-1234-1234-123456789abc"; var args = _commandDefinition.Parse([ - "--subscription", guidSubscriptionId, - "--use-azd-pipeline-config", "false" + "--subscription", guidSubscriptionId ]); // act @@ -142,8 +119,7 @@ public async Task Should_handle_non_guid_subscription_id() { // arrange var args = _commandDefinition.Parse([ - "--subscription", "my-subscription-name", - "--use-azd-pipeline-config", "false" + "--subscription", "my-subscription-name" ]); // act @@ -161,8 +137,7 @@ public async Task Should_include_service_principal_creation_steps() { // arrange var args = _commandDefinition.Parse([ - "--subscription", "test-subscription-id", - "--use-azd-pipeline-config", "false" + "--subscription", "test-subscription-id" ]); // act diff --git a/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/Plan/GetCommandTests.cs b/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/Plan/GetCommandTests.cs index 4a2120cdd7..95d39710ed 100644 --- a/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/Plan/GetCommandTests.cs +++ b/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/Commands/Plan/GetCommandTests.cs @@ -39,9 +39,7 @@ public async Task GetPlan_Should_Return_Expected_Result() var args = _commandDefinition.Parse([ "--workspace-folder", "C:/", "--project-name", "django", - "--target-app-service", "ContainerApp", - "--provisioning-tool", "AZD", - "--azd-iac-options", "bicep" + "--target-app-service", "ContainerApp" ]); // act @@ -56,15 +54,13 @@ public async Task GetPlan_Should_Return_Expected_Result() } [Fact] - public async Task Should_get_plan_with_default_iac_options() + public async Task Should_get_plan_for_webapp() { // arrange var args = _commandDefinition.Parse([ "--workspace-folder", "C:/test", "--project-name", "myapp", - "--target-app-service", "WebApp", - "--provisioning-tool", "azd" - // No azd-iac-options provided - should default to "bicep" + "--target-app-service", "WebApp" ]); // act @@ -85,8 +81,7 @@ public async Task Should_get_plan_for_kubernetes() var args = _commandDefinition.Parse([ "--workspace-folder", "C:/k8s-project", "--project-name", "k8s-app", - "--target-app-service", "AKS", - "--provisioning-tool", "azcli" + "--target-app-service", "AKS" ]); // act @@ -107,8 +102,7 @@ public async Task Should_get_plan_with_default_target_service() var args = _commandDefinition.Parse([ "--workspace-folder", "C:/", "--project-name", "default-app", - "--target-app-service", "unknown-service", // This should default to Container Apps - "--provisioning-tool", "AZD" + "--target-app-service", "unknown-service" // This should default to Container Apps ]); // act var result = await _command.ExecuteAsync(_context, args, TestContext.Current.CancellationToken); diff --git a/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/DeploymentPlanTemplateUtilV2Tests.cs b/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/DeploymentPlanTemplateUtilV2Tests.cs index 902b209c02..37c075c2f6 100644 --- a/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/DeploymentPlanTemplateUtilV2Tests.cs +++ b/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/DeploymentPlanTemplateUtilV2Tests.cs @@ -9,21 +9,17 @@ namespace Azure.Mcp.Tools.Deploy.UnitTests; public sealed class DeploymentPlanTemplateUtilV2Tests { [Theory] - [InlineData("TestProject", "ContainerApp", "AZD", "bicep")] - [InlineData("", "WebApp", "AzCli", "")] - [InlineData("MyApp", "AKS", "AZD", "terraform")] + [InlineData("", "WebApp")] + [InlineData("TestProject", "ContainerApp")] public void GetPlanTemplate_ValidInputs_ReturnsFormattedTemplate( string projectName, - string targetAppService, - string provisioningTool, - string azdIacOptions) + string targetAppService) { // Act var result = DeploymentPlanTemplateUtil.GetPlanTemplate( projectName, - targetAppService, - provisioningTool, - azdIacOptions); + targetAppService + ); // Assert Assert.NotNull(result); @@ -38,17 +34,9 @@ public void GetPlanTemplate_ValidInputs_ReturnsFormattedTemplate( // Should not contain unprocessed placeholders for main content Assert.DoesNotContain("{{Title}}", result); - Assert.DoesNotContain("{{ProvisioningTool}}", result); - - // Should contain appropriate provisioning tool - if (provisioningTool.ToLowerInvariant() == "azd") - { - Assert.Contains("azd up", result); - } - else - { - Assert.Contains("Azure CLI", result); - } + + // Should contain Azure CLI content + Assert.Contains("Azure CLI", result); } [Fact] @@ -57,9 +45,7 @@ public void GetPlanTemplate_EmptyProjectName_UsesDefaultTitle() // Act var result = DeploymentPlanTemplateUtil.GetPlanTemplate( "", - "ContainerApp", - "AZD", - "bicep"); + "ContainerApp"); // Assert Assert.Contains("Azure Deployment Plan", result); @@ -75,9 +61,7 @@ public void GetPlanTemplate_WithProjectName_UsesProjectSpecificTitle() // Act var result = DeploymentPlanTemplateUtil.GetPlanTemplate( projectName, - "ContainerApp", - "AZD", - "bicep"); + "ContainerApp"); // Assert Assert.Contains($"Azure Deployment Plan for {projectName} Project", result); @@ -96,37 +80,19 @@ public void GetPlanTemplate_DifferentTargetServices_MapsToCorrectAzureHost( // Act var result = DeploymentPlanTemplateUtil.GetPlanTemplate( "TestProject", - targetAppService, - "AZD", - "bicep"); + targetAppService); // Assert Assert.Contains(expectedAzureHost, result); } - [Fact] - public void GetPlanTemplate_AzdWithoutIacOptions_DefaultsToBicep() - { - // Act - var result = DeploymentPlanTemplateUtil.GetPlanTemplate( - "TestProject", - "ContainerApp", - "azd", - ""); - - // Assert - Assert.Contains("bicep", result); - } - [Fact] public void GetPlanTemplate_AksTarget_IncludesKubernetesSteps() { // Act var result = DeploymentPlanTemplateUtil.GetPlanTemplate( "TestProject", - "AKS", - "AZD", - "bicep"); + "AKS"); // Assert Assert.Contains("kubectl apply", result); @@ -135,14 +101,12 @@ public void GetPlanTemplate_AksTarget_IncludesKubernetesSteps() } [Fact] - public void GetPlanTemplate_ContainerAppWithAzCli_IncludesDockerSteps() + public void GetPlanTemplate_ContainerApp_IncludesDockerSteps() { // Act var result = DeploymentPlanTemplateUtil.GetPlanTemplate( "TestProject", - "ContainerApp", - "AzCli", - ""); + "ContainerApp"); // Assert Assert.Contains("Build and Push Docker Image", result); diff --git a/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/TemplateServiceTests.cs b/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/TemplateServiceTests.cs index a290a30b26..65c706fdc9 100644 --- a/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/TemplateServiceTests.cs +++ b/tools/Azure.Mcp.Tools.Deploy/tests/Azure.Mcp.Tools.Deploy.UnitTests/TemplateServiceTests.cs @@ -42,8 +42,7 @@ public void ProcessTemplate_WithReplacements_ReplacesPlaceholders() var replacements = new Dictionary { { "Title", "Test Deployment Plan" }, - { "ProjectName", "TestProject" }, - { "ProvisioningTool", "AZD" } + { "ProjectName", "TestProject" } }; // Act @@ -52,10 +51,8 @@ public void ProcessTemplate_WithReplacements_ReplacesPlaceholders() // Assert Assert.Contains("Test Deployment Plan", result); Assert.Contains("TestProject", result); - Assert.Contains("AZD", result); Assert.DoesNotContain("{{Title}}", result); Assert.DoesNotContain("{{ProjectName}}", result); - Assert.DoesNotContain("{{ProvisioningTool}}", result); } [Fact]