From 304ecd8cc9e2a36fb126c9efb6e8cc2e336ddd75 Mon Sep 17 00:00:00 2001 From: tvaron3 Date: Thu, 1 May 2025 12:32:47 -0400 Subject: [PATCH 1/4] change workloads based on feedback --- .../azure-cosmos/tests/workloads/dev.md | 10 +++++++- .../tests/workloads/r_w_q_workload.py | 1 - .../tests/workloads/shutdown_workloads.sh | 24 +++++++++++++++++-- .../tests/workloads/workload_configs.py | 3 +++ .../tests/workloads/workload_utils.py | 21 +++++++++++----- 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/tests/workloads/dev.md b/sdk/cosmos/azure-cosmos/tests/workloads/dev.md index 0123acb0ea9e..9835acfc2efd 100644 --- a/sdk/cosmos/azure-cosmos/tests/workloads/dev.md +++ b/sdk/cosmos/azure-cosmos/tests/workloads/dev.md @@ -3,12 +3,13 @@ This directory contains the scale testing workloads for the SDK. The workloads a and scalability of the SDK under various conditions. There are different types of workloads and each will create a log file when run. These logs are named in this format `--.log`. -### Setup VM +### Setup Scale Testing 1. Create a VM in Azure with the following configuration: - 8 vCPUs - 32 GB RAM - Ubuntu - Accelerated networking +1. Create a Azure App Insights Resource (Optional) 1. Fork and clone this repository 1. Go to azure cosmos folder - `cd azure-sdk-for-python/sdk/cosmos/azure-cosmos` @@ -19,6 +20,7 @@ file when run. These logs are named in this format `----` to see the logs in real time + +### Close Workloads +- If you want to keep the logs and stop the scripts, + `./shutdown_workloads.sh --no-remove-logs` +- If you want to remove the logs and stop the scripts, + `./shutdown_workloads.sh` diff --git a/sdk/cosmos/azure-cosmos/tests/workloads/r_w_q_workload.py b/sdk/cosmos/azure-cosmos/tests/workloads/r_w_q_workload.py index 368497bc60d3..c10502785cb9 100644 --- a/sdk/cosmos/azure-cosmos/tests/workloads/r_w_q_workload.py +++ b/sdk/cosmos/azure-cosmos/tests/workloads/r_w_q_workload.py @@ -1,6 +1,5 @@ # The MIT License (MIT) # Copyright (c) Microsoft Corporation. All rights reserved. -import os import sys from azure.cosmos import documents diff --git a/sdk/cosmos/azure-cosmos/tests/workloads/shutdown_workloads.sh b/sdk/cosmos/azure-cosmos/tests/workloads/shutdown_workloads.sh index 846f5d61aeaa..693cfeb7f19a 100755 --- a/sdk/cosmos/azure-cosmos/tests/workloads/shutdown_workloads.sh +++ b/sdk/cosmos/azure-cosmos/tests/workloads/shutdown_workloads.sh @@ -1,8 +1,28 @@ #!/bin/bash # cspell:disable + +NO_REMOVE_LOGS=false + +while [[ "$#" -gt 0 ]]; do + case $1 in + --no-remove-logs) + NO_REMOVE_LOGS=true + ;; + *) + echo "Unknown parameter: $1" + exit 1 + ;; + esac + shift +done + # Kill all python3 processes pkill -f "python3 ./r_" pkill -f "python3 ./w_" -# Remove all files with .logs in their filename -rm -f *.log* +if [ "$NO_REMOVE_LOGS" = false ]; then + # Remove all files with .log in their filename + rm -f *.log* + # Remove envoy log files + rm envoy/logs/* +fi diff --git a/sdk/cosmos/azure-cosmos/tests/workloads/workload_configs.py b/sdk/cosmos/azure-cosmos/tests/workloads/workload_configs.py index ec2d24a2cb8f..e74ae231eedd 100644 --- a/sdk/cosmos/azure-cosmos/tests/workloads/workload_configs.py +++ b/sdk/cosmos/azure-cosmos/tests/workloads/workload_configs.py @@ -9,6 +9,9 @@ COSMOS_KEY = "" COSMOS_CONTAINER = "scale_cont" COSMOS_DATABASE = "scale_db" +USER_AGENT_PREFIX = "" +APP_INSIGHTS_CONNECTION_STRING = "" +CIRCUIT_BREAKER_ENABLED = False USE_MULTIPLE_WRITABLE_LOCATIONS = False CONCURRENT_REQUESTS = 100 CONCURRENT_QUERIES = 2 diff --git a/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py b/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py index 72621222e296..acb1ce9c41e3 100644 --- a/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py +++ b/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py @@ -6,14 +6,17 @@ import random from datetime import datetime from logging.handlers import RotatingFileHandler -from workload_configs import NUMBER_OF_LOGICAL_PARTITIONS, PARTITION_KEY + +from azure.monitor.opentelemetry import configure_azure_monitor +from workload_configs import (NUMBER_OF_LOGICAL_PARTITIONS, PARTITION_KEY, USER_AGENT_PREFIX, + APP_INSIGHTS_CONNECTION_STRING, CIRCUIT_BREAKER_ENABLED) _NOISY_ERRORS = set([404, 409, 412]) _NOISY_SUB_STATUS_CODES = set([0, None]) _REQUIRED_ATTRIBUTES = ["resource_type", "verb", "operation_type", "status_code", "sub_status_code", "duration"] def get_user_agent(client_id): - return str(client_id) + "-" + datetime.now().strftime("%Y%m%d-%H%M%S") + return USER_AGENT_PREFIX + "-" + str(client_id) + "-" + datetime.now().strftime("%Y%m%d-%H%M%S") def get_random_item(): random_int = random.randint(0, NUMBER_OF_LOGICAL_PARTITIONS) @@ -103,15 +106,21 @@ async def perform_query_concurrently(container, excluded_locations): items = [item async for item in results] def create_logger(file_name): - logger = logging.getLogger('azure.cosmos') + os.environ["AZURE_COSMOS_ENABLE_CIRCUIT_BREAKER"] = str(CIRCUIT_BREAKER_ENABLED) + logger = logging.getLogger() + if APP_INSIGHTS_CONNECTION_STRING: + configure_azure_monitor( + logger_name="azure.cosmos", + connection_string=APP_INSIGHTS_CONNECTION_STRING, + ) prefix = os.path.splitext(file_name)[0] + "-" + str(os.getpid()) # Create a rotating file handler handler = RotatingFileHandler( - "log-" + prefix + "-" + datetime.now().strftime("%Y%m%d-%H%M%S") + '.log', + "log-" + get_user_agent(prefix) + '.log', maxBytes=1024 * 1024 * 10, # 10 mb - backupCount=3 + backupCount=2 ) - logger.setLevel(logging.DEBUG) + logger.setLevel(logging.INFO) # create filters for the logger handler to reduce the noise workload_logger_filter = WorkloadLoggerFilter() handler.addFilter(workload_logger_filter) From 15f6d563685935cbdf1f7e43ccfef7e1d1a9018d Mon Sep 17 00:00:00 2001 From: tvaron3 Date: Mon, 4 Aug 2025 23:26:30 -0700 Subject: [PATCH 2/4] add staging yml file --- sdk/cosmos/cosmos-staging.yml | 73 +++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 sdk/cosmos/cosmos-staging.yml diff --git a/sdk/cosmos/cosmos-staging.yml b/sdk/cosmos/cosmos-staging.yml new file mode 100644 index 000000000000..209196e03594 --- /dev/null +++ b/sdk/cosmos/cosmos-staging.yml @@ -0,0 +1,73 @@ +# Variable 'ACCOUNT_HOST' was defined in the Variables tab +# Variable 'ACCOUNT_KEY' was defined in the Variables tab +# Variable 'cosmosdb.azurecr.io.Password' was defined in the Variables tab +# Variable 'cosmosdb.azurecr.io.Username' was defined in the Variables tab +# Variable 'AZURE_COSMOS_ENABLE_CIRCUIT_BREAKER' was defined in the Variables tab +# Agent Queue 'Azure Pipelines' was used with unrecognized Agent Specification, vmImage property must be specified to determine image - https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops&tabs=yaml#software +name: $(date:yyyyMMdd)$(rev:.r) +resources: + repositories: + - repository: self + type: git + ref: refs/heads/main +jobs: +- job: Job_1 + displayName: Python 3 + timeoutInMinutes: 240 + pool: + name: Azure Pipelines + steps: + - checkout: self + submodules: recursive + fetchDepth: 5 + fetchTags: true + - task: UsePythonVersion@0 + name: '' + displayName: Use Python 3.x + condition: succeededOrFailed() + - task: Bash@3 + displayName: Bash Script + inputs: + targetType: inline + script: >- + cd ./sdk/cosmos/azure-cosmos + + pip install tox==4.5.0 + + tox -c ../../../eng/tox/tox.ini --root . -e whl -- -k 'not cosmosEmulator and not cosmosMultiRegion and not cosmosCircuitBreaker and not cosmosCircuitBreakerMultiRegion' + - task: AzurePowerShell@5 + displayName: Clean up resources + condition: always() + inputs: + ConnectedServiceNameARM: d71e80c4-c3df-4a27-9fc7-f25563bbb0d1 + ScriptType: InlineScript + Inline: "$uri = New-Object System.Uri(\"$(ACCOUNT_HOST)\")\n\n$accountName = $uri.Host.Split('.')[0]\nWrite-Host \"Clean up databases from $accountName\"\n\n$res = Get-AzResource -Name $accountName -ResourceType \"Microsoft.DocumentDb/databaseAccounts\"\n\nif ($res -ne $null) {\n\n $account = Get-AzCosmosDBAccount -ResourceId $res.ResourceId\n\n $dbs = Get-AzCosmosDBSqlDatabase -ParentObject $account\n\n $dbs | Write-Output\n\n $dbs | Remove-AzCosmosDBSqlDatabase\n \n Write-Host \"$accountName is cleaned\"\n}\nelse {\n Write-Host \"Unable to find $accountName\"\n}" + errorActionPreference: continue + TargetAzurePs: LatestVersion + - task: Bash@3 + displayName: Additional clean up using Python SDK - install SDK + condition: succeededOrFailed() + inputs: + targetType: inline + script: pip install azure-cosmos + - task: PythonScript@0 + displayName: Additional clean up using Python SDK + condition: succeededOrFailed() + inputs: + scriptSource: inline + script: >- + # python tests creates databases with special characters (!@$%^&*()-~`'_[]{}|;:,.<>) and PS and Portal unable to remove them. This task additionally clean resource that couldn't be removed using previous PS task + + + import os + + import azure.cosmos.cosmos_client as cosmos_client + + + client = cosmos_client.CosmosClient("$(ACCOUNT_HOST)", "$(ACCOUNT_KEY)") + + + for db in client.list_databases(): + + client.delete_database(db["id"]) +... From 1bbdc8ea9b87d5ab86ff9f0af7c989c459235228 Mon Sep 17 00:00:00 2001 From: tvaron3 Date: Mon, 4 Aug 2025 23:44:04 -0700 Subject: [PATCH 3/4] add staging yml file --- sdk/cosmos/cosmos-staging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/cosmos-staging.yml b/sdk/cosmos/cosmos-staging.yml index 209196e03594..a0906fe2a5e7 100644 --- a/sdk/cosmos/cosmos-staging.yml +++ b/sdk/cosmos/cosmos-staging.yml @@ -39,7 +39,7 @@ jobs: displayName: Clean up resources condition: always() inputs: - ConnectedServiceNameARM: d71e80c4-c3df-4a27-9fc7-f25563bbb0d1 + ConnectedServiceNameARM: 136f7343-9f47-40ec-badb-d988cb1429a2 ScriptType: InlineScript Inline: "$uri = New-Object System.Uri(\"$(ACCOUNT_HOST)\")\n\n$accountName = $uri.Host.Split('.')[0]\nWrite-Host \"Clean up databases from $accountName\"\n\n$res = Get-AzResource -Name $accountName -ResourceType \"Microsoft.DocumentDb/databaseAccounts\"\n\nif ($res -ne $null) {\n\n $account = Get-AzCosmosDBAccount -ResourceId $res.ResourceId\n\n $dbs = Get-AzCosmosDBSqlDatabase -ParentObject $account\n\n $dbs | Write-Output\n\n $dbs | Remove-AzCosmosDBSqlDatabase\n \n Write-Host \"$accountName is cleaned\"\n}\nelse {\n Write-Host \"Unable to find $accountName\"\n}" errorActionPreference: continue From bbb20658693e9e3c2dd3046234f64313d41ebc0d Mon Sep 17 00:00:00 2001 From: tvaron3 Date: Wed, 12 Nov 2025 14:18:12 -0500 Subject: [PATCH 4/4] change region --- sdk/cosmos/test-resources.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/test-resources.bicep b/sdk/cosmos/test-resources.bicep index cfaee9a3ff0f..982e9222e647 100644 --- a/sdk/cosmos/test-resources.bicep +++ b/sdk/cosmos/test-resources.bicep @@ -39,7 +39,7 @@ var multiRegionConfiguration = [ isZoneRedundant: false } { - locationName: 'West US' + locationName: 'East US 2' provisioningState: 'Succeeded' failoverPriority: 1 isZoneRedundant: false