From 7d5d9721642b39f5070c927f213c73c46d80fb20 Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Fri, 27 Mar 2026 15:18:31 -0700 Subject: [PATCH 01/12] Init commit --- .../AspNetBenchProfileTests.cs | 135 +++- .../NginxWrkProfileTests.cs | 321 ++++++++ .../AspNetBench/AspNetBenchExecutorTests.cs | 200 ----- .../AspNetOrchardServerExecutorTests.cs | 249 +++++++ .../AspNetBench/AspNetServerExecutorTests.cs | 287 ++++++++ .../AspNetBench/BombardierExecutorTests.cs | 108 +++ .../Examples/Nginx/NginxVersionExample.txt | 4 + .../Examples/Wrk/wrkErrorExample1.txt | 10 + .../Examples/Wrk/wrkErrorExample2.txt | 14 + .../Examples/Wrk/wrkStandardExample1.txt | 96 +++ .../Examples/Wrk/wrkStandardExample2.txt | 204 ++++++ .../Examples/Wrk/wrkStandardExample3.txt | 14 + .../Nginx/NginxServerExecutorTest.cs | 542 ++++++++++++++ .../VirtualClient.Actions.UnitTests.csproj | 1 + .../Wrk/Wrk2ExecutorTest.cs | 532 ++++++++++++++ .../Wrk/WrkExecutorTest.cs | 683 ++++++++++++++++++ .../Wrk/WrkMetricsParserTest.cs | 174 +++++ .../ASPNET/AspNetBenchBaseExecutor.cs | 336 --------- .../ASPNET/AspNetBenchClientExecutor.cs | 64 -- .../ASPNET/AspNetBenchExecutor.cs | 250 ------- .../ASPNET/AspNetBenchServerExecutor.cs | 62 -- .../ASPNET/AspNetOrchardServerExecutor.cs | 421 +++++++++++ .../ASPNET/AspNetServerExecutor.cs | 437 +++++++++++ .../Bombardier/BombardierExecutor.cs | 529 ++++++++++++++ .../BombardierMetricsParser.cs | 0 .../Nginx/NginxCommand.cs | 38 + .../Nginx/NginxExtensions.cs | 66 ++ .../Nginx/NginxServerExecutor.cs | 355 +++++++++ .../VirtualClient.Actions.csproj | 1 + .../VirtualClient.Actions/Wrk/Wrk2Executor.cs | 56 ++ .../VirtualClient.Actions/Wrk/WrkExecutor.cs | 578 +++++++++++++++ .../{ASPNET => Wrk}/WrkMetricParser.cs | 0 .../VirtualClient.Actions/Wrk/runwrk.sh | 2 + .../profiles/PERF-ASPNET-ORCHARD-WRK.json | 161 +++++ .../profiles/PERF-ASPNET-TEJSON-WRK.json | 150 ++++ .../profiles/PERF-ASPNETBENCH-AFFINITY.json | 93 +++ .../profiles/PERF-ASPNETBENCH-MULTI.json | 37 +- .../profiles/PERF-ASPNETBENCH.json | 27 +- .../profiles/PERF-WEB-NGINX-WRK-RP.json | 404 +++++++++++ .../profiles/PERF-WEB-NGINX-WRK.json | 382 ++++++++++ .../profiles/PERF-WEB-NGINX-WRK2-RP.json | 421 +++++++++++ .../profiles/PERF-WEB-NGINX-WRK2.json | 399 ++++++++++ 42 files changed, 7889 insertions(+), 954 deletions(-) create mode 100644 src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs delete mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetBenchExecutorTests.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Nginx/NginxVersionExample.txt create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample1.txt create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample2.txt create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample1.txt create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample2.txt create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample3.txt create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/Wrk2ExecutorTest.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs delete mode 100644 src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchBaseExecutor.cs delete mode 100644 src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchClientExecutor.cs delete mode 100644 src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchExecutor.cs delete mode 100644 src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchServerExecutor.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs rename src/VirtualClient/VirtualClient.Actions/{ASPNET => Bombardier}/BombardierMetricsParser.cs (100%) create mode 100644 src/VirtualClient/VirtualClient.Actions/Nginx/NginxCommand.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Wrk/Wrk2Executor.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs rename src/VirtualClient/VirtualClient.Actions/{ASPNET => Wrk}/WrkMetricParser.cs (100%) create mode 100644 src/VirtualClient/VirtualClient.Actions/Wrk/runwrk.sh create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-ORCHARD-WRK.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-TEJSON-WRK.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-AFFINITY.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK-RP.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2-RP.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2.json diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs index 046a97deb8..20287363df 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs @@ -6,6 +6,7 @@ namespace VirtualClient.Actions using System; using System.Collections.Generic; using System.Linq; + using System.Net; using System.Threading; using System.Threading.Tasks; using NUnit.Framework; @@ -17,19 +18,46 @@ namespace VirtualClient.Actions public class AspNetBenchProfileTests { private DependencyFixture mockFixture; + private string clientAgentId; + private string serverAgentId; [OneTimeSetUp] public void SetupFixture() { - this.mockFixture = new DependencyFixture(); + this.clientAgentId = $"{Environment.MachineName}-Client"; + this.serverAgentId = $"{Environment.MachineName}-Server"; + ComponentTypeCache.Instance.LoadComponentTypes(TestDependencies.TestDirectory); } + [SetUp] + public void Setup() + { + this.mockFixture = new DependencyFixture(); + } + [Test] [TestCase("PERF-ASPNETBENCH.json")] public void AspNetBenchWorkloadProfileParametersAreInlinedCorrectly(string profile) { - this.mockFixture.Setup(PlatformID.Unix); + this.mockFixture.Setup(PlatformID.Unix, agentId: this.clientAgentId).SetupLayout( + new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), + new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); + + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + WorkloadAssert.ParameterReferencesInlined(executor.Profile); + } + } + + [Test] + [TestCase("PERF-ASPNETBENCH-AFFINITY.json")] + public void AspNetBenchAffinityWorkloadProfileParametersAreInlinedCorrectly(string profile) + { + this.mockFixture.Setup(PlatformID.Unix, agentId: this.clientAgentId).SetupLayout( + new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), + new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) { WorkloadAssert.ParameterReferencesInlined(executor.Profile); @@ -42,22 +70,41 @@ public async Task AspNetBenchWorkloadProfileExecutesTheExpectedWorkloadsOnWindow { IEnumerable expectedCommands = this.GetProfileExpectedCommands(PlatformID.Win32NT); this.SetupDefaultMockBehaviors(PlatformID.Win32NT); - // Setup the expectations for the workload - // - Workload package is installed and exists. - // - Workload binaries/executables exist on the file system. - // - The workload generates valid results. this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => { IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); - if (arguments.Contains("bombardier", StringComparison.OrdinalIgnoreCase)) + + // Add bombardier results for any bombardier execution (with or without affinity) + if (command.Contains("bombardier", StringComparison.OrdinalIgnoreCase) || + arguments.Contains("bombardier", StringComparison.OrdinalIgnoreCase)) { - process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_AspNetBench.txt")); + if (arguments.Contains("--version")) + { + process.StandardOutput.Append("bombardier version 1.2.5"); + } + else + { + process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_AspNetBench.txt")); + } } return process; }; + // Setup API client for client-server communication + this.SetupApiClient(this.serverAgentId, "1.2.3.4"); + + // Execute server actions + this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.serverAgentId); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + executor.ExecuteDependencies = false; + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + } + + // Execute client actions + this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.clientAgentId); using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) { executor.ExecuteDependencies = false; @@ -73,22 +120,41 @@ public async Task AspNetBenchWorkloadProfileExecutesTheExpectedWorkloadsOnUnixPl { IEnumerable expectedCommands = this.GetProfileExpectedCommands(PlatformID.Unix); this.SetupDefaultMockBehaviors(PlatformID.Unix); - // Setup the expectations for the workload - // - Workload package is installed and exists. - // - Workload binaries/executables exist on the file system. - // - The workload generates valid results. this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => { IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); - if (arguments.Contains("bombardier", StringComparison.OrdinalIgnoreCase)) + + // Add bombardier results for any bombardier execution (with or without affinity) + if (command.Contains("bombardier", StringComparison.OrdinalIgnoreCase) || + arguments.Contains("bombardier", StringComparison.OrdinalIgnoreCase)) { - process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_AspNetBench.txt")); + if (arguments.Contains("--version")) + { + process.StandardOutput.Append("bombardier version 1.2.5"); + } + else + { + process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_AspNetBench.txt")); + } } return process; }; + // Setup API client for client-server communication + this.SetupApiClient(this.serverAgentId, "1.2.3.4"); + + // Execute server actions + this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.serverAgentId); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + executor.ExecuteDependencies = false; + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + } + + // Execute client actions + this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.clientAgentId); using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) { executor.ExecuteDependencies = false; @@ -106,9 +172,11 @@ private IEnumerable GetProfileExpectedCommands(PlatformID platform) case PlatformID.Win32NT: commands = new List { + @"pkill dotnet", + @"fuser -n tcp -k 9876", @"dotnet\.exe build -c Release -p:BenchmarksTargetFramework=net8.0", - @"dotnet\.exe .+Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept:.+ keep-alive", - @"bombardier\.exe --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:9876/json --print r --format json" + @"dotnet\.exe .+Benchmarks\.dll --nonInteractive true --scenarios json --urls http://\*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept:.+ keep-alive", + @"bombardier\.exe --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://1\.2\.3\.4:9876/json --print r --format json" }; break; @@ -116,9 +184,11 @@ private IEnumerable GetProfileExpectedCommands(PlatformID platform) commands = new List { @"chmod \+x .+bombardier", + @"pkill dotnet", + @"fuser -n tcp -k 9876", @"dotnet build -c Release -p:BenchmarksTargetFramework=net8.0", - @"dotnet .+Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept:.+ keep-alive", - @"bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:9876/json --print r --format json" + @"dotnet .+Benchmarks\.dll --nonInteractive true --scenarios json --urls http://\*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept:.+ keep-alive", + @"bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://1\.2\.3\.4:9876/json --print r --format json" }; break; } @@ -130,21 +200,38 @@ private void SetupDefaultMockBehaviors(PlatformID platform) { if (platform == PlatformID.Win32NT) { - this.mockFixture.Setup(PlatformID.Win32NT); + this.mockFixture.Setup(PlatformID.Win32NT, agentId: this.clientAgentId).SetupLayout( + new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), + new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); + this.mockFixture.SetupPackage("aspnetbenchmarks", expectedFiles: @"aspnetbench"); - this.mockFixture.SetupPackage("bombardier", expectedFiles: @"win-x64\bombardier.exe"); - this.mockFixture.SetupPackage("dotnetsdk", expectedFiles: @"packages\dotnet\dotnet.exe"); + this.mockFixture.SetupPackage("bombardier", expectedFiles: @"bombardier.exe"); + this.mockFixture.SetupPackage("dotnetsdk", expectedFiles: @"dotnet.exe"); } else { - this.mockFixture.Setup(PlatformID.Unix); + this.mockFixture.Setup(PlatformID.Unix, agentId: this.clientAgentId).SetupLayout( + new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), + new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); this.mockFixture.SetupPackage("aspnetbenchmarks", expectedFiles: @"aspnetbench"); - this.mockFixture.SetupPackage("bombardier", expectedFiles: @"linux-x64\bombardier"); - this.mockFixture.SetupPackage("dotnetsdk", expectedFiles: @"packages\dotnet\dotnet"); + this.mockFixture.SetupPackage("bombardier", expectedFiles: @"bombardier"); + this.mockFixture.SetupPackage("dotnetsdk", expectedFiles: @"dotnet"); } this.mockFixture.SetupDisks(withRemoteDisks: false); } + + private void SetupApiClient(string serverName, string serverIPAddress) + { + IPAddress.TryParse(serverIPAddress, out IPAddress ipAddress); + IApiClient apiClient = this.mockFixture.ApiClientManager.GetOrCreateApiClient(serverName, ipAddress); + + State state = new State(); + state.Online(true); + + apiClient.CreateStateAsync(nameof(State), state, CancellationToken.None) + .GetAwaiter().GetResult(); + } } } diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs new file mode 100644 index 0000000000..99900730c8 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs @@ -0,0 +1,321 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Net; + using System.Threading; + using System.Threading.Tasks; + using NUnit.Framework; + using VirtualClient.Common; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Functional")] + public class NginxWrkProfileTests + { + private DependencyFixture mockFixture; + private string clientAgentId; + private string serverAgentId; + + [OneTimeSetUp] + public void SetupFixture() + { + this.clientAgentId = $"{Environment.MachineName}-Client"; + this.serverAgentId = $"{Environment.MachineName}-Server"; + + ComponentTypeCache.Instance.LoadComponentTypes(TestDependencies.TestDirectory); + } + + [SetUp] + public void Setup() + { + this.mockFixture = new DependencyFixture(); + } + + [Test] + [TestCase("PERF-WEB-NGINX-WRK.json")] + [TestCase("PERF-WEB-NGINX-WRK2.json")] + public void NginxWrkProfileParametersAreInlinedCorrectly(string profile) + { + this.mockFixture.Setup(PlatformID.Unix, agentId: this.clientAgentId).SetupLayout( + new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), + new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); + + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + WorkloadAssert.ParameterReferencesInlined(executor.Profile); + } + } + + [Test] + [TestCase("PERF-WEB-NGINX-WRK.json")] + [TestCase("PERF-WEB-NGINX-WRK2.json")] + public void NginxWrkProfileParametersAreAvailable(string profile) + { + this.mockFixture.Setup(PlatformID.Unix, agentId: this.clientAgentId).SetupLayout( + new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), + new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); + + var serverPrams = new List { "PackageName", "Role", "Timeout" }; + + var reverseProxyPrams = new List { "PackageName", "Role", "Timeout" }; + + var clientPrams = new List { "PackageName", "Role", "Timeout", "TestDuration", "FileSizeInKB", "Connection", "ThreadCount", "CommandArguments", "MetricScenario", "Scenario" }; + + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + foreach (var actionBlock in executor.Profile.Actions) + { + string role = actionBlock.Parameters["Role"].ToString(); + + if (role.Equals("server", StringComparison.OrdinalIgnoreCase)) + { + foreach (var pram in serverPrams) + { + if (!actionBlock.Parameters.ContainsKey(pram)) + { + Assert.False(true, $"{actionBlock.Type} does not have {pram} parameter."); + } + } + } + else if (role.Equals("reverseproxy", StringComparison.OrdinalIgnoreCase)) + { + foreach (var pram in reverseProxyPrams) + { + if (!actionBlock.Parameters.ContainsKey(pram)) + { + Assert.False(true, $"{actionBlock.Type} does not have {pram} parameter."); + } + } + } + else + { + foreach (var pram in clientPrams) + { + if (!actionBlock.Parameters.ContainsKey(pram)) + { + Assert.False(true, $"{actionBlock.Type} does not have {pram} parameter."); + } + } + } + } + } + } + + [Test] + [TestCase("PERF-WEB-NGINX-WRK.json")] + public async Task NginxWrkProfileExecutesTheExpectedWorkloadsOnUnixPlatform(string profile) + { + IEnumerable expectedCommands = this.GetProfileExpectedCommands(); + this.SetupDefaultMockBehaviors(); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); + + // Handle nginx version check (uses stderr) + if (arguments.Contains("nginx -V", StringComparison.OrdinalIgnoreCase)) + { + process.StandardError.Append("nginx version: nginx/1.18.0\nbuilt with OpenSSL 1.1.1f\nTLS SNI support enabled"); + } + + // Handle nginx setup scripts + if (arguments.Contains("setup-reset.sh") || arguments.Contains("setup-content.sh") || arguments.Contains("setup-config.sh") || arguments.Contains("reset.sh")) + { + process.StandardOutput.Append("Script executed successfully"); + } + + // Handle nginx start/stop commands + if (arguments.Contains("systemctl")) + { + process.StandardOutput.Append("Service command executed"); + } + + // Add wrk results for any wrk execution + if (command.Contains("wrk", StringComparison.OrdinalIgnoreCase) || + arguments.Contains("wrk", StringComparison.OrdinalIgnoreCase)) + { + if (arguments.Contains("--version")) + { + process.StandardOutput.Append("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer"); + } + else + { + process.StandardOutput.Append(TestDependencies.GetResourceFileContents("wrkStandardExample1.txt")); + } + } + + return process; + }; + + // Setup API client for client-server communication + this.SetupApiClient(this.serverAgentId, "1.2.3.4"); + + // Execute server actions + this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.serverAgentId); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + executor.ExecuteDependencies = false; + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + } + + // Execute client actions + this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.clientAgentId); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + executor.ExecuteDependencies = false; + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + + WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); + } + } + + [Test] + [TestCase("PERF-WEB-NGINX-WRK2.json")] + public async Task NginxWrk2ProfileExecutesTheExpectedWorkloadsOnUnixPlatform(string profile) + { + IEnumerable expectedCommands = this.GetProfileExpectedCommandsForWrk2(); + this.SetupDefaultMockBehaviors(); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); + + // Handle nginx version check (uses stderr) + if (arguments.Contains("nginx -V", StringComparison.OrdinalIgnoreCase)) + { + process.StandardError.Append("nginx version: nginx/1.18.0\nbuilt with OpenSSL 1.1.1f\nTLS SNI support enabled"); + } + + // Handle nginx setup scripts + if (arguments.Contains("setup-reset.sh") || arguments.Contains("setup-content.sh") || arguments.Contains("setup-config.sh") || arguments.Contains("reset.sh")) + { + process.StandardOutput.Append("Script executed successfully"); + } + + // Handle nginx start/stop commands + if (arguments.Contains("systemctl")) + { + process.StandardOutput.Append("Service command executed"); + } + + // Add wrk2 results for any wrk execution + if (command.Contains("wrk", StringComparison.OrdinalIgnoreCase) || + arguments.Contains("wrk", StringComparison.OrdinalIgnoreCase)) + { + if (arguments.Contains("--version")) + { + process.StandardOutput.Append("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer"); + } + else + { + process.StandardOutput.Append(TestDependencies.GetResourceFileContents("wrkStandardExample1.txt")); + } + } + + return process; + }; + + // Setup API client for client-server communication + this.SetupApiClient(this.serverAgentId, "1.2.3.4"); + + // Execute server actions + this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.serverAgentId); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + executor.ExecuteDependencies = false; + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + } + + // Execute client actions + this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.clientAgentId); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + executor.ExecuteDependencies = false; + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + + WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); + } + } + + private IEnumerable GetProfileExpectedCommands() + { + // Expected commands for PERF-WEB-NGINX-WRK.json + return new List + { + @"chmod \+x .+wrk", + @"sudo systemctl stop nginx", + @"sudo systemctl start nginx", + @"bash .+runwrk\.sh --version", + // Sample commands for various connection scenarios + @"bash .+runwrk\.sh ""--latency --threads \d+ --connections 100 --duration 150s --timeout 10s https://1\.2\.3\.4/api_new/1kb""", + @"bash .+runwrk\.sh ""--latency --threads \d+ --connections 1000 --duration 150s --timeout 10s https://1\.2\.3\.4/api_new/1kb""", + @"bash .+runwrk\.sh ""--latency --threads \d+ --connections 5000 --duration 150s --timeout 10s https://1\.2\.3\.4/api_new/1kb""", + @"bash .+runwrk\.sh ""--latency --threads \d+ --connections 10000 --duration 150s --timeout 10s https://1\.2\.3\.4/api_new/1kb""" + }; + } + + private IEnumerable GetProfileExpectedCommandsForWrk2() + { + // Expected commands for PERF-WEB-NGINX-WRK2.json + return new List + { + @"chmod \+x .+wrk", + @"sudo systemctl stop nginx", + @"sudo systemctl start nginx", + @"bash .+runwrk\.sh --version", + // Sample commands for various rate and connection scenarios + @"bash .+runwrk\.sh ""--rate 1000 --latency --threads \d+ --connections 100 --duration 150s --timeout 10s https://1\.2\.3\.4/api_new/1kb""", + @"bash .+runwrk\.sh ""--rate 1000 --latency --threads \d+ --connections 1000 --duration 150s --timeout 10s https://1\.2\.3\.4/api_new/1kb""" + }; + } + + private void SetupDefaultMockBehaviors() + { + this.mockFixture.Setup(PlatformID.Unix, agentId: this.clientAgentId).SetupLayout( + new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), + new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); + + // Setup nginx configuration package with expected files + string nginxPackagePath = this.mockFixture.GetPackagePath("nginxconfiguration"); + this.mockFixture.SetupPackage("nginxconfiguration", expectedFiles: new string[] + { + "setup-reset.sh", + "setup-config.sh", + "setup-content.sh", + "reset.sh", + "nginx.conf" + }); + + // Mock the required files exist in the filesystem + this.mockFixture.SetupFile($"{nginxPackagePath}/linux-x64/setup-reset.sh"); + this.mockFixture.SetupFile($"{nginxPackagePath}/linux-x64/setup-config.sh"); + this.mockFixture.SetupFile($"{nginxPackagePath}/linux-x64/setup-content.sh"); + this.mockFixture.SetupFile($"{nginxPackagePath}/linux-x64/reset.sh"); + + // Setup wrk configuration package + this.mockFixture.SetupPackage("wrkconfiguration", expectedFiles: @"runwrk.sh"); + + // Setup wrk package + this.mockFixture.SetupPackage("wrk", expectedFiles: @"wrk"); + + this.mockFixture.SetupDisks(withRemoteDisks: false); + } + + private void SetupApiClient(string serverName, string serverIPAddress) + { + IPAddress.TryParse(serverIPAddress, out IPAddress ipAddress); + IApiClient apiClient = this.mockFixture.ApiClientManager.GetOrCreateApiClient(serverName, ipAddress); + + State state = new State(); + state.Online(true); + + apiClient.CreateStateAsync(nameof(State), state, CancellationToken.None) + .GetAwaiter().GetResult(); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetBenchExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetBenchExecutorTests.cs deleted file mode 100644 index b95aa5a1bd..0000000000 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetBenchExecutorTests.cs +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.Diagnostics; - using System.IO; - using System.Linq; - using System.Reflection; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Extensions.DependencyInjection; - using Moq; - using NUnit.Framework; - using VirtualClient.Common; - using VirtualClient.Common.Telemetry; - using VirtualClient.Contracts; - - [TestFixture] - [Category("Unit")] - public class AspNetBenchExecutorTests : MockFixture - { - public void SetupTest(PlatformID platform) - { - if (platform == PlatformID.Win32NT) - { - this.Setup(PlatformID.Win32NT); - - DependencyPath mockAspNetBenchPackage = new DependencyPath("aspnetbenchmarks", this.PlatformSpecifics.GetPackagePath("aspnetbenchmarks")); - DependencyPath mockDotNetPackage = new DependencyPath("dotnetsdk", this.PlatformSpecifics.GetPackagePath("dotnet")); - DependencyPath mockBombardierPackage = new DependencyPath("bombardier", this.PlatformSpecifics.GetPackagePath("bombardier")); - this.PackageManager.OnGetPackage(mockAspNetBenchPackage.Name).ReturnsAsync(mockAspNetBenchPackage); - this.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); - this.PackageManager.OnGetPackage(mockBombardierPackage.Name).ReturnsAsync(mockBombardierPackage); - } - else - { - this.Setup(PlatformID.Unix); - - DependencyPath mockAspNetBenchPackage = new DependencyPath("aspnetbenchmarks", this.PlatformSpecifics.GetPackagePath("aspnetbenchmarks")); - DependencyPath mockDotNetPackage = new DependencyPath("dotnetsdk", this.PlatformSpecifics.GetPackagePath("dotnet")); - DependencyPath mockBombardierPackage = new DependencyPath("bombardier", this.PlatformSpecifics.GetPackagePath("bombardier")); - this.PackageManager.OnGetPackage(mockAspNetBenchPackage.Name).ReturnsAsync(mockAspNetBenchPackage); - this.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); - this.PackageManager.OnGetPackage(mockBombardierPackage.Name).ReturnsAsync(mockBombardierPackage); - } - - this.File.Reset(); - this.File.Setup(f => f.Exists(It.IsAny())) - .Returns(true); - this.Directory.Setup(f => f.Exists(It.IsAny())) - .Returns(true); - this.FileSystem.SetupGet(fs => fs.File).Returns(this.File.Object); - - this.Parameters = new Dictionary() - { - { nameof(AspNetBenchExecutor.PackageName), "aspnetbenchmarks" }, - { nameof(AspNetBenchExecutor.DotNetSdkPackageName), "dotnetsdk" }, - { nameof(AspNetBenchExecutor.BombardierPackageName), "bombardier" }, - { nameof(AspNetBenchExecutor.TargetFramework), "net123.321" }, - { nameof(AspNetBenchExecutor.Port), "12321" } - }; - } - - [Test] - public void AspNetBenchExecutorThrowsIfCannotFindAspNetBenchPackage() - { - this.SetupTest(PlatformID.Win32NT); - this.PackageManager.OnGetPackage("aspnetbenchmarks").ReturnsAsync(value: null); - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); - } - } - - [Test] - public void AspNetBenchExecutorThrowsIfCannotFindBombardierPackage() - { - this.SetupTest(PlatformID.Unix); - this.PackageManager.OnGetPackage("bombardier").ReturnsAsync(value: null); - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); - } - } - - [Test] - public void AspNetBenchExecutorThrowsIfCannotFindDotNetSDKPackage() - { - this.SetupTest(PlatformID.Unix); - this.PackageManager.OnGetPackage("dotnetsdk").ReturnsAsync(value: null); - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); - } - } - - [Test] - public async Task AspNetBenchExecutorRunsTheExpectedWorkloadCommandInLinux() - { - this.SetupTest(PlatformID.Unix); - - string packageDirectory = this.GetPackagePath(); - ProcessStartInfo expectedInfo = new ProcessStartInfo(); - List expectedCommands = new List() - { - $@"sudo chmod +x ""{packageDirectory}/bombardier/linux-x64/bombardier""", - $@"sudo {packageDirectory}/dotnet/dotnet build -c Release -p:BenchmarksTargetFramework=net123.321", - $@"sudo {packageDirectory}/dotnet/dotnet {packageDirectory}/aspnetbenchmarks/src/Benchmarks/bin/Release/net123.321/Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:12321 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive""", - $@"sudo {packageDirectory}/bombardier/linux-x64/bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:12321/json --print r --format json" - }; - - int commandExecuted = 0; - this.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => - { - if (expectedCommands.Any(c => c == $"{exe} {arguments}")) - { - commandExecuted++; - } - - IProcessProxy process = new InMemoryProcess() - { - ExitCode = 0, - OnStart = () => true, - OnHasExited = () => true - }; - string exampleResults = MockFixture.ReadFile(MockFixture.ExamplesDirectory, "Bombardier", "BombardierExample.txt"); - process.StandardOutput.Append(exampleResults); - return process; - }; - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); - } - - Assert.AreEqual(4, commandExecuted); - } - - [Test] - public async Task AspNetBenchExecutorRunsTheExpectedWorkloadCommandInWindows() - { - this.SetupTest(PlatformID.Win32NT); - - string packageDirectory = this.GetPackagePath(); - ProcessStartInfo expectedInfo = new ProcessStartInfo(); - - List expectedCommands = new List() - { - $@"{packageDirectory}\dotnet\dotnet.exe build -c Release -p:BenchmarksTargetFramework=net123.321", - $@"{packageDirectory}\dotnet\dotnet.exe {packageDirectory}\aspnetbenchmarks\src\Benchmarks\bin\Release\net123.321\Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:12321 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive""", - $@"{packageDirectory}\bombardier\win-x64\bombardier.exe --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:12321/json --print r --format json" - }; - - int commandExecuted = 0; - this.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => - { - if (expectedCommands.Any(c => c == $"{exe} {arguments}")) - { - commandExecuted++; - } - - IProcessProxy process = new InMemoryProcess() - { - ExitCode = 0, - OnStart = () => true, - OnHasExited = () => true - }; - string exampleResults = MockFixture.ReadFile(MockFixture.ExamplesDirectory, "Bombardier", "BombardierExample.txt"); - process.StandardOutput.Append(exampleResults); - return process; - }; - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); - } - - Assert.AreEqual(3, commandExecuted); - } - - private class TestAspNetBenchExecutor : AspNetBenchExecutor - { - public TestAspNetBenchExecutor(IServiceCollection dependencies, IDictionary parameters) - : base(dependencies, parameters) - { - } - - public new Task ExecuteAsync(EventContext context, CancellationToken cancellationToken) - { - return base.ExecuteAsync(context, cancellationToken); - } - } - } -} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs new file mode 100644 index 0000000000..194d862db6 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs @@ -0,0 +1,249 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + using System.Net; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class AspNetOrchardServerExecutorTests + { + private MockFixture mockFixture; + private DependencyPath mockOrchardCorePackage; + private DependencyPath mockDotNetPackage; + + [Test] + public void AspNetOrchardServerOrchardExecutorThrowsIfCannotFindAspNetOrchardPackage() + { + this.SetupDefaultMockBehaviors(PlatformID.Win32NT); + this.mockFixture.PackageManager.OnGetPackage("orchardcore").ReturnsAsync(value: null); + + using (TestAspNetOrchardServerExecutor executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void AspNetOrchardServerOrchardExecutorThrowsIfCannotFindDotNetSDKPackage() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.PackageManager.OnGetPackage("dotnetsdk").ReturnsAsync(value: null); + + using (TestAspNetOrchardServerExecutor executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public async Task AspNetOrchardServerExecutorRunsTheExpectedWorkloadCommandInLinux() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + + this.mockFixture.TrackProcesses(); + + using (var executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None); + + this.mockFixture.Tracking.AssertCommandsExecuted(true, + "pkill OrchardCore", + "fuser -n tcp -k 5014", + $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet publish -c Release --sc -f net9\\.0 {Regex.Escape(this.mockOrchardCorePackage.Path)}/src/OrchardCore\\.Cms\\.Web/OrchardCore\\.Cms\\.Web\\.csproj", + $"nohup {Regex.Escape(this.mockOrchardCorePackage.Path)}/src/OrchardCore\\.Cms\\.Web/bin/Release/net9\\.0/linux-x64/publish/OrchardCore\\.Cms\\.Web --urls http://\\*:5014" + ); + + this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 1); + } + } + + [Test] + public async Task AspNetOrchardServerExecutorRunsTheExpectedWorkloadCommandInWindows() + { + this.SetupDefaultMockBehaviors(PlatformID.Win32NT); + + this.mockFixture.TrackProcesses(); + + using (var executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None); + + this.mockFixture.Tracking.AssertCommandsExecuted(true, + "pkill OrchardCore", + "fuser -n tcp -k 5014", + $"{Regex.Escape(this.mockDotNetPackage.Path)}\\\\dotnet\\.exe publish -c Release --sc -f net9\\.0 {Regex.Escape(this.mockOrchardCorePackage.Path)}\\\\src\\\\OrchardCore\\.Cms\\.Web\\\\OrchardCore\\.Cms\\.Web\\.csproj", + $"nohup {Regex.Escape(this.mockOrchardCorePackage.Path)}\\\\src\\\\OrchardCore\\.Cms\\.Web\\\\bin\\\\Release\\\\net9\\.0\\\\win-x64\\\\publish\\\\OrchardCore\\.Cms\\.Web --urls http://\\*:5014" + ); + + this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 1); + } + } + + [Test] + public async Task AspNetServerExecutorInitializeAsyncSetsCorrectPaths() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.Layout = new EnvironmentLayout(new List + { + new ClientInstance("Server", "1.2.3.4", ClientRole.Server), + new ClientInstance("Client", "5.6.7.8", ClientRole.Client) + }); + + using (TestAspNetOrchardServerExecutor executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.InitializeAsync(EventContext.None, CancellationToken.None); + + // Verify correct paths are set + string expectedAspNetDir = this.mockFixture.Combine( + this.mockOrchardCorePackage.Path, + "src", + "OrchardCore.Cms.Web"); + + Assert.AreEqual(expectedAspNetDir, executor.AspnetOrchardDirectory); + + string expectedDotNetPath = this.mockFixture.Combine( + this.mockDotNetPackage.Path, + "dotnet"); + + Assert.AreEqual(expectedDotNetPath, executor.DotNetExePath); + + // Verify API client is initialized + Assert.IsNotNull(executor.ServerApi); + } + } + + [Test] + public void AspNetOrchardServerExecutorThrowsWhenBindToCoresIsTrueButCoreAffinityIsNotProvided() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.Parameters[nameof(AspNetOrchardServerExecutor.BindToCores)] = true; + this.mockFixture.Parameters.Remove(nameof(AspNetOrchardServerExecutor.CoreAffinity)); + + using (TestAspNetOrchardServerExecutor executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.Throws(() => executor.Validate()); + } + } + + [Test] + public async Task AspNetOrchardServerExecutorExecutesWithCoreAffinityOnLinux() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.Parameters[nameof(AspNetOrchardServerExecutor.BindToCores)] = true; + this.mockFixture.Parameters[nameof(AspNetOrchardServerExecutor.CoreAffinity)] = "0-3"; + + this.mockFixture.TrackProcesses(); + + using (var executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None); + + // Verify numactl was used with correct core affinity + // Actual format: /bin/bash -c "numactl -C 0-3 nohup ..." + this.mockFixture.Tracking.AssertCommandsExecuted(true, + "pkill OrchardCore", + "fuser -n tcp -k 5014", + $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet publish -c Release --sc -f net9\\.0 {Regex.Escape(this.mockOrchardCorePackage.Path)}/src/OrchardCore\\.Cms\\.Web/OrchardCore\\.Cms\\.Web\\.csproj", + "/bin/bash -c \\\"numactl -C 0-3 .*" + ); + } + } + + private class TestAspNetOrchardServerExecutor : AspNetOrchardServerExecutor + { + public TestAspNetOrchardServerExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + this.ServerRetryPolicy = Policy.NoOpAsync(); + } + + public string AspnetOrchardDirectory + { + get + { + var field = typeof(AspNetOrchardServerExecutor).GetField("aspnetOrchardDirectory", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public string DotNetExePath + { + get + { + var field = typeof(AspNetOrchardServerExecutor).GetField("dotnetExePath", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public new Task InitializeAsync(EventContext context, CancellationToken cancellationToken) + { + return base.InitializeAsync(context, cancellationToken); + } + + public new void Validate() + { + base.Validate(); + } + } + + private void SetupDefaultMockBehaviors(PlatformID platform) + { + if (platform == PlatformID.Win32NT) + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Win32NT); + this.mockOrchardCorePackage = new DependencyPath("orchardcore", this.mockFixture.PlatformSpecifics.GetPackagePath("orchardcore")); + this.mockDotNetPackage = new DependencyPath("dotnetsdk", this.mockFixture.PlatformSpecifics.GetPackagePath("dotnet")); + this.mockFixture.PackageManager.OnGetPackage(mockOrchardCorePackage.Name).ReturnsAsync(mockOrchardCorePackage); + this.mockFixture.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); + } + else + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Unix); + this.mockOrchardCorePackage = new DependencyPath("orchardcore", this.mockFixture.PlatformSpecifics.GetPackagePath("orchardcore")); + this.mockDotNetPackage = new DependencyPath("dotnetsdk", this.mockFixture.PlatformSpecifics.GetPackagePath("dotnet")); + this.mockFixture.PackageManager.OnGetPackage(mockOrchardCorePackage.Name).ReturnsAsync(mockOrchardCorePackage); + this.mockFixture.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); + } + + this.mockFixture.File.Reset(); + this.mockFixture.File.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.mockFixture.Directory.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.mockFixture.FileSystem.SetupGet(fs => fs.File).Returns(this.mockFixture.File.Object); + + this.mockFixture.Parameters = new Dictionary() + { + { nameof(AspNetOrchardServerExecutor.PackageName), "orchardcore" }, + { nameof(AspNetOrchardServerExecutor.DotNetSdkPackageName), "dotnetsdk" }, + { nameof(AspNetOrchardServerExecutor.TargetFramework), "net9.0" }, + { nameof(AspNetOrchardServerExecutor.ServerPort), "5014" } + }; + + this.mockFixture.ApiClient.OnUpdateState(nameof(State)) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK)); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs new file mode 100644 index 0000000000..6d655c0b75 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs @@ -0,0 +1,287 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Linq; + using System.Net; + using System.Reflection; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + using VirtualClient.Actions.Memtier; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class AspNetServerExecutorTests + { + private MockFixture mockFixture; + private DependencyPath mockAspNetBenchPackage; + private DependencyPath mockDotNetPackage; + + [Test] + public void AspNetServerExecutorThrowsIfCannotFindAspNetBenchPackage() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.PackageManager.OnGetPackage("aspnetbenchmarks").ReturnsAsync(value: null); + + using (TestAspNetServerExecutor executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void AspNetServerExecutorThrowsIfCannotFindDotNetSDKPackage() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.PackageManager.OnGetPackage("dotnetsdk").ReturnsAsync(value: null); + + using (TestAspNetServerExecutor executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public async Task AspNetServerExecutorRunsTheExpectedWorkloadCommandInLinux() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + + this.mockFixture + .TrackProcesses() + .SetupProcessOutput( + ".*", + File.ReadAllText(Path.Combine( + Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), + "Examples", "Bombardier", "BombardierExample.txt"))); + + using (var executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None); + + this.mockFixture.Tracking.AssertCommandsExecuted(true, + "pkill dotnet", + "fuser -n tcp -k 9876", + $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet build -c Release -p:BenchmarksTargetFramework=net8\\.0", + $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet {Regex.Escape(this.mockAspNetBenchPackage.Path)}/src/Benchmarks/bin/Release/net8\\.0/Benchmarks\\.dll --nonInteractive true --scenarios json --urls http://\\*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header \\\"Accept: application/json,text/html;q=0\\.9,application/xhtml\\+xml;q=0\\.9,application/xml;q=0\\.8,\\*/\\*;q=0\\.7\\\" --header \\\"Connection: keep-alive\\\"" + ); + + this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 1); + } + } + + [Test] + public async Task AspNetServerExecutorRunsTheExpectedWorkloadCommandInWindows() + { + this.SetupDefaultMockBehaviors(PlatformID.Win32NT); + + this.mockFixture + .TrackProcesses() + .SetupProcessOutput( + ".*", + File.ReadAllText(Path.Combine( + Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), + "Examples", "Bombardier", "BombardierExample.txt"))); + + using (var executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None); + + this.mockFixture.Tracking.AssertCommandsExecuted(true, + "pkill dotnet", + "fuser -n tcp -k 9876", + $"{Regex.Escape(this.mockDotNetPackage.Path)}\\\\dotnet\\.exe build -c Release -p:BenchmarksTargetFramework=net8\\.0", + $"{Regex.Escape(this.mockDotNetPackage.Path)}\\\\dotnet\\.exe {Regex.Escape(this.mockAspNetBenchPackage.Path)}\\\\src\\\\Benchmarks\\\\bin\\\\Release\\\\net8\\.0\\\\Benchmarks\\.dll --nonInteractive true --scenarios json --urls http://\\*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header \\\"Accept: application/json,text/html;q=0\\.9,application/xhtml\\+xml;q=0\\.9,application/xml;q=0\\.8,\\*/\\*;q=0\\.7\\\" --header \\\"Connection: keep-alive\\\"" + ); + + this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 1); + } + } + + [Test] + public async Task AspNetServerExecutorInitializeAsyncSetsCorrectPaths() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.Layout = new EnvironmentLayout(new List + { + new ClientInstance("Server", "1.2.3.4", ClientRole.Server), + new ClientInstance("Client", "5.6.7.8", ClientRole.Client) + }); + + using (TestAspNetServerExecutor executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.InitializeAsync(EventContext.None, CancellationToken.None); + + // Verify correct paths are set + string expectedAspNetDir = this.mockFixture.Combine( + this.mockAspNetBenchPackage.Path, + "src", + "Benchmarks"); + + Assert.AreEqual(expectedAspNetDir, executor.AspNetBenchDirectory); + + string expectedDotNetPath = this.mockFixture.Combine( + this.mockDotNetPackage.Path, + "dotnet"); + + Assert.AreEqual(expectedDotNetPath, executor.DotNetExePath); + + // Verify API client is initialized + Assert.IsNotNull(executor.ServerApi); + } + } + + [Test] + public void AspNetServerExecutorThrowsWhenBindToCoresIsTrueButCoreAffinityIsNotProvided() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.Parameters[nameof(AspNetServerExecutor.BindToCores)] = true; + this.mockFixture.Parameters.Remove(nameof(AspNetServerExecutor.CoreAffinity)); + + using (TestAspNetServerExecutor executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.Throws(() => executor.Validate()); + } + } + + [Test] + public async Task AspNetServerExecutorExecutesWithCoreAffinityOnLinux() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.Parameters[nameof(AspNetServerExecutor.BindToCores)] = true; + this.mockFixture.Parameters[nameof(AspNetServerExecutor.CoreAffinity)] = "0-7"; + + this.mockFixture + .TrackProcesses() + .SetupProcessOutput( + ".*", + File.ReadAllText(Path.Combine( + Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), + "Examples", "Bombardier", "BombardierExample.txt"))); + + using (var executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None); + + // Verify numactl was used with correct core affinity + // The actual command format is: /bin/bash -c "numactl -C 0-7 /path/to/dotnet /path/to/dll ..." + this.mockFixture.Tracking.AssertCommandsExecuted(true, + "pkill dotnet", + "fuser -n tcp -k 9876", + $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet build -c Release -p:BenchmarksTargetFramework=net8\\.0", + "/bin/bash -c \\\"numactl -C 0-7 .*" + ); + } + } + + private class TestAspNetServerExecutor : AspNetServerExecutor + { + public TestAspNetServerExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + this.ServerRetryPolicy = Policy.NoOpAsync(); + } + + public string AspNetBenchDirectory + { + get + { + var field = typeof(AspNetServerExecutor).GetField("aspnetBenchDirectory", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public string AspNetBenchDllPath + { + get + { + var field = typeof(AspNetServerExecutor).GetField("aspnetBenchDllPath", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public string DotNetExePath + { + get + { + var field = typeof(AspNetServerExecutor).GetField("dotnetExePath", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public new Task InitializeAsync(EventContext context, CancellationToken cancellationToken) + { + return base.InitializeAsync(context, cancellationToken); + } + + public new void Validate() + { + base.Validate(); + } + } + + private void SetupDefaultMockBehaviors(PlatformID platform) + { + if (platform == PlatformID.Win32NT) + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Win32NT); + this.mockAspNetBenchPackage = new DependencyPath("aspnetbenchmarks", this.mockFixture.PlatformSpecifics.GetPackagePath("aspnetbenchmarks")); + this.mockDotNetPackage = new DependencyPath("dotnetsdk", this.mockFixture.PlatformSpecifics.GetPackagePath("dotnet")); + + this.mockFixture.PackageManager.OnGetPackage(mockAspNetBenchPackage.Name).ReturnsAsync(mockAspNetBenchPackage); + this.mockFixture.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); + + this.mockFixture.ApiClient.OnUpdateState(nameof(ServerState)) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK)); + } + else + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Unix); + + this.mockAspNetBenchPackage = new DependencyPath("aspnetbenchmarks", this.mockFixture.PlatformSpecifics.GetPackagePath("aspnetbenchmarks")); + this.mockDotNetPackage = new DependencyPath("dotnetsdk", this.mockFixture.PlatformSpecifics.GetPackagePath("dotnet")); + this.mockFixture.PackageManager.OnGetPackage(mockAspNetBenchPackage.Name).ReturnsAsync(mockAspNetBenchPackage); + this.mockFixture.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); + } + + this.mockFixture.File.Reset(); + this.mockFixture.File.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.mockFixture.Directory.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.mockFixture.FileSystem.SetupGet(fs => fs.File).Returns(this.mockFixture.File.Object); + + this.mockFixture.Parameters = new Dictionary() + { + { nameof(AspNetServerExecutor.PackageName), "aspnetbenchmarks" }, + { nameof(AspNetServerExecutor.DotNetSdkPackageName), "dotnetsdk" }, + { nameof(AspNetServerExecutor.TargetFramework), "net8.0" }, + { nameof(AspNetServerExecutor.ServerPort), "9876" }, + { nameof(AspNetServerExecutor.AspNetCoreThreadCount), "1" }, + { nameof(AspNetServerExecutor.DotNetSystemNetSocketsThreadCount), "1" } + }; + + this.mockFixture.ApiClient.OnUpdateState(nameof(State)) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK)); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs new file mode 100644 index 0000000000..5e3facd268 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Threading; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class BombardierExecutorTests + { + private MockFixture mockFixture; + private DependencyPath mockPackage; + + [SetUp] + public void SetupDefaults() + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Unix); + + this.mockPackage = new DependencyPath("bombardier", this.mockFixture.GetPackagePath("bombardier")); + + this.mockFixture.File.Reset(); + this.mockFixture.File.Setup(f => f.Exists(It.IsAny())).Returns(true); + this.mockFixture.FileSystem.SetupGet(fs => fs.File).Returns(this.mockFixture.File.Object); + + this.mockFixture.Layout = new EnvironmentLayout(new List + { + new ClientInstance($"{Environment.MachineName}", "1.2.3.4", ClientRole.Client), + new ClientInstance($"{Environment.MachineName}-Server", "1.2.3.5", ClientRole.Server) + }); + + this.mockFixture.Parameters = new Dictionary + { + { nameof(BombardierExecutor.PackageName), "bombardier" }, + { nameof(BombardierExecutor.Scenario), "Bombardier_Benchmark" }, + { nameof(BombardierExecutor.CommandArguments), "--connections 200 --duration 15s http://{ServerIp}:9090/json" } + }; + } + + [Test] + public void BombardierExecutorGetBombardierVersionParsesVersionWithVPrefix() + { + this.mockFixture.SetupProcessOutput(".*--version.*", "bombardier version v1.2.5 linux/arm64"); + + using (TestBombardierExecutor executor = new TestBombardierExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + executor.PackageDirectory = this.mockPackage.Path; + string version = executor.GetBombardierVersion(EventContext.None, CancellationToken.None); + Assert.AreEqual("1.2.5", version); + } + } + + [Test] + public void BombardierExecutorGetBombardierVersionParsesVersionWithoutVPrefix() + { + this.mockFixture.SetupProcessOutput(".*--version.*", "bombardier version 1.2.5"); + + using (TestBombardierExecutor executor = new TestBombardierExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + executor.PackageDirectory = this.mockPackage.Path; + string version = executor.GetBombardierVersion(EventContext.None, CancellationToken.None); + Assert.AreEqual("1.2.5", version); + } + } + + [Test] + public void BombardierExecutorGetBombardierVersionThrowsOnUnparsableOutput() + { + this.mockFixture.SetupProcessOutput(".*--version.*", "unrecognized output"); + + using (TestBombardierExecutor executor = new TestBombardierExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + executor.PackageDirectory = this.mockPackage.Path; + WorkloadException exception = Assert.Throws( + () => executor.GetBombardierVersion(EventContext.None, CancellationToken.None)); + + Assert.AreEqual(ErrorReason.CriticalWorkloadFailure, exception.Reason); + } + } + + private class TestBombardierExecutor : BombardierExecutor + { + public TestBombardierExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + } + + public new string PackageDirectory + { + get => base.PackageDirectory; + set => base.PackageDirectory = value; + } + + public new string GetBombardierVersion(EventContext telemetryContext, CancellationToken cancellationToken) + { + return base.GetBombardierVersion(telemetryContext, cancellationToken); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Nginx/NginxVersionExample.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Nginx/NginxVersionExample.txt new file mode 100644 index 0000000000..adb96f528d --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Nginx/NginxVersionExample.txt @@ -0,0 +1,4 @@ +nginx version: nginx/1.18.0 (Ubuntu) +built with OpenSSL 1.1.1f 31 Mar 2020 +TLS SNI support enabled +configure arguments: --with-cc-opt='-g -O2 -fdebug-prefix-map=/build/nginx-lUTckl/nginx-1.18.0=. -fstack-protector-strong -Wformat -Werror=format-security -fPIC -Wdate-time -D_FORTIFY_SOURCE=2' --with-ld-opt='-Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now -fPIC' --prefix=/usr/share/nginx --conf-path=/etc/nginx/nginx.conf --http-log-path=/var/log/nginx/access.log --error-log-path=/var/log/nginx/error.log --lock-path=/var/lock/nginx.lock --pid-path=/run/nginx.pid --modules-path=/usr/lib/nginx/modules --http-client-body-temp-path=/var/lib/nginx/body --http-fastcgi-temp-path=/var/lib/nginx/fastcgi --http-proxy-temp-path=/var/lib/nginx/proxy --http-scgi-temp-path=/var/lib/nginx/scgi --http-uwsgi-temp-path=/var/lib/nginx/uwsgi --with-debug --with-compat --with-pcre-jit --with-http_ssl_module --with-http_stub_status_module --with-http_realip_module --with-http_auth_request_module --with-http_v2_module --with-http_dav_module --with-http_slice_module --with-threads --with-http_addition_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_image_filter_module=dynamic --with-http_sub_module --with-http_xslt_module=dynamic --with-stream=dynamic --with-stream_ssl_module --with-mail=dynamic --with-mail_ssl_module \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample1.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample1.txt new file mode 100644 index 0000000000..c21ca9cf78 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample1.txt @@ -0,0 +1,10 @@ +Running 2m test @ http://10.1.0.15/api_new/10kb + 1 threads and 10000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 102.45ms 17.10ms 134.53ms 76.19% + Req/Sec 0.67 1.15 2.00 66.67% + 21 requests in 2.50m, 6.69KB read + Socket errors: connect 0, read 1645610, write 16, timeout 0 + Non-2xx or 3xx responses: 21 +Requests/sec: 0.14 +Transfer/sec: 45.63B \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample2.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample2.txt new file mode 100644 index 0000000000..f46572ad5c --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample2.txt @@ -0,0 +1,14 @@ +Running 30s test @ http://10.1.0.13/index.html + 5 threads and 100 connections + Thread calibration: mean lat.: 1.019ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.009ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.988ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.963ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.958ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.98ms 468.49us 8.97ms 70.07% + Req/Sec 420.73 49.56 0.89k 84.45% + 59956 requests in 30.00s, 18.64MB read + Non-2xx or 3xx responses: 59956 +Requests/sec: 1998.42 +Transfer/sec: 636.13KB \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample1.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample1.txt new file mode 100644 index 0000000000..c3c069347b --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample1.txt @@ -0,0 +1,96 @@ +Running 1m test @ https://10.1.0.26/api_new/1kb + 2 threads and 20 connections + Thread calibration: mean lat.: 4197.437ms, rate sampling interval: 15204ms + Thread calibration: mean lat.: 4197.977ms, rate sampling interval: 15147ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 29.27s 12.10s 50.20s 57.59% + Req/Sec 8.17k 163.44 8.32k 66.67% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 29.56s + 75.000% 39.75s + 90.000% 45.97s + 99.000% 49.74s + 99.900% 50.17s + 99.990% 50.23s + 99.999% 50.23s +100.000% 50.23s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 8372.223 0.000000 2 1.00 + 12525.567 0.100000 81614 1.11 + 16605.183 0.200000 163195 1.25 + 20660.223 0.300000 244674 1.43 + 24805.375 0.400000 326214 1.67 + 29556.735 0.500000 407688 2.00 + 31604.735 0.550000 448517 2.22 + 33619.967 0.600000 489638 2.50 + 35618.815 0.650000 530556 2.86 + 37683.199 0.700000 570904 3.33 + 39747.583 0.750000 611770 4.00 + 40796.159 0.775000 632420 4.44 + 41811.967 0.800000 652450 5.00 + 42827.775 0.825000 672721 5.71 + 43876.351 0.850000 693461 6.67 + 44892.159 0.875000 713577 8.00 + 45449.215 0.887500 723987 8.89 + 45973.503 0.900000 734157 10.00 + 46497.791 0.912500 744472 11.43 + 47054.847 0.925000 754722 13.33 + 47579.135 0.937500 764883 16.00 + 47841.279 0.943750 769668 17.78 + 48103.423 0.950000 774858 20.00 + 48365.567 0.956250 780158 22.86 + 48627.711 0.962500 785303 26.67 + 48857.087 0.968750 789786 32.00 + 48988.159 0.971875 792512 35.56 + 49119.231 0.975000 795257 40.00 + 49250.303 0.978125 797991 45.71 + 49381.375 0.981250 800387 53.33 + 49512.447 0.984375 802822 64.00 + 49577.983 0.985938 804076 71.11 + 49643.519 0.987500 805311 80.00 + 49709.055 0.989062 806567 91.43 + 49774.591 0.990625 807793 106.67 + 49840.127 0.992188 809006 128.00 + 49872.895 0.992969 809631 142.22 + 49905.663 0.993750 810256 160.00 + 49938.431 0.994531 810831 182.86 + 50003.967 0.995313 811970 213.33 + 50036.735 0.996094 812575 256.00 + 50036.735 0.996484 812575 284.44 + 50069.503 0.996875 813155 320.00 + 50069.503 0.997266 813155 365.71 + 50102.271 0.997656 813642 426.67 + 50135.039 0.998047 814120 512.00 + 50135.039 0.998242 814120 568.89 + 50135.039 0.998437 814120 640.00 + 50167.807 0.998633 814706 731.43 + 50167.807 0.998828 814706 853.33 + 50167.807 0.999023 814706 1024.00 + 50167.807 0.999121 814706 1137.78 + 50167.807 0.999219 814706 1280.00 + 50167.807 0.999316 814706 1462.86 + 50200.575 0.999414 815155 1706.67 + 50200.575 0.999512 815155 2048.00 + 50200.575 0.999561 815155 2275.56 + 50200.575 0.999609 815155 2560.00 + 50200.575 0.999658 815155 2925.71 + 50200.575 0.999707 815155 3413.33 + 50200.575 0.999756 815155 4096.00 + 50200.575 0.999780 815155 4551.11 + 50200.575 0.999805 815155 5120.00 + 50200.575 0.999829 815155 5851.43 + 50200.575 0.999854 815155 6826.67 + 50200.575 0.999878 815155 8192.00 + 50200.575 0.999890 815155 9102.22 + 50233.343 0.999902 815242 10240.00 + 50233.343 1.000000 815242 inf +#[Mean = 29266.261, StdDeviation = 12102.812] +#[Max = 50200.576, Total count = 815242] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 978315 requests in 1.00m, 1.17GB read +Requests/sec: 16305.17 +Transfer/sec: 20.01MB \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample2.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample2.txt new file mode 100644 index 0000000000..d6a5c1349d --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample2.txt @@ -0,0 +1,204 @@ +Running 30s test @ http://10.1.0.13/index.html + 2 threads and 100 connections + Thread calibration: mean lat.: 1.137ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.015ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.54ms 0.86ms 9.76ms 70.18% + Req/Sec 1.05k 1.39k 5.55k 89.70% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.43ms + 75.000% 1.98ms + 90.000% 2.68ms + 99.000% 3.96ms + 99.900% 6.93ms + 99.990% 8.99ms + 99.999% 9.77ms +100.000% 9.77ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.175 0.000000 1 1.00 + 0.566 0.100000 3954 1.11 + 0.776 0.200000 7913 1.25 + 1.003 0.300000 11866 1.43 + 1.226 0.400000 15800 1.67 + 1.427 0.500000 19773 2.00 + 1.517 0.550000 21738 2.22 + 1.610 0.600000 23711 2.50 + 1.715 0.650000 25678 2.86 + 1.837 0.700000 27663 3.33 + 1.982 0.750000 29634 4.00 + 2.067 0.775000 30635 4.44 + 2.163 0.800000 31614 5.00 + 2.261 0.825000 32598 5.71 + 2.381 0.850000 33586 6.67 + 2.519 0.875000 34567 8.00 + 2.601 0.887500 35060 8.89 + 2.683 0.900000 35553 10.00 + 2.769 0.912500 36051 11.43 + 2.867 0.925000 36538 13.33 + 2.981 0.937500 37038 16.00 + 3.047 0.943750 37281 17.78 + 3.105 0.950000 37534 20.00 + 3.163 0.956250 37773 22.86 + 3.235 0.962500 38022 26.67 + 3.327 0.968750 38271 32.00 + 3.375 0.971875 38392 35.56 + 3.435 0.975000 38515 40.00 + 3.507 0.978125 38636 45.71 + 3.589 0.981250 38765 53.33 + 3.689 0.984375 38886 64.00 + 3.749 0.985938 38946 71.11 + 3.825 0.987500 39008 80.00 + 3.911 0.989062 39068 91.43 + 4.011 0.990625 39130 106.67 + 4.167 0.992188 39193 128.00 + 4.271 0.992969 39223 142.22 + 4.383 0.993750 39255 160.00 + 4.535 0.994531 39284 182.86 + 4.719 0.995313 39315 213.33 + 4.967 0.996094 39346 256.00 + 5.087 0.996484 39364 284.44 + 5.383 0.996875 39377 320.00 + 5.623 0.997266 39392 365.71 + 5.851 0.997656 39408 426.67 + 6.087 0.998047 39423 512.00 + 6.195 0.998242 39431 568.89 + 6.331 0.998437 39439 640.00 + 6.543 0.998633 39446 731.43 + 6.663 0.998828 39454 853.33 + 7.011 0.999023 39462 1024.00 + 7.351 0.999121 39466 1137.78 + 7.511 0.999219 39470 1280.00 + 7.603 0.999316 39473 1462.86 + 7.687 0.999414 39477 1706.67 + 7.951 0.999512 39481 2048.00 + 8.099 0.999561 39484 2275.56 + 8.115 0.999609 39485 2560.00 + 8.207 0.999658 39487 2925.71 + 8.335 0.999707 39489 3413.33 + 8.599 0.999756 39491 4096.00 + 8.631 0.999780 39492 4551.11 + 8.783 0.999805 39493 5120.00 + 8.927 0.999829 39494 5851.43 + 8.983 0.999854 39495 6826.67 + 8.991 0.999878 39496 8192.00 + 8.991 0.999890 39496 9102.22 + 9.231 0.999902 39497 10240.00 + 9.231 0.999915 39497 11702.86 + 9.271 0.999927 39498 13653.33 + 9.271 0.999939 39498 16384.00 + 9.271 0.999945 39498 18204.44 + 9.503 0.999951 39499 20480.00 + 9.503 0.999957 39499 23405.71 + 9.503 0.999963 39499 27306.67 + 9.503 0.999969 39499 32768.00 + 9.503 0.999973 39499 36408.89 + 9.767 0.999976 39500 40960.00 + 9.767 1.000000 39500 inf +#[Mean = 1.537, StdDeviation = 0.861] +#[Max = 9.760, Total count = 39500] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + + Latency Distribution (HdrHistogram - Uncorrected Latency (measured without taking delayed starts into account)) + 50.000% 483.00us + 75.000% 1.12ms + 90.000% 1.71ms + 99.000% 2.87ms + 99.900% 5.76ms + 99.990% 8.02ms + 99.999% 8.41ms +100.000% 8.41ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.135 0.000000 1 1.00 + 0.242 0.100000 4008 1.11 + 0.287 0.200000 7981 1.25 + 0.335 0.300000 11912 1.43 + 0.392 0.400000 15829 1.67 + 0.483 0.500000 19763 2.00 + 0.552 0.550000 21732 2.22 + 0.652 0.600000 23706 2.50 + 0.802 0.650000 25676 2.86 + 0.959 0.700000 27650 3.33 + 1.115 0.750000 29633 4.00 + 1.200 0.775000 30626 4.44 + 1.285 0.800000 31612 5.00 + 1.375 0.825000 32597 5.71 + 1.475 0.850000 33586 6.67 + 1.586 0.875000 34567 8.00 + 1.645 0.887500 35064 8.89 + 1.713 0.900000 35550 10.00 + 1.788 0.912500 36050 11.43 + 1.872 0.925000 36541 13.33 + 1.964 0.937500 37032 16.00 + 2.018 0.943750 37282 17.78 + 2.081 0.950000 37532 20.00 + 2.147 0.956250 37779 22.86 + 2.219 0.962500 38023 26.67 + 2.301 0.968750 38268 32.00 + 2.353 0.971875 38395 35.56 + 2.397 0.975000 38514 40.00 + 2.459 0.978125 38638 45.71 + 2.531 0.981250 38761 53.33 + 2.625 0.984375 38888 64.00 + 2.679 0.985938 38945 71.11 + 2.741 0.987500 39007 80.00 + 2.821 0.989062 39068 91.43 + 2.907 0.990625 39130 106.67 + 3.029 0.992188 39193 128.00 + 3.119 0.992969 39223 142.22 + 3.209 0.993750 39254 160.00 + 3.371 0.994531 39284 182.86 + 3.583 0.995313 39315 213.33 + 3.793 0.996094 39346 256.00 + 3.943 0.996484 39363 284.44 + 4.175 0.996875 39377 320.00 + 4.423 0.997266 39392 365.71 + 4.703 0.997656 39408 426.67 + 5.007 0.998047 39423 512.00 + 5.163 0.998242 39431 568.89 + 5.223 0.998437 39439 640.00 + 5.355 0.998633 39446 731.43 + 5.651 0.998828 39454 853.33 + 5.783 0.999023 39462 1024.00 + 5.887 0.999121 39466 1137.78 + 5.971 0.999219 39470 1280.00 + 6.075 0.999316 39473 1462.86 + 6.147 0.999414 39477 1706.67 + 6.715 0.999512 39481 2048.00 + 6.963 0.999561 39483 2275.56 + 7.059 0.999609 39485 2560.00 + 7.195 0.999658 39487 2925.71 + 7.295 0.999707 39489 3413.33 + 7.423 0.999756 39491 4096.00 + 7.571 0.999780 39492 4551.11 + 7.687 0.999805 39493 5120.00 + 7.739 0.999829 39494 5851.43 + 7.823 0.999854 39495 6826.67 + 8.015 0.999878 39496 8192.00 + 8.015 0.999890 39496 9102.22 + 8.163 0.999902 39497 10240.00 + 8.163 0.999915 39497 11702.86 + 8.223 0.999927 39498 13653.33 + 8.223 0.999939 39498 16384.00 + 8.223 0.999945 39498 18204.44 + 8.399 0.999951 39499 20480.00 + 8.399 0.999957 39499 23405.71 + 8.399 0.999963 39499 27306.67 + 8.399 0.999969 39499 32768.00 + 8.399 0.999973 39499 36408.89 + 8.407 0.999976 39500 40960.00 + 8.407 1.000000 39500 inf +#[Mean = 0.782, StdDeviation = 0.678] +#[Max = 8.400, Total count = 39500] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 58902 requests in 30.00s, 18.31MB read + Non-2xx or 3xx responses: 58902 +Requests/sec: 1963.39 +Transfer/sec: 624.98KB \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample3.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample3.txt new file mode 100644 index 0000000000..28ff69c259 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample3.txt @@ -0,0 +1,14 @@ +Running 2m test @ http://10.9.0.7/api_new/10kb + 32 threads and 5000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 4.12ms 2.38ms 219.68ms 73.08% + Req/Sec 38.62k 3.74k 82.02k 72.05% + Latency Distribution + 50% 3.85ms + 75% 5.05ms + 90% 7.27ms + 99% 11.16ms + 184596465 requests in 2.50m, 56.05GB read + Non-2xx or 3xx responses: 184596465 +Requests/sec: 1229838.61 +Transfer/sec: 382.35MB \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs new file mode 100644 index 0000000000..33df870ace --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs @@ -0,0 +1,542 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net.Http; + using System.Reflection; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Common; + using VirtualClient.Contracts; + using Microsoft.CodeAnalysis; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Moq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Telemetry; + using Architecture = System.Runtime.InteropServices.Architecture; + + [TestFixture] + [NUnit.Framework.Category("Unit")] + public class NginxServerExecutorTest + { + private MockFixture mockFixture; + private InMemoryProcess memoryProcess; + private Item serverState; + private TimeSpan timeout = TimeSpan.FromMinutes(10); + private string packageName = "nginxconfiguration"; + + [SetUp] + public void SetupTests() + { + this.serverState = new Item(nameof(State), new State()); + this.mockFixture = new MockFixture(); + this.memoryProcess = new InMemoryProcess + { + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true, + StandardError = new ConcurrentBuffer(new StringBuilder($"nginx version: v1\n")) + }; + } + + [Test] + [TestCase(PlatformID.Win32NT, Architecture.X64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public void NginxServerExecutorThrowsErrorIfPlatformIsWrong(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters.Add(nameof(this.packageName), this.packageName); + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void NginxServerExecutorThrowsErrorPackageIsMissing(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxServerExecutorInitializesAsExpected(PlatformID platform, Architecture architecture) + { + /* + When Nginx Server Initialize, these are the expectations: + 1.Set up Api Client for client and server. + 2.Verify packages are installed and shell scripts exist inside. + 3.Set up reset(execute: "setup - reset.sh") - only applicable for first time + a.This will ensure, server's sysctl configuration is saved so when virtual client exits, it is able to reset the server back to its original state. + 4. Set up content. (execute: "setup-content.sh FileSizeInKB") + a.This will create a file that can be used during testing. + 5. Set up config (execute: "setup-config.sh) + a.This will make changes to server configuration. + 6. Delete any states that is saved in Server. + */ + + this.mockFixture.Setup(platform, architecture); + this.mockFixture.Parameters.Add(nameof(this.packageName), this.packageName); + this.mockFixture.Parameters.Add("FileSizeInKb", 5); + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + + // pkg setup + DependencyFixture fixture = new DependencyFixture(); + fixture.Setup(platform, architecture); + fixture.SetupPackage(this.packageName); + string packagePath = executor.PlatformSpecifics.ToPlatformSpecificPath(fixture.PackageManager.FirstOrDefault(), platform, architecture).Path; + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(y => y == this.packageName), It.IsAny())) + .ReturnsAsync(fixture.PackageManager.FirstOrDefault()); + + string resetFilePath = executor.PlatformSpecifics.Combine(packagePath, "reset.sh"); + string resetOutput = Guid.NewGuid().ToString(); + int processCount = 0; + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (processCount == 0) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-reset.sh"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(resetOutput)); + } + else if (processCount == 1) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-content.sh 5"); + } + else if (processCount == 2) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-config.sh auto Client 1.2.3.5"); + } + else + { + Assert.Fail("Only 3 process expected."); + } + + processCount++; + return this.memoryProcess; + }; + + string[] expectedFiles = new string[] + { + executor.PlatformSpecifics.Combine(packagePath, "setup-reset.sh"), + executor.PlatformSpecifics.Combine(packagePath, "setup-content.sh"), + executor.PlatformSpecifics.Combine(packagePath,"setup-config.sh") + }; + + this.mockFixture.FileSystem.Setup(x => x.File.Exists(It.Is(x => x == resetFilePath))).Returns(false); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.Is(x => x != resetFilePath))) + .Returns(true) + .Callback((string fileName) => + { + if (!expectedFiles.Any(y => y == fileName)) + { + Assert.Fail($"Unexpected File Name: {fileName}. \n{string.Join("\n", expectedFiles)}"); + } + }); + + // Create file for reset + Mock mockFileStream = new Mock(); + this.mockFixture.FileStream.Setup(f => f.New(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(mockFileStream.Object) + .Callback((string path, FileMode mode, FileAccess access, FileShare share) => + { + Assert.AreEqual(resetFilePath, path); + Assert.IsTrue(mode == FileMode.Create); + Assert.IsTrue(access == FileAccess.ReadWrite); + Assert.IsTrue(share == FileShare.ReadWrite); + }); + + mockFileStream + .Setup(x => x.Write(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((byte[] data, int offset, int count) => + { + byte[] byteData = Encoding.Default.GetBytes(resetOutput); + Assert.AreEqual(offset, 0); + Assert.AreEqual(count, byteData.Length); + Assert.AreEqual(data, byteData); + }); + + + + await executor.InitializeAsync().ConfigureAwait(false); + Assert.AreEqual(processCount, 3); + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(4)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxServerExecutorInitializesAsExpectedForSecondTime(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters.Add(nameof(this.packageName), this.packageName); + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + DependencyFixture fixture = new DependencyFixture(); + fixture.Setup(platform, architecture); + fixture.SetupPackage(this.packageName); + string packagePath = executor.PlatformSpecifics.ToPlatformSpecificPath(fixture.PackageManager.FirstOrDefault(), platform, architecture).Path; + + int processCount = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (processCount == 0) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-content.sh 1"); + } + else if (processCount == 1) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-config.sh auto Client 1.2.3.5"); + } + else + { + Assert.Fail("Only 2 process expected."); + } + + processCount++; + return this.memoryProcess; + }; + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(y => y == this.packageName), It.IsAny())) + .ReturnsAsync(fixture.PackageManager.FirstOrDefault()); + + var expectedFiles = new string[] + { + executor.PlatformSpecifics.Combine(packagePath, "setup-reset.sh"), + executor.PlatformSpecifics.Combine(packagePath, "setup-content.sh"), + executor.PlatformSpecifics.Combine(packagePath,"setup-config.sh"), + executor.PlatformSpecifics.Combine(packagePath, "reset.sh") + }; + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string fileName) => + { + if(!expectedFiles.Any(y => y == fileName)) + { + Assert.Fail($"Unexpected File Name: {fileName}. \n{string.Join("\n", expectedFiles)}"); + } + }); + + await executor.InitializeAsync().ConfigureAwait(false); + Assert.AreEqual(processCount, 2); + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(4)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxServerExecutorResetsServerAsExpected(PlatformID platform, Architecture architecture) + { + TimeSpan timeout = TimeSpan.FromMinutes(5); + this.mockFixture.Setup(platform, architecture, nameof(State)); + int processCount = 0; + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (processCount == 0) + { + // During every reset, nginx will delete existing content. + // "delete-content.sh" is downloaded from blob. + Assert.AreEqual(arguments, "bash reset.sh"); + } + else + { + Assert.AreEqual(arguments, "systemctl disable nginx", NginxCommand.Stop.ConvertToCommandArgs()); + } + + Assert.AreEqual(command, "sudo"); + Assert.IsNull(workingDir); + processCount++; + return this.memoryProcess; + }; + + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + await executor.ResetNginxAsync().ConfigureAwait(false); + Assert.AreEqual(processCount, 2); + + this.mockFixture.ApiClient.Verify(x => x.UpdateStateAsync( + It.Is(x => x == nameof(State)), + It.Is>(x => x.Definition.Online(null) == false), + It.IsAny(), + It.IsAny>()), + Times.Exactly(1)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void ResetServerWillSwallowExceptions(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => { throw new SchemaException(); }; + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Assert.DoesNotThrowAsync(async () => await executor.ResetNginxAsync().ConfigureAwait(false)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void NginxServerExecutorWillResetServerDuringDispose(PlatformID platform, Architecture architecture) + { + // Dispose will call reset nginx + TimeSpan timeout = TimeSpan.FromMinutes(5); + this.mockFixture.Setup(platform, architecture, nameof(State)); + int processCount = 0; + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (processCount == 0) + { + // During every reset, nginx will delete existing content. + // "delete-content.sh" is downloaded from blob. + Assert.AreEqual(arguments, "bash reset.sh"); + } + else + { + Assert.AreEqual(arguments, "systemctl disable nginx", NginxCommand.Stop.ConvertToCommandArgs()); + } + + Assert.AreEqual(command, "sudo"); + Assert.IsNull(workingDir); + processCount++; + return this.memoryProcess; + }; + + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + executor.Dispose(true); + Assert.AreEqual(processCount, 2); + this.mockFixture.ApiClient.Verify(x => x.UpdateStateAsync( + It.Is(x => x == nameof(State)), + It.Is>(x => x.Definition.Online(null) == false), + It.IsAny(), + It.IsAny>()), + Times.Exactly(1)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxServerExecutorRunsAsExpected(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters.Add(nameof(this.packageName), this.packageName); + this.mockFixture.Parameters.Add(nameof(this.timeout), this.timeout.ToString()); + this.mockFixture.Parameters.Add("pollingInterval", TimeSpan.FromSeconds(1).ToString()); + + int nginxServiceCalls = 0; + int shellScriptCalls = 0; + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + DependencyFixture fixture = new DependencyFixture(); + fixture.Setup(platform, architecture); + fixture.SetupPackage(packageName); + + string packagePath = executor.PlatformSpecifics.ToPlatformSpecificPath(fixture.PackageManager.FirstOrDefault(), platform, architecture).Path; + this.mockFixture.PackageManager.Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())).ReturnsAsync(fixture.PackageManager.FirstOrDefault()); + this.mockFixture.FileSystem.Setup(x => x.File.Exists(It.IsAny())).Returns(true); + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.AreEqual(command, "sudo"); + + if (new[] {"bash setup-content.sh 1", "bash setup-config.sh", "bash setup-config.sh auto Client 1.2.3.5", "bash reset.sh" }.Contains(arguments, StringComparer.OrdinalIgnoreCase)) + { + shellScriptCalls++; + Assert.AreEqual(workingDir, packagePath); + } + else if (new[] { "systemctl restart nginx", "systemctl disable nginx"}.Contains(arguments, StringComparer.OrdinalIgnoreCase)) + { + nginxServiceCalls++; + Assert.IsNull(workingDir); + } + else if (arguments == "nginx -V") + { + nginxServiceCalls++; + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Nginx"); + string outputPath = Path.Combine(examplesDirectory, @"NginxVersionExample.txt"); + string rawText = File.ReadAllText(outputPath); + this.memoryProcess.StandardError = new ConcurrentBuffer(new StringBuilder(rawText)); + } + else + { + Assert.Fail($"Unexpected Arguments: {arguments}"); + } + + return this.memoryProcess; + }; + + + Item onlineClientState = new Item(nameof(State), new State()); + onlineClientState.Definition.Online(true); + + Item expiredState = new Item(nameof(State), new State()); + expiredState.Definition.Timeout(DateTime.UtcNow.AddDays(-1)); + + // Set up for polling online client state. + this.mockFixture.ApiClient.SetupSequence(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK, onlineClientState)) // Polling for online state + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK, expiredState)); // Get client state + + this.mockFixture.ApiClient.Setup(x => x.CreateStateAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + + this.mockFixture.ApiClient.Setup(x => x.UpdateStateAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + + await executor.InitializeAsync().ConfigureAwait(false); + await executor.ExecuteAsync().ConfigureAwait(false); + + Assert.AreEqual(shellScriptCalls, 3); + Assert.AreEqual(nginxServiceCalls, 3); + + // nginx version and local online state + this.mockFixture.ApiClient.Verify(x => x.CreateStateAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>()), Times.Exactly(2)); + + // first loop and reset + this.mockFixture.ApiClient.Verify(x => x.UpdateStateAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny>()), Times.Exactly(2)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void NginxServerExecutorWillResetServerIfFailure(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + int processCount = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (arguments == "nginx -V") + { + // This will ensure nginx version is empty therefore ArgumentException will be thrown. + this.memoryProcess.StandardError = new ConcurrentBuffer(new StringBuilder(string.Empty)); + } + else if (arguments == "systemctl disable nginx" || arguments == "bash reset.sh") + { + } + else + { + Assert.Fail(); + } + + processCount++; + return this.memoryProcess; + }; + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => await executor.ExecuteAsync().ConfigureAwait(false)); + Assert.AreEqual(processCount, 3); + + // Expected to delete local server state & will create new offline state. + this.mockFixture.ApiClient.Verify(x => x.UpdateStateAsync( + It.Is(x => x == nameof(State)), + It.Is>(x => x.Definition.Online(null) == false), + It.IsAny(), + It.IsAny>()), + Times.Exactly(1)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxExecutorParsesNginxVersion(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Nginx"); + string outputPath = Path.Combine(examplesDirectory, @"NginxVersionExample.txt"); + string rawText = File.ReadAllText(outputPath); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (arguments == "nginx -V") + { + // This will ensure nginx version is empty therefore ArgumentException will be thrown. + this.memoryProcess.StandardError = new ConcurrentBuffer(new StringBuilder(rawText)); + } + else + { + Assert.Fail(); + } + + return this.memoryProcess; + }; + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Dictionary version = await executor.GetNginxVersionAsync().ConfigureAwait(false); + + Assert.AreEqual(version["nginxVersion"], "nginx/1.18.0 (Ubuntu)"); + Assert.IsTrue(version["sslVersion"].Contains("1.1.1f", StringComparison.OrdinalIgnoreCase)); + Assert.AreEqual(version["serverNameIndicationSupport"], "TLS SNI support enabled"); + Assert.IsTrue(version["arguments"].StartsWith("--with-cc-opt='-g -O2")); + } + + private class TestNginxServerExecutor : NginxServerExecutor + { + public TestNginxServerExecutor(MockFixture mockFixture, IDictionary parameters = null) + : base(mockFixture.Dependencies, mockFixture.Parameters) + { + this.ServerApi = mockFixture.ApiClient.Object; + this.ClientApi = mockFixture.ApiClient.Object; + } + + public new void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async Task InitializeAsync() + { + await base.InitializeAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ExecuteAsync() + { + await base.ExecuteAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ResetNginxAsync() + { + await base.ResetNginxAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task> GetNginxVersionAsync() + { + return await base.GetNginxVersionAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/VirtualClient.Actions.UnitTests.csproj b/src/VirtualClient/VirtualClient.Actions.UnitTests/VirtualClient.Actions.UnitTests.csproj index c41956fee4..4edb5fd60e 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/VirtualClient.Actions.UnitTests.csproj +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/VirtualClient.Actions.UnitTests.csproj @@ -101,6 +101,7 @@ + diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/Wrk2ExecutorTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/Wrk2ExecutorTest.cs new file mode 100644 index 0000000000..ce9e42d36a --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/Wrk2ExecutorTest.cs @@ -0,0 +1,532 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Reflection; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Common; + using VirtualClient.Contracts; + using Microsoft.CodeAnalysis; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Moq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Telemetry; + using Architecture = System.Runtime.InteropServices.Architecture; + + [TestFixture] + [Category("Unit")] + public class Wrk2ExecutorTests + { + private string ClientStateId = nameof(ClientStateId); + private string ServerStateId = nameof(ServerStateId); + + private MockFixture mockFixture; + private DependencyFixture dependencyFixture; + private InMemoryProcess memoryProcess; + private Dictionary defaultProperties; + private string packageName = "wrk2"; + private string scriptpackageName = "wrkconfiguration"; + + [SetUp] + public void SetupTests() + { + this.mockFixture = new MockFixture(); + this.memoryProcess = new InMemoryProcess + { + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + + this.defaultProperties = new Dictionary() + { + { "PackageName", this.packageName }, + { "Scenario", "1000r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb" }, + { "CommandArguments", "--latency --threads{ThreadCount} --connections{Connection} --duration{Duration.TotalSeconds}s" }, + { "Connection", 100 }, + { "ThreadCount", 10 }, + { "MaxCoreCount", 10 }, + { "TestDuration", "00:02:30"}, + { "Timeout", "00:20:00"}, + { "FileSizeInKB", 10}, + { "Role", "Client"}, + { "Tags", "Networking,NGINX,WRK2"}, + }; + + dependencyFixture = new DependencyFixture(); + } + + [Test] + [TestCase(PlatformID.Win32NT, Architecture.X64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public void Wrk2ExecutorThrowsErrorIfPlatformIsWrong(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ClientStateId); + this.mockFixture.Parameters = this.defaultProperties; + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + Assert.IsFalse(VirtualClientComponent.IsSupported(executor)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void Wrk2ExecutorThrowsErrorIfPackageIsMissing(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ClientStateId); + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + + this.mockFixture.Parameters = this.defaultProperties; + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.packageName); + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.FirstOrDefault()); + + TestWrk2Executor executor2 = new TestWrk2Executor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor2.InitializeAsync().ConfigureAwait(false); + }); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void Wrk2ExecutorOnlySupportsWrk2(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ClientStateId); + this.mockFixture.Parameters = new Dictionary() + { + { "PackageName", "wrk" }, + }; + + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage("wrk"); + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.FirstOrDefault()); + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + DependencyException exc = Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + + Assert.AreEqual(exc.Message, "TestWrk2Executor did not find correct package in the directory. Supported Package: wrk2. Package Provided: wrk"); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task Wrk2ExecutorInitializesAsExpected(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ServerStateId); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.packageName); + dependencyFixture.SetupPackage(this.scriptpackageName); + TestWrk2Executor dummyExecutor = new TestWrk2Executor(this.mockFixture); + + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.packageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptpackageName).FirstOrDefault(), platform, architecture).Path; + string[] expectedFiles = new string[] + { + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath, "setup-reset.sh"), + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath,"setup-config.sh"), + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath,"reset.sh"), + dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath,"wrk"), + dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath,"runwrk.sh") + }; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string fileName) => + { + if (!expectedFiles.Any(y => y == fileName)) + { + Assert.Fail($"Unexpected File Name: {fileName}. \n{string.Join("\n", expectedFiles)}"); + } + }); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptpackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptpackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.packageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.packageName).FirstOrDefault()); + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(expectedFiles.Count())); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + [Ignore("Unit test is way too complex and needs to be refactored.")] + public async Task Wrk2ExecutorExecutesAsyncAsExpected(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ServerStateId); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.packageName); + dependencyFixture.SetupPackage(this.scriptpackageName); + TestWrk2Executor dummyExecutor = new TestWrk2Executor(this.mockFixture); + + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.packageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptpackageName).FirstOrDefault(), platform, architecture).Path; + string[] expectedFiles = new string[] + { + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath, "setup-reset.sh"), + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath,"setup-config.sh"), + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath,"reset.sh"), + dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath,"wrk"), + dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath,"runwrk.sh") + }; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string fileName) => + { + if (!expectedFiles.Any(y => y == fileName)) + { + Assert.Fail($"Unexpected File Name: {fileName}. \n{string.Join("\n", expectedFiles)}"); + } + }); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptpackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptpackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.packageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.packageName).FirstOrDefault()); + + Item serverState = new Item(this.ServerStateId, new State()); + serverState.Definition.Online(true); + Item clientState = new Item(this.ClientStateId, new State()); + + // create local state + this.mockFixture.ApiClient + .Setup(x => x.CreateStateAsync(It.Is(y => y == this.ClientStateId), It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)) + .Callback((string id, State state, CancellationToken _, IAsyncPolicy __) => + { + Assert.IsTrue(state.Online()); + }); + + this.mockFixture.ApiClient.Setup(s => s.GetStateAsync(It.Is(x => x == this.ServerStateId), It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => + { + Item result = new Item(this.ServerStateId, new State()); + result.Definition.Online(true); + return this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, result); + }); + + this.mockFixture.ApiClient.Setup(s => s.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => + { + // both ServerStateId and serverVersion will be covered with this set up. + Item result = new Item(this.ServerStateId, new State()); + result.Definition.Online(true); + return this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, result); + }); + + // update local state before running working + this.mockFixture.ApiClient + .Setup(x => x.UpdateStateAsync(It.Is(y => y == this.ClientStateId), It.IsAny>(), It.IsAny(), It.IsAny>())) + .Callback((string id, Item state, CancellationToken _, IAsyncPolicy __) => + { + Assert.IsTrue(state.Definition.Online()); + }); + + // delete local state before exiting + this.mockFixture.ApiClient.Setup(x => x.DeleteStateAsync(It.Is(y => y == this.ClientStateId), It.IsAny(), It.IsAny>())); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.IsTrue(command == "sudo" || command.EndsWith("wrk")); + + if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")} --version")); + } + + return this.memoryProcess; + }; + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + Assert.ThrowsAsync(async () => + { + await executor.ExecuteAsync().ConfigureAwait(false); + }, "wrk2 did not write metrics to console."); + + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(8)); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.IsTrue(command == "sudo" || command.EndsWith("wrk")); + + if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")} --version")); + } + else + { + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + + return this.memoryProcess; + }; + TestWrk2Executor executor2 = new TestWrk2Executor(this.mockFixture); + await executor2.InitializeAsync().ConfigureAwait(false); + await executor2.ExecuteAsync().ConfigureAwait(false); + + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(15)); + this.mockFixture.ApiClient.Verify(x => x.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())); + } + + [Test] + [TestCase("-L -R 1000 -t 10 -c 50 -d 100s --timeout 10s https://{serverip}/api_new/1kb")] + [TestCase("{serverip}_{clientip}_{reverseproxyip}_{serverip}")] + public void WrkClientExecutorReturnsCorrectArguments(string commandArg) + { + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); + ClientInstance serverInstance = new ClientInstance(name: nameof(State), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(State), ipAddress: "5.6.7.8", role: ClientRole.Client); + ClientInstance reverseProxyInstance = new ClientInstance(name: nameof(State), ipAddress: "9.0.1.2", role: ClientRole.ReverseProxy); + + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance, reverseProxyInstance }); + + this.mockFixture.Parameters = new Dictionary() + { + { "CommandArguments", commandArg } + }; + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + string expected = executor.GetCommandLineArguments(); + + string result = commandArg + .Replace("{serverip}", "1.2.3.4") + .Replace("{clientip}", "5.6.7.8") + .Replace("{reverseproxyip}", "9.0.1.2"); + + Assert.AreEqual(expected, result); + } + + [Test] + public void Wrk2ExecutorThrowsWhenBindToCoresIsTrueButCoreAffinityIsNotProvided() + { + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); + ClientInstance serverInstance = new ClientInstance(name: nameof(State), ipAddress: "1.2.3.4", role: ClientRole.Server); + + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance }); + this.mockFixture.Parameters = new Dictionary() + { + { "PackageName", "wrk2" }, + { "CommandArguments", "-t 10 -c 100 -d 15s http://{ServerIp}:9876/json" }, + { "BindToCores", true } + }; + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + Assert.Throws(() => executor.Validate()); + } + + [Test] + [Ignore("Unit test requires additional state management setup and needs to be simplified.")] + public async Task Wrk2ExecutorExecutesWithCoreAffinityOnLinux() + { + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64, nameof(State)); + ClientInstance serverInstance = new ClientInstance(name: nameof(State), ipAddress: "1.2.3.4", role: ClientRole.Server); + + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance }); + this.mockFixture.Parameters = new Dictionary() + { + { "PackageName", "wrk2" }, + { "CommandArguments", "-t 10 -c 100 -d 15s http://{ServerIp}:9876/json" }, + { "BindToCores", true }, + { "CoreAffinity", "0-7" }, + { "TargetService", "server" } + }; + + DependencyFixture dependencyFixture = new DependencyFixture(); + dependencyFixture.Setup(PlatformID.Unix, Architecture.X64); + dependencyFixture.SetupPackage("wrk2"); + dependencyFixture.SetupPackage("wrkconfiguration"); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((string name, CancellationToken token) => dependencyFixture.PackageManager.FirstOrDefault(p => p.Name == name)); + + this.mockFixture.FileSystem.Setup(x => x.File.Exists(It.IsAny())).Returns(true); + + // Setup API client for state management + this.mockFixture.ApiClient + .Setup(x => x.CreateStateAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK)); + + this.mockFixture.ApiClient + .Setup(x => x.UpdateStateAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK)); + + this.mockFixture.ApiClient + .Setup(x => x.DeleteStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.NoContent)); + + this.mockFixture.ApiClient.Setup(s => s.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => + { + Item result = new Item(nameof(State), new State()); + result.Definition.Online(true); + return this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, result); + }); + + this.mockFixture + .TrackProcesses() + .SetupProcessOutput( + "--version", + "wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer") + .SetupProcessOutput( + ".*wrk.*-t 10 -c 100.*", + File.ReadAllText(Path.Combine( + Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), + "Examples", "Wrk", "wrkStandardExample1.txt"))); + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + await executor.InitializeAsync(); + await executor.ExecuteAsync(); + + // Verify numactl was used with correct core affinity + this.mockFixture.Tracking.AssertCommandsExecuted(true, + "sudo bash -c \\\"numactl -C 0-7 .*wrk.*" + ); + + this.mockFixture.Tracking.AssertCommandExecutedTimes("numactl", 1); + } + + [Test] + public async Task WrkClientExecutorReturnsCorrectArguments() + { + string commandArgumentInput = @"--rate 1000 --latency --threads 10 --connections 100 --duration 100s --timeout 10s https://{serverip}/api_new/5kb"; + ClientInstance serverInstance = new ClientInstance(name: nameof(State), ipAddress: "1.2.3.4", role: ClientRole.Server); + + string directory = @"some/random\dir/name/"; + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance }); + this.mockFixture.Parameters = new Dictionary() + { + {"CommandArguments", commandArgumentInput }, + { "Scenario", "bar" }, + { "ToolName", "wrk" }, + { "PackageName", "wrk" }, + { "FileSizeInKB", 5}, + { "TestDuration", TimeSpan.FromSeconds(60).ToString() } + }; + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + executor.PackageDirectory = directory; + string result = executor.GetCommandLineArguments(); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string file) => + { + string result = executor.Combine(directory, "runwrk.sh"); + Assert.AreEqual(file, result); + }); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + string results = commandArgumentInput.Replace("{serverip}", "1.2.3.4"); + Assert.AreEqual(command, "sudo"); + if (arguments.Contains("--version")) + { + Assert.AreEqual(arguments, $"bash {executor.Combine(directory, "runwrk.sh")} --version"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + } + else + { + Assert.AreEqual(arguments, $"bash {executor.Combine(directory, "runwrk.sh")} \"{results}\""); + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + Assert.AreEqual(workingDir, directory); + return this.memoryProcess; + }; + + await executor.ExecuteWorkloadAsync(result, workingDir: directory).ConfigureAwait(false); + } + + public void SetUpWorkloadOutput() + { + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample2.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + + private class TestWrk2Executor : Wrk2Executor + { + public TestWrk2Executor(MockFixture mockFixture) + : base(mockFixture.Dependencies, mockFixture.Parameters) + { + this.ServerApi = mockFixture.ApiClient.Object; + this.ClientFlowRetryPolicy = Policy.NoOpAsync(); + this.ClientRetryPolicy = Policy.NoOpAsync(); + } + + public new void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public string GetCommandLineArguments() + { + return base.GetCommandLineArguments(CancellationToken.None); + } + + public async Task InitializeAsync() + { + await base.InitializeAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ExecuteAsync() + { + await base.ExecuteAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ExecuteWorkloadAsync(string commandArguments, string workingDir) + { + await base.ExecuteWorkloadAsync(commandArguments, workingDir, EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public new void Validate() + { + base.Validate(); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs new file mode 100644 index 0000000000..ef1c179329 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs @@ -0,0 +1,683 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Reflection; + using System.Text; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Common; + using VirtualClient.Actions.Memtier; + using VirtualClient.Contracts; + using Microsoft.CodeAnalysis; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Moq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Telemetry; + using Architecture = System.Runtime.InteropServices.Architecture; + + [TestFixture] + [Category("Unit")] + public class WrkExecutorTests + { + private MockFixture mockFixture; + private DependencyFixture dependencyFixture; + private InMemoryProcess memoryProcess; + private Dictionary defaultProperties; + private string wrkPackageName = "wrk"; + private string scriptPackageName = "wrkconfiguration"; + + [SetUp] + public void SetupTests() + { + this.mockFixture = new MockFixture(); + this.memoryProcess = new InMemoryProcess + { + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + this.defaultProperties = new Dictionary() + { + { "PackageName", this.wrkPackageName }, + { "Scenario", "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb" }, + { "CommandArguments", "--latency --threads{ThreadCount} --connections{Connection} --duration{Duration.TotalSeconds}s" }, + { "Connection", 100 }, + { "ThreadCount", 10 }, + { "MaxCoreCount", 10 }, + { "TestDuration", "00:02:30"}, + { "Timeout", "00:20:00"}, + { "FileSizeInKB", 10}, + { "Role", "Client"}, + { "Tags", "Networking,NGINX,WRK"}, + }; + + dependencyFixture = new DependencyFixture(); + } + + public void SetSingleServerInstance() + { + // Setup: + // One server instance running on port 9876 with affinity to 4 logical processors + this.mockFixture.ApiClient.OnGetState(nameof(ServerState)) + .ReturnsAsync(this.mockFixture.CreateHttpResponse( + HttpStatusCode.OK, + new Item(nameof(ServerState), new ServerState(new List + { + new PortDescription + { + CpuAffinity = "0,1,2,3", + Port = 9876 + } + })))); + + this.mockFixture.ApiClientManager.Setup(mgr => mgr.GetOrCreateApiClient(It.IsAny(), It.IsAny())) + .Returns((id, instance) => this.mockFixture.ApiClient.Object); + + this.mockFixture.ApiClient.OnGetHeartbeat() + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + + this.mockFixture.ApiClient.OnGetServerOnline() + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + } + + [Test] + [TestCase("-L -t 10 -c 50 -d 100s --timeout 10s https://{serverip}/api_new/1kb")] + [TestCase("{serverip}_{clientip}_{reverseproxyip}")] + public void WrkClientExecutorReturnsCorrectArguments(string commandArg) + { + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + ClientInstance reverseProxyInstance = new ClientInstance(name: nameof(ClientRole.ReverseProxy), ipAddress: "9.0.1.2", role: ClientRole.ReverseProxy); + + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance, reverseProxyInstance }); + + this.mockFixture.Parameters = new Dictionary() + { + { "CommandArguments", commandArg } + }; + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + string results = commandArg + .Replace("{serverip}", "1.2.3.4") + .Replace("{clientip}", "5.6.7.8") + .Replace("{reverseproxyip}", "9.0.1.2"); + + Assert.AreEqual(executor.GetCommandLineArguments(), results); + } + + [Test] + public async Task WrkClientExecutorRunsWorkloadWithCorrectArguments() + { + string commandArgumentInput = @"--latency --threads 5 --connections 100 --duration 60s --timeout 10s https://{serverip}/api_new/5kb"; + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + ClientInstance reverseProxyInstance = new ClientInstance(name: nameof(ClientRole.ReverseProxy), ipAddress: "9.0.1.2", role: ClientRole.ReverseProxy); + + string directory = @"some/random\dir/name/"; + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance, reverseProxyInstance }); + this.mockFixture.Parameters = new Dictionary() + { + {"CommandArguments", commandArgumentInput }, + { "Scenario", "bar" }, + { "ToolName", "wrk" }, + { "PackageName", "wrk" }, + { "FileSizeInKB", 5}, + { "TestDuration", TimeSpan.FromSeconds(60).ToString() }, + { "TargetService", "server"} + }; + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + executor.PackageDirectory = directory; + this.SetUpWorkloadOutput(); + string result = executor.GetCommandLineArguments(); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string file) => + { + string result = executor.Combine(directory, "runwrk.sh"); + Assert.AreEqual(file, result); + }); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + string results = commandArgumentInput.Replace("{serverip}", "1.2.3.4"); + Assert.AreEqual(command, "sudo"); + if (arguments.Contains("--version")) + { + Assert.AreEqual(arguments, $"bash {executor.Combine(directory, "runwrk.sh")} --version"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + } + else + { + Assert.AreEqual(arguments, $"bash {executor.Combine(directory, "runwrk.sh")} \"{results}\""); + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + Assert.AreEqual(workingDir, directory); + return this.memoryProcess; + }; + + await executor.ExecuteWorkloadAsync(result, workingDir: directory).ConfigureAwait(false); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task WrkClientExecutorSetsServerWarmedUpFlagAfterWarmupExecution(PlatformID platform, Architecture architecture) + { + // Arrange + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance }); + + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + + // Setup parameters with WarmUp=true + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.Parameters.Add("WarmUp", true); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((string packageName, CancellationToken cancellationToken) => + dependencyFixture.PackageManager.FirstOrDefault(pkg => pkg.Name == packageName)); + + // Setup API client for server heartbeat and state + this.mockFixture.ApiClient.Setup(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => + { + Item result = new Item(nameof(State), new State()); + result.Definition.Online(true); + return this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, result); + }); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + } + else + { + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + + return this.memoryProcess; + }; + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + + // Act + await executor.InitializeAsync().ConfigureAwait(false); + await executor.ExecuteAsync().ConfigureAwait(false); + + // Assert + Assert.IsTrue(executor.GetIsServerWarmedUp(), "IsServerWarmedUp flag should be set to true after warm-up execution"); + } + + [Test] + [TestCase(PlatformID.Win32NT, Architecture.X64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public void WrkClientExecutorThrowsErrorIfPlatformIsWrong(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters = this.defaultProperties; + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + Assert.IsFalse(VirtualClientComponent.IsSupported(executor)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void WrkExecutorOnlySupportsWrkandWrk2(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters = new Dictionary() + { + { "PackageName", "wrk2" }, + }; + + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage("wrk2"); + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.FirstOrDefault()); + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + DependencyException exc = Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + + Assert.AreEqual(exc.Message, "TestWrkExecutor did not find correct package in the directory. Supported Package: wrk. Package Provided: wrk2"); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void WrkClientExecutorThrowsErrorIfPackageIsMissing(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.SetSingleServerInstance(); + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + + this.mockFixture.Parameters = this.defaultProperties; + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.FirstOrDefault()); + + TestWrkExecutor executor2 = new TestWrkExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor2.InitializeAsync().ConfigureAwait(false); + }); + + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task WrkClientExecutorSkipsExecutionWhenWarmupAndServerIsWarmedUp(PlatformID platform, Architecture architecture) + { + // Arrange + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance }); + + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.Parameters.Add("WarmUp", true); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((string packageName, CancellationToken cancellationToken) => + dependencyFixture.PackageManager.FirstOrDefault(pkg => pkg.Name == packageName)); + + // Used to track if ExecuteWorkloadAsync is called + bool workloadExecuted = false; + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + executor.SetIsServerWarmedUp(true); // Simulate server already warmed up + + // Override the ExecuteWorkloadAsync to track if it's called + executor.ExecuteWorkloadAsyncCallback = () => workloadExecuted = true; + + // Act + await executor.InitializeAsync().ConfigureAwait(false); + await executor.ExecuteAsync().ConfigureAwait(false); + + // Assert + Assert.IsFalse(workloadExecuted, "WorkloadAsync should not be executed when WarmUp=true and server is already warmed up"); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task WrkClientExecutorSetsUpWrkClientForFirstTime(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + TestWrkExecutor dummyExecutor = new TestWrkExecutor(this.mockFixture); + + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault(), platform, architecture).Path; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.Is(x => x == "reset.sh"))) + .Returns(false); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.Is(x => x != "reset.sh"))) + .Returns(true); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.wrkPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault()); + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task WrkClientExecutorInitializesAsExpected(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + TestWrkExecutor dummyExecutor = new TestWrkExecutor(this.mockFixture); + + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault(), platform, architecture).Path; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string file) => + { + if (file.EndsWith("wrk")) + { + string result = dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "wrk"); + Assert.AreEqual(file, result); + } + else + { + string result = dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath, Path.GetFileName(file)); + Assert.AreEqual(file, result); + } + }); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.wrkPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault()); + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64, "server")] + [TestCase(PlatformID.Unix, Architecture.Arm64, "server")] + [TestCase(PlatformID.Unix, Architecture.X64, "rp")] + [TestCase(PlatformID.Unix, Architecture.Arm64, "rp")] + public async Task WrkClientExecutorExecutesAsyncAsExpected(PlatformID platform, Architecture architecture, string targetService) + { + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + ClientInstance reverseProxyInstance = new ClientInstance(name: nameof(ClientRole.ReverseProxy), ipAddress: "9.0.1.2", role: ClientRole.ReverseProxy); + + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance, reverseProxyInstance }); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + TestWrkExecutor dummyExecutor = new TestWrkExecutor(this.mockFixture); + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault(), platform, architecture).Path; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.Parameters.Add("TargetService", targetService); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string file) => + { + if (file.EndsWith("wrk")) + { + string result = dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "wrk"); + Assert.AreEqual(file, result); + } + else if (file.EndsWith("runwrk.sh")) + { + string result = dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh"); + Assert.AreEqual(file, result); + } + else + { + string result = dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath, Path.GetFileName(file)); + Assert.AreEqual(file, result); + } + }); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.wrkPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault()); + + // create local state + this.mockFixture.ApiClient + .Setup(x => x.CreateStateAsync(It.Is(y => y == nameof(State)), It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)) + .Callback((string id, State state, CancellationToken _, IAsyncPolicy __) => + { + Assert.IsTrue(state.Online()); + }); + + this.mockFixture.ApiClient.Setup(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => + { + Item result = new Item(nameof(State), new State()); + result.Definition.Online(true); + return this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, result); + }); + + // delete local state before exiting + this.mockFixture.ApiClient.Setup(x => x.DeleteStateAsync(It.Is(y => y == nameof(State)), It.IsAny(), It.IsAny>())); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.AreEqual(command, "sudo"); + + if (arguments.Contains("chmod")) + { + Assert.AreEqual(arguments, $"chmod +x \"{dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "wrk")}\""); + } + else if (arguments.Contains("setup-config")) + { + Assert.AreEqual(arguments, $"bash setup-config.sh"); + } + else if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")} --version")); + } + else + { + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")}")); + } + + return this.memoryProcess; + }; + //this.SetUpWorkloadOutput(); + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + Assert.ThrowsAsync(async () => + { + await executor.ExecuteAsync().ConfigureAwait(false); + }); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.AreEqual(command, "sudo"); + + if (arguments.Contains("chmod")) + { + Assert.AreEqual(arguments, $"chmod +x \"{dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "wrk")}\""); + } + else if (arguments.Contains("setup-config")) + { + Assert.AreEqual(arguments, $"bash setup-config.sh"); + } + else if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")} --version")); + } + else + { + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")}")); + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + + return this.memoryProcess; + }; + TestWrkExecutor executor2 = new TestWrkExecutor(this.mockFixture); + await executor2.InitializeAsync().ConfigureAwait(false); + await executor2.ExecuteAsync().ConfigureAwait(false); + + this.mockFixture.ApiClient.Verify(x => x.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())); + } + + [Test] + public void GetWrkVersionReturnsCorrectVersion() + { + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + string expectedVersion = "4.2.0"; + string wrkOutput = $"wrk {expectedVersion} [epoll] Copyright (C) 2012 Will Glozer"; + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture + .TrackProcesses() + .SetupProcessOutput("--version", wrkOutput); + + string actualVersion = executor.GetWrkVersion(); + + Assert.AreEqual(expectedVersion, actualVersion); + this.mockFixture.Tracking.AssertCommandsExecuted(true, + $"sudo bash {Regex.Escape(executor.Combine(executor.PackageDirectory, WrkExecutor.WrkRunShell))} --version" + ); + } + + [Test] + public void GetWrkVersion_ThrowsException_WhenVersionCannotBeParsed() + { + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture + .TrackProcesses() + .SetupProcessOutput("--version", "Invalid output without version"); + + WorkloadException exception = Assert.Throws(() => executor.GetWrkVersion()); + Assert.AreEqual("Failed to parse wrk version from output.", exception.Message); + + this.mockFixture.Tracking.AssertCommandsExecuted(true, "sudo bash .* --version"); + } + + + public void SetUpWorkloadOutput() + { + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("")); + } + + private class TestWrkExecutor : WrkExecutor + { + public Action ExecuteWorkloadAsyncCallback { get; set; } + + public TestWrkExecutor(MockFixture mockFixture) + : base(mockFixture.Dependencies, mockFixture.Parameters) + { + this.ServerApi = mockFixture.ApiClient.Object; + this.ReverseProxyApi = mockFixture.ApiClient.Object; + this.ClientFlowRetryPolicy = Policy.NoOpAsync(); + this.ClientRetryPolicy = Policy.NoOpAsync(); + } + + public new void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public string GetCommandLineArguments() + { + return base.GetCommandLineArguments(CancellationToken.None); + } + + public bool GetIsServerWarmedUp() + { + return base.IsServerWarmedUp; + } + + public string GetWrkVersion() + { + return base.GetWrkVersion(EventContext.None, CancellationToken.None); + } + + public void SetIsServerWarmedUp(bool value) + { + base.IsServerWarmedUp = value; + } + + public async Task InitializeAsync() + { + await base.InitializeAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task SetupWrkClient() + { + await base.SetupWrkClient(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ExecuteWorkloadAsync(string commandArguments, string workingDir) + { + await base.ExecuteWorkloadAsync(commandArguments, workingDir, EventContext.None, CancellationToken.None).ConfigureAwait(false); + ExecuteWorkloadAsyncCallback?.Invoke(); + } + + public async Task ExecuteAsync() + { + await base.ExecuteAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs new file mode 100644 index 0000000000..648808b058 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs @@ -0,0 +1,174 @@ +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using VirtualClient.Contracts; +using NUnit.Framework; +using VirtualClient; +using VirtualClient.Actions; + +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + [TestFixture] + [Category("Unit")] + public class WrkMetricsParserTest + { + private static string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + + [Test] + [TestCase(true)] + [TestCase(false)] + [TestCase(null)] + public void WRKParsesResultsCorrectly01(bool emitSpectrum) + { + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + IList actualMetrics = parser.Parse(emitSpectrum); + + Assert.AreEqual(parser.GetTestConfig(), "Running 1m test @ https://10.1.0.26/api_new/1kb with 2 threads and 20 connections"); + MetricAssert.Exists(actualMetrics, "latency_p50", 29.56 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p75", 39.75 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p90", 45.97 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99", 49.74 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_9", 50.17 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_99", 50.23 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_999", 50.23 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p100", 50.23 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "requests/sec", 16305.17); + MetricAssert.Exists(actualMetrics, "transfers/sec", 20.01, MetricUnit.Megabytes); + + if (emitSpectrum == true) + { + + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_000000", 8372.223); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_887500", 45449.215); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_997266", 50069.503); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_999805", 50200.575); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_999902", 50233.343); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p1_000000", 50233.343); + } + + } + + [Test] + public void WRKParsesResultsCorrectly02() + { + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample2.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + IList actualMetrics = parser.Parse(true); + + // Error + MetricAssert.Exists(actualMetrics, "Non-2xx or 3xx responses", 58902); + + // Raw data + MetricAssert.Exists(actualMetrics, "requests/sec", 1963.39); + MetricAssert.Exists(actualMetrics, "transfers/sec", 0.61033203125, MetricUnit.Megabytes); + + //Latency Distribution + MetricAssert.Exists(actualMetrics, "latency_p50", 1.43 , MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p75", 1.98 , MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p90", 2.68 , MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99", 3.96, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_9", 6.93, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_99", 8.99, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_999", 9.77, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p100", 9.77, MetricUnit.Milliseconds); + + //Uncorrected Latency Distribution + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p50", 483 * 0.001, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p75", 1.12, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p90", 1.71, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p99", 2.87, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p99_9", 5.76, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p99_99", 8.02, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p99_999", 8.41, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p100", 8.41, MetricUnit.Milliseconds); + + // latency spectrum + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_000000", 0.175); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_775000", 2.067); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_937500", 2.981); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_998437", 6.331); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_999219", 7.511); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_999969", 9.503); + + // Uncorrected latency spectrum + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_000000", 0.135); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_100000", 0.242); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_200000", 0.287); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_981250", 2.531); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_996094", 3.793); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_999939", 8.223); + + Assert.AreEqual(parser.GetTestConfig(), "Running 30s test @ http://10.1.0.13/index.html with 2 threads and 100 connections"); + } + + [Test] + public void WRKParsesResultsCorrectly03() + { + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample3.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + IList actualMetrics = parser.Parse(true); + + MetricAssert.Exists(actualMetrics, "Non-2xx or 3xx responses", 184596465); + Assert.AreEqual(parser.GetTestConfig(), "Running 2m test @ http://10.9.0.7/api_new/10kb with 32 threads and 5000 connections"); + // Raw data + MetricAssert.Exists(actualMetrics, "requests/sec", 1229838.61); + MetricAssert.Exists(actualMetrics, "transfers/sec", 382.35, MetricUnit.Megabytes); + + //Latency Distribution + MetricAssert.Exists(actualMetrics, "latency_p50", 3.85, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p75", 5.05, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p90", 7.27, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99", 11.16, MetricUnit.Milliseconds); + } + + [Test] + public void WRKParsesErrorCorrectly01() + { + string outputPath = Path.Combine(examplesDirectory, @"wrkErrorExample1.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + Assert.AreEqual(parser.GetTestConfig(), "Running 2m test @ http://10.1.0.15/api_new/10kb with 1 threads and 10000 connections"); + + WorkloadException exc = Assert.Throws(() => + { + parser.Parse(); + }); + + Assert.AreEqual(exc.Message, "Socket errors: connect 0, read 1645610, write 16, timeout 0"); + } + + [Test] + public void WRKParsesErrorCorrectly02() + { + string outputPath = Path.Combine(examplesDirectory, @"wrkErrorExample2.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + Assert.AreEqual(parser.GetTestConfig(), "Running 30s test @ http://10.1.0.13/index.html with 5 threads and 100 connections"); + + Assert.DoesNotThrow(() => { parser.Parse(); }); + + IList actualMetrics = parser.Parse(); + MetricAssert.Exists(actualMetrics, "Non-2xx or 3xx responses", 59956); + + MetricAssert.Exists(actualMetrics, "requests/sec", 1998.42); + MetricAssert.Exists(actualMetrics, "transfers/sec", 636.13/1024.0, MetricUnit.Megabytes); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchBaseExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchBaseExecutor.cs deleted file mode 100644 index 1628f0192e..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchBaseExecutor.cs +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.IO.Abstractions; - using System.Runtime.InteropServices; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.CodeAnalysis; - using Microsoft.Extensions.DependencyInjection; - using VirtualClient.Common; - using VirtualClient.Common.Extensions; - using VirtualClient.Common.Platform; - using VirtualClient.Common.Telemetry; - using VirtualClient.Contracts; - using VirtualClient.Contracts.Metadata; - using static System.Net.Mime.MediaTypeNames; - - /// - /// The AspNetBench workload executor. - /// - [SupportedPlatforms("linux-arm64,linux-x64,win-arm64,win-x64")] - public abstract class AspNetBenchBaseExecutor : VirtualClientMultiRoleComponent - { - private IFileSystem fileSystem; - private IPackageManager packageManager; - private IStateManager stateManager; - private ISystemManagement systemManagement; - - private string dotnetExePath; - private string aspnetBenchDirectory; - private string aspnetBenchDllPath; - private string bombardierFilePath; - private string wrkFilePath; - private string serverArgument; - private string clientArgument; - - /// - /// Constructor for - /// - /// Provides required dependencies to the component. - /// Parameters defined in the profile or supplied on the command line. - public AspNetBenchBaseExecutor(IServiceCollection dependencies, IDictionary parameters) - : base(dependencies, parameters) - { - this.systemManagement = this.Dependencies.GetService(); - this.packageManager = this.systemManagement.PackageManager; - this.stateManager = this.systemManagement.StateManager; - this.fileSystem = this.systemManagement.FileSystem; - } - - /// - /// The name of the package where the AspNetBench package is downloaded. - /// - public string TargetFramework - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.TargetFramework)).ToLower(); - } - } - - /// - /// The port for ASPNET to run. - /// - public string Port - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.Port), "9876"); - } - } - - /// - /// The name of the package where the bombardier package is downloaded. - /// - public string BombardierPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.BombardierPackageName), "bombardier"); - } - } - - /// - /// The name of the package where the wrk package is downloaded. - /// - public string WrkPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.WrkPackageName), "wrk"); - } - } - - /// - /// The name of the package where the DotNetSDK package is downloaded. - /// - public string DotNetSdkPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.DotNetSdkPackageName), "dotnetsdk"); - } - } - - /// - /// ASPNETCORE_threadCount - /// - public string AspNetCoreThreadCount - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.AspNetCoreThreadCount), 1); - } - } - - /// - /// DOTNET_SYSTEM_NET_SOCKETS_THREAD_COUNT - /// - public string DotNetSystemNetSocketsThreadCount - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.DotNetSystemNetSocketsThreadCount), 1); - } - } - - /// - /// wrk commandline - /// - public string WrkCommandLine - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.WrkCommandLine), string.Empty); - } - } - - /// - /// Initializes the environment for execution of the AspNetBench workload. - /// - protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - // This workload needs three packages: aspnetbenchmarks, dotnetsdk, bombardier - DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, CancellationToken.None) - .ConfigureAwait(false); - - if (workloadPackage != null) - { - // the directory we are looking for is at the src/Benchmarks - this.aspnetBenchDirectory = this.Combine(workloadPackage.Path, "src", "Benchmarks"); - } - - try - { - // Check for Bombardier Package, if not available try wrk package - DependencyPath bombardierPackage = await this.GetPlatformSpecificPackageAsync(this.BombardierPackageName, cancellationToken); - - if (bombardierPackage != null) - { - this.bombardierFilePath = this.Combine(bombardierPackage.Path, this.Platform == PlatformID.Unix ? "bombardier" : "bombardier.exe"); - await this.systemManagement.MakeFileExecutableAsync(this.bombardierFilePath, this.Platform, cancellationToken); - } - } - catch (DependencyException) - { - DependencyPath wrkPackage = await this.packageManager.GetPackageAsync(this.WrkPackageName, cancellationToken); - - if (wrkPackage != null) - { - this.wrkFilePath = this.Combine(wrkPackage.Path, "wrk"); - await this.systemManagement.MakeFileExecutableAsync(this.wrkFilePath, this.Platform, cancellationToken); - } - } - } - - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - /// - protected async Task BuildAspNetBenchAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - DependencyPath dotnetSdkPackage = await this.packageManager.GetPackageAsync(this.DotNetSdkPackageName, CancellationToken.None) - .ConfigureAwait(false); - - if (dotnetSdkPackage == null) - { - throw new DependencyException( - $"The expected DotNet SDK package does not exist on the system or is not registered.", - ErrorReason.WorkloadDependencyMissing); - } - - this.dotnetExePath = this.Combine(dotnetSdkPackage.Path, this.Platform == PlatformID.Unix ? "dotnet" : "dotnet.exe"); - // ~/vc/packages/dotnet/dotnet build -c Release -p:BenchmarksTargetFramework=net9.0 - // Build the aspnetbenchmark project - string buildArgument = $"build -c Release -p:BenchmarksTargetFramework={this.TargetFramework}"; - await this.ExecuteCommandAsync(this.dotnetExePath, buildArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken) - .ConfigureAwait(false); - - // "C:\Users\vcvmadmin\Benchmarks\src\Benchmarks\bin\Release\net9.0\Benchmarks.dll" - this.aspnetBenchDllPath = this.Combine( - this.aspnetBenchDirectory, - "bin", - "Release", - this.TargetFramework, - "Benchmarks.dll"); - } - - /// - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// - protected void CaptureMetrics(IProcessProxy process, EventContext telemetryContext) - { - try - { - this.MetadataContract.AddForScenario( - "AspNetBench", - $"{this.clientArgument},{this.serverArgument}", - toolVersion: null); - - this.MetadataContract.Apply(telemetryContext); - - WrkMetricParser parser = new WrkMetricParser(process.StandardOutput.ToString()); - - this.Logger.LogMetrics( - toolName: "AspNetBench", - scenarioName: $"ASP.NET_{this.TargetFramework}_Performance", - process.StartTime, - process.ExitTime, - parser.Parse(), - metricCategorization: "json", - scenarioArguments: $"Client: {this.clientArgument} | Server: {this.serverArgument}", - this.Tags, - telemetryContext); - } - catch (Exception exc) - { - throw new WorkloadResultsException($"Failed to parse bombardier output.", exc, ErrorReason.InvalidResults); - } - } - - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected Task StartAspNetServerAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - // Example: - // dotnet \Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:5000 --server Kestrel --kestrelTransport Sockets --protocol http - // --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" --header "Connection: keep-alive" - - string options = $"--nonInteractive true --scenarios json --urls http://*:{this.Port} --server Kestrel --kestrelTransport Sockets --protocol http"; - string headers = @"--header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive"""; - this.serverArgument = $"{this.aspnetBenchDllPath} {options} {headers}"; - - return this.ExecuteCommandAsync(this.dotnetExePath, this.serverArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken); - } - - /// - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected async Task RunBombardierAsync(string ipAddress, EventContext telemetryContext, CancellationToken cancellationToken) - { - using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) - { - // https://pkg.go.dev/github.com/codesenberg/bombardier - // ./bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:5000/json --print r --format json - this.clientArgument = $"--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ipAddress}:{this.Port}/json --print r --format json"; - - using (IProcessProxy process = await this.ExecuteCommandAsync(this.bombardierFilePath, this.clientArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken, runElevated: true) - .ConfigureAwait(false)) - { - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "AspNetBench", logToFile: true); - - process.ThrowIfWorkloadFailed(); - this.CaptureMetrics(process, telemetryContext); - } - } - } - } - - /// - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected async Task RunWrkAsync(string ipAddress, EventContext telemetryContext, CancellationToken cancellationToken) - { - using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) - { - // https://pkg.go.dev/github.com/codesenberg/bombardier - // ./wrk -t 256 -c 256 -d 15s --timeout 10s http://10.1.0.23:9876/json --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" - this.clientArgument = this.WrkCommandLine; - this.clientArgument = this.clientArgument.Replace("{ipAddress}", ipAddress); - this.clientArgument = this.clientArgument.Replace("{port}", this.Port); - - using (IProcessProxy process = await this.ExecuteCommandAsync(this.wrkFilePath, this.clientArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken, runElevated: true) - .ConfigureAwait(false)) - { - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "wrk", logToFile: true); - - process.ThrowIfWorkloadFailed(); - this.CaptureMetrics(process, telemetryContext); - } - } - } - } - } -} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchClientExecutor.cs deleted file mode 100644 index f8af30830a..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchClientExecutor.cs +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.Linq; - using System.Net; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Logging; - using Polly; - using VirtualClient; - using VirtualClient.Actions.Memtier; - using VirtualClient.Actions.NetworkPerformance; - using VirtualClient.Common; - using VirtualClient.Common.Contracts; - using VirtualClient.Common.Extensions; - using VirtualClient.Common.Telemetry; - using VirtualClient.Contracts; - using VirtualClient.Contracts.Metadata; - - /// - /// AspNetBench Benchmark Client Executor. - /// - public class AspNetBenchClientExecutor : AspNetBenchBaseExecutor - { - /// - /// Initializes a new instance of the class. - /// - /// Provides all of the required dependencies to the Virtual Client component. - /// An enumeration of key-value pairs that can control the execution of the component./param> - public AspNetBenchClientExecutor(IServiceCollection dependencies, IDictionary parameters = null) - : base(dependencies, parameters) - { - this.PollingTimeout = TimeSpan.FromMinutes(40); - } - - /// - /// Executes client side. - /// - protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - await this.WaitForRoleAsync(ClientRole.Server, telemetryContext, cancellationToken).ConfigureAwait(false); - string serverIPAddress = this.GetLayoutClientInstances(ClientRole.Server).First().IPAddress; - await this.RunWrkAsync(serverIPAddress, telemetryContext, cancellationToken).ConfigureAwait(false); - } - - /// - /// Initializes the environment and dependencies for client of AspNetBench Benchmark workload. - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - await base.InitializeAsync(telemetryContext, cancellationToken).ConfigureAwait(false); - - this.RegisterToTerminateRole(ClientRole.Server); - } - } -} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchExecutor.cs deleted file mode 100644 index b67cc23cfc..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchExecutor.cs +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.IO.Abstractions; - using System.Runtime.InteropServices; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.CodeAnalysis; - using Microsoft.Extensions.DependencyInjection; - using VirtualClient.Common; - using VirtualClient.Common.Extensions; - using VirtualClient.Common.Telemetry; - using VirtualClient.Contracts; - using VirtualClient.Contracts.Metadata; - - /// - /// The AspNetBench workload executor. - /// - public class AspNetBenchExecutor : VirtualClientComponent - { - private IFileSystem fileSystem; - private IPackageManager packageManager; - private IStateManager stateManager; - private ISystemManagement systemManagement; - - private string dotnetExePath; - private string aspnetBenchDirectory; - private string aspnetBenchDllPath; - private string bombardierFilePath; - private string serverArgument; - private string clientArgument; - - private Action killServer; - - /// - /// Constructor for - /// - /// Provides required dependencies to the component. - /// Parameters defined in the profile or supplied on the command line. - public AspNetBenchExecutor(IServiceCollection dependencies, IDictionary parameters) - : base(dependencies, parameters) - { - this.systemManagement = this.Dependencies.GetService(); - this.packageManager = this.systemManagement.PackageManager; - this.stateManager = this.systemManagement.StateManager; - this.fileSystem = this.systemManagement.FileSystem; - } - - /// - /// The name of the package where the AspNetBench package is downloaded. - /// - public string TargetFramework - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchExecutor.TargetFramework)).ToLower(); - } - } - - /// - /// The port for ASPNET to run. - /// - public string Port - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchExecutor.Port), "9876"); - } - } - - /// - /// The name of the package where the bombardier package is downloaded. - /// - public string BombardierPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchExecutor.BombardierPackageName), "bombardier"); - } - } - - /// - /// The name of the package where the DotNetSDK package is downloaded. - /// - public string DotNetSdkPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchExecutor.DotNetSdkPackageName), "dotnetsdk"); - } - } - - /// - /// Executes the AspNetBench workload. - /// - protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - Task serverTask = this.StartAspNetServerAsync(cancellationToken); - await this.RunBombardierAsync(telemetryContext, cancellationToken); - - this.killServer.Invoke(); - } - - /// - /// Initializes the environment for execution of the AspNetBench workload. - /// - protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - // This workload needs three packages: aspnetbenchmarks, dotnetsdk, bombardier - DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, CancellationToken.None, throwIfNotfound: true); - - // the directory we are looking for is at the src/Benchmarks - this.aspnetBenchDirectory = this.Combine(workloadPackage.Path, "src", "Benchmarks"); - - DependencyPath bombardierPackage = await this.GetPlatformSpecificPackageAsync(this.BombardierPackageName, cancellationToken); - this.bombardierFilePath = this.Combine(bombardierPackage.Path, this.Platform == PlatformID.Unix ? "bombardier" : "bombardier.exe"); - - await this.systemManagement.MakeFileExecutableAsync(this.bombardierFilePath, this.Platform, cancellationToken); - - DependencyPath dotnetSdkPackage = await this.packageManager.GetPackageAsync(this.DotNetSdkPackageName, CancellationToken.None); - - if (dotnetSdkPackage == null) - { - throw new DependencyException( - $"The expected DotNet SDK package does not exist on the system or is not registered.", - ErrorReason.WorkloadDependencyMissing); - } - - this.dotnetExePath = this.Combine(dotnetSdkPackage.Path, this.Platform == PlatformID.Unix ? "dotnet" : "dotnet.exe"); - - // ~/vc/packages/dotnet/dotnet build -c Release -p:BenchmarksTargetFramework=net9.0 - // Build the aspnetbenchmark project - string buildArgument = $"build -c Release -p:BenchmarksTargetFramework={this.TargetFramework}"; - await this.ExecuteCommandAsync(this.dotnetExePath, buildArgument, this.aspnetBenchDirectory, cancellationToken); - - // "C:\Users\vcvmadmin\Benchmarks\src\Benchmarks\bin\Release\net9.0\Benchmarks.dll" - this.aspnetBenchDllPath = this.Combine( - this.aspnetBenchDirectory, - "bin", - "Release", - this.TargetFramework, - "Benchmarks.dll"); - } - - private void CaptureMetrics(IProcessProxy process, EventContext telemetryContext) - { - try - { - this.MetadataContract.AddForScenario( - "AspNetBench", - $"{this.clientArgument},{this.serverArgument}", - toolVersion: null); - - this.MetadataContract.Apply(telemetryContext); - - BombardierMetricsParser parser = new BombardierMetricsParser(process.StandardOutput.ToString()); - - this.Logger.LogMetrics( - toolName: "AspNetBench", - scenarioName: $"ASP.NET_{this.TargetFramework}_Performance", - process.StartTime, - process.ExitTime, - parser.Parse(), - metricCategorization: "json", - scenarioArguments: $"Client: {this.clientArgument} | Server: {this.serverArgument}", - this.Tags, - telemetryContext); - } - catch (Exception exc) - { - throw new WorkloadResultsException($"Failed to parse bombardier output.", exc, ErrorReason.InvalidResults); - } - } - - private Task StartAspNetServerAsync(CancellationToken cancellationToken) - { - // Example: - // dotnet \Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:5000 --server Kestrel --kestrelTransport Sockets --protocol http - // --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" --header "Connection: keep-alive" - - string options = $"--nonInteractive true --scenarios json --urls http://localhost:{this.Port} --server Kestrel --kestrelTransport Sockets --protocol http"; - string headers = @"--header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive"""; - this.serverArgument = $"{this.aspnetBenchDllPath} {options} {headers}"; - - return this.ExecuteCommandAsync(this.dotnetExePath, this.serverArgument, this.aspnetBenchDirectory, cancellationToken, isServer: true); - } - - private async Task RunBombardierAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) - { - // https://pkg.go.dev/github.com/codesenberg/bombardier - // ./bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:5000/json --print r --format json - this.clientArgument = $"--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:{this.Port}/json --print r --format json"; - - using (IProcessProxy process = await this.ExecuteCommandAsync(this.bombardierFilePath, this.clientArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken, runElevated: true) - .ConfigureAwait(false)) - { - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "AspNetBench", logToFile: true); - - process.ThrowIfWorkloadFailed(); - this.CaptureMetrics(process, telemetryContext); - } - } - } - } - - private async Task ExecuteCommandAsync(string pathToExe, string commandLineArguments, string workingDirectory, CancellationToken cancellationToken, bool isServer = false) - { - if (!cancellationToken.IsCancellationRequested) - { - this.Logger.LogTraceMessage($"Executing process '{pathToExe}' '{commandLineArguments}' at directory '{workingDirectory}'."); - - EventContext telemetryContext = EventContext.Persisted() - .AddContext("command", pathToExe) - .AddContext("commandArguments", commandLineArguments); - - using (IProcessProxy process = this.systemManagement.ProcessManager.CreateElevatedProcess(this.Platform, pathToExe, commandLineArguments, workingDirectory)) - { - if (isServer) - { - this.killServer = () => process.SafeKill(this.Logger); - } - - this.CleanupTasks.Add(() => process.SafeKill(this.Logger)); - await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); - - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext); - - if (!isServer) - { - // We will kill the server at the end, exit code is -1, and we don't want it to log as failure. - process.ThrowIfWorkloadFailed(); - } - } - } - } - } - } -} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchServerExecutor.cs deleted file mode 100644 index db60f3b6b6..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchServerExecutor.cs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Extensions.DependencyInjection; - using VirtualClient; - using VirtualClient.Actions.Memtier; - using VirtualClient.Common.Contracts; - using VirtualClient.Common.Telemetry; - - /// - /// Redis Benchmark Client Executor. - /// - public class AspNetBenchServerExecutor : AspNetBenchBaseExecutor - { - /// - /// Initializes a new instance of the class. - /// - /// Provides all of the required dependencies to the Virtual Client component. - /// An enumeration of key-value pairs that can control the execution of the component./param> - public AspNetBenchServerExecutor(IServiceCollection dependencies, IDictionary parameters = null) - : base(dependencies, parameters) - { - } - - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - await this.BuildAspNetBenchAsync(telemetryContext, cancellationToken).ConfigureAwait(false); - - await this.StartAspNetServerAsync(telemetryContext, cancellationToken).ConfigureAwait(false); - await this.WaitAsync(cancellationToken) - .ConfigureAwait(false); - } - - private async Task GetServerStateAsync(IApiClient serverApiClient, CancellationToken cancellationToken) - { - Item state = await serverApiClient.GetStateAsync( - nameof(ServerState), - cancellationToken); - - if (state == null) - { - throw new WorkloadException( - $"Expected server state information missing. The server did not return state indicating the details for the Memcached server(s) running.", - ErrorReason.WorkloadUnexpectedAnomaly); - } - - return state.Definition; - } - } -} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs new file mode 100644 index 0000000000..3877a86d40 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs @@ -0,0 +1,421 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO.Abstractions; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// AspNet Orchard Server Executor. + /// + public class AspNetOrchardServerExecutor : VirtualClientMultiRoleComponent + { + private Task serverProcess; + private bool disposed; + private IFileSystem fileSystem; + private IPackageManager packageManager; + private IStateManager stateManager; + private ISystemManagement systemManagement; + + private string dotnetExePath; + private string aspnetOrchardDirectory; + private string aspnetOrchardPublishPath; + + /// + /// Initializes a new instance of the class. + /// + /// Provides all of the required dependencies to the Virtual Client component. + /// An enumeration of key-value pairs that can control the execution of the component./param> + public AspNetOrchardServerExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + this.systemManagement = this.Dependencies.GetService(); + this.packageManager = this.systemManagement.PackageManager; + this.stateManager = this.systemManagement.StateManager; + this.fileSystem = this.systemManagement.FileSystem; + this.ServerRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(3, (retries) => TimeSpan.FromSeconds(retries)); + } + + /// + /// The name of the package where the AspNetBench package is downloaded. + /// + public string TargetFramework + { + get + { + return this.Parameters.GetValue(nameof(AspNetOrchardServerExecutor.TargetFramework)).ToLower(); + } + } + + /// + /// The port to run for Orchard Server. + /// + public string ServerPort + { + get + { + return this.Parameters.GetValue(nameof(AspNetOrchardServerExecutor.ServerPort), "5014"); + } + } + + /// + /// API Client that is used to communicate with server-hosted instance of the Virtual Client Server. + /// + public IApiClient ServerApi { get; set; } + + /// + /// The name of the package where the wrk package is downloaded. + /// + public string WrkPackageName + { + get + { + return this.Parameters.GetValue(nameof(AspNetOrchardServerExecutor.WrkPackageName), "wrk"); + } + } + + /// + /// The name of the package where the DotNetSDK package is downloaded. + /// + public string DotNetSdkPackageName + { + get + { + return this.Parameters.GetValue(nameof(AspNetOrchardServerExecutor.DotNetSdkPackageName), "dotnetsdk"); + } + } + + /// + /// Gets or sets whether to bind the workload to specific CPU cores. + /// + public bool BindToCores + { + get + { + return this.Parameters.GetValue(nameof(this.BindToCores), defaultValue: false); + } + } + + /// + /// Gets the CPU core affinity specification (e.g., "0-3", "0,2,4,6"). + /// + public string CoreAffinity + { + get + { + this.Parameters.TryGetValue(nameof(this.CoreAffinity), out IConvertible value); + return value?.ToString(); + } + } + + /// + /// A retry policy to apply to the server when starting to handle transient issues that + /// would otherwise prevent it from starting successfully. + /// + protected IAsyncPolicy ServerRetryPolicy { get; set; } + + /// + /// Disposes of resources used by the executor including shutting down any + /// instances of Redis server running. + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + if (!this.disposed) + { + this.KillServerInstancesAsync(null, CancellationToken.None) + .GetAwaiter().GetResult(); + this.disposed = true; + } + } + } + + /// + /// Initializes the environment for execution of the AspNetBench workload. + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, cancellationToken) + .ConfigureAwait(false); + if (workloadPackage == null) + { + throw new DependencyException( + $"The expected workload package '{this.PackageName}' does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } + else + { + this.aspnetOrchardDirectory = this.Combine(workloadPackage.Path, "src", "OrchardCore.Cms.Web"); + } + + DependencyPath dotnetSdkPackage = await this.packageManager.GetPackageAsync(this.DotNetSdkPackageName, cancellationToken) + .ConfigureAwait(false); + if (dotnetSdkPackage == null) + { + throw new DependencyException( + $"The expected DotNet SDK package does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } + + this.dotnetExePath = this.Combine(dotnetSdkPackage.Path, this.Platform == PlatformID.Unix ? "dotnet" : "dotnet.exe"); + + this.InitializeApiClients(); + } + + /// + /// Validates the component parameters. + /// + protected override void Validate() + { + base.Validate(); + + if (this.BindToCores) + { + this.ThrowIfParameterNotDefined(nameof(this.CoreAffinity)); + } + } + + /// + /// Initializes API client. + /// + protected void InitializeApiClients() + { + IApiClientManager clientManager = this.Dependencies.GetService(); + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + } + + /// + /// + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + /// + protected async Task BuildAspNetOrchardAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + string buildArgument = $"publish -c Release --sc -f {this.TargetFramework} {this.Combine(this.aspnetOrchardDirectory, "OrchardCore.Cms.Web.csproj")}"; + await this.ExecuteCommandAsync(this.dotnetExePath, buildArgument, this.aspnetOrchardDirectory, telemetryContext, cancellationToken) + .ConfigureAwait(false); + + // bin/Release/net9.0/linux-x64/publish + this.aspnetOrchardPublishPath = this.Combine( + this.aspnetOrchardDirectory, + "bin", + "Release", + this.TargetFramework, + this.PlatformArchitectureName, + "publish"); + } + + /// + /// + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + protected override Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return this.Logger.LogMessageAsync($"{this.TypeName}.ExecuteServer", telemetryContext, async () => + { + try + { + this.SetServerOnline(false); + + await this.ServerApi.PollForHeartbeatAsync(TimeSpan.FromMinutes(5), cancellationToken); + + await this.DeleteStateAsync(telemetryContext, cancellationToken); + await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + await this.BuildAspNetOrchardAsync(telemetryContext, cancellationToken); + this.StartServerInstances(telemetryContext, cancellationToken); + + await this.SaveStateAsync(telemetryContext, cancellationToken); + this.SetServerOnline(true); + + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) + { + await Task.WhenAny(this.serverProcess); + if (cancellationToken.IsCancellationRequested) + { + await Task.WhenAll(this.serverProcess); + } + } + } + catch + { + this.SetServerOnline(false); + await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + throw; + } + }); + } + + private Task DeleteStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + return this.Logger.LogMessageAsync($"{this.TypeName}.DeleteState", relatedContext, async () => + { + using (HttpResponseMessage response = await this.ServerApi.DeleteStateAsync(nameof(State), cancellationToken)) + { + relatedContext.AddResponseContext(response); + if (response.StatusCode != HttpStatusCode.NoContent) + { + response.ThrowOnError(ErrorReason.HttpNonSuccessResponse); + } + } + }); + } + + private Task KillServerInstancesAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + this.Logger.LogTraceMessage($"{this.TypeName}.KillServerInstances"); + this.ExecuteCommandAsync("pkill", "OrchardCore", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); + this.ExecuteCommandAsync("fuser", $"-n tcp -k {this.ServerPort}", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); + + return this.WaitAsync(TimeSpan.FromSeconds(3), cancellationToken); + } + + private void StartServerInstances(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + + this.Logger.LogMessage($"{this.TypeName}.StartServerInstances", relatedContext, () => + { + try + { + string command = "nohup"; + string workingDirectory = this.aspnetOrchardDirectory; + string commandArguments = $"{this.Combine(this.aspnetOrchardPublishPath, "OrchardCore.Cms.Web")} --urls http://*:{this.ServerPort}"; + + relatedContext.AddContext("command", command); + relatedContext.AddContext("commandArguments", commandArguments); + relatedContext.AddContext("workingDir", workingDirectory); + relatedContext.AddContext("bindToCores", this.BindToCores); + relatedContext.AddContext("coreAffinity", this.CoreAffinity); + + this.serverProcess = this.StartServerInstanceAsync(command, commandArguments, workingDirectory, relatedContext, cancellationToken); + } + catch (OperationCanceledException) + { + // Expected whenever certain operations (e.g. Task.Delay) are cancelled. + } + }); + } + + private Task StartServerInstanceAsync(string command, string commandArguments, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken) + { + return (this.ServerRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => + { + try + { + IProcessProxy process = null; + ProcessAffinityConfiguration affinityConfig = null; + + if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) + { + affinityConfig = ProcessAffinityConfiguration.Create( + this.Platform, + this.CoreAffinity); + + telemetryContext.AddContext("affinityMask", affinityConfig.ToString()); + + if (this.Platform == PlatformID.Win32NT) + { + process = this.systemManagement.ProcessManager.CreateProcess( + command, + commandArguments, + workingDirectory); + } + else + { + string fullCommandLine = $"{command} {commandArguments}"; + + LinuxProcessAffinityConfiguration linuxConfig = (LinuxProcessAffinityConfiguration)affinityConfig; + string wrappedCommand = linuxConfig.GetCommandWithAffinity(null, fullCommandLine); + + process = this.systemManagement.ProcessManager.CreateProcess( + "/bin/bash", + $"-c {wrappedCommand}", + workingDirectory); + } + } + else + { + process = await this.ExecuteCommandAsync(command, commandArguments, workingDirectory, telemetryContext, cancellationToken); + } + + using (process) + { + if (affinityConfig != null) + { + if (this.Platform == PlatformID.Win32NT) + { + process.Start(); + process.ApplyAffinity((WindowsProcessAffinityConfiguration)affinityConfig); + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + } + else + { + await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "Orchard"); + process.ThrowIfWorkloadFailed(successCodes: new int[] { 0 }); + } + } + } + catch (OperationCanceledException) + { + // Expected whenever certain operations (e.g. Task.Delay) are cancelled. + } + catch (Exception exc) + { + this.Logger.LogMessage( + $"{this.TypeName}.StartServerInstanceError", + LogLevel.Error, + telemetryContext.Clone().AddError(exc)); + + throw; + } + }); + } + + private Task SaveStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + return this.Logger.LogMessageAsync($"{this.TypeName}.SaveState", relatedContext, async () => + { + Item serverState = new Item(nameof(State), new State()); + serverState.Definition.Online(true); + using (HttpResponseMessage response = await this.ServerApi.UpdateStateAsync(serverState.Id, serverState, cancellationToken)) + { + relatedContext.AddResponseContext(response); + response.ThrowOnError(ErrorReason.HttpNonSuccessResponse); + } + }); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs new file mode 100644 index 0000000000..7ba8cbd6ed --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs @@ -0,0 +1,437 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO.Abstractions; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// AspNet Server Executor. + /// + [SupportedPlatforms("linux-arm64,linux-x64,win-arm64,win-x64")] + public class AspNetServerExecutor : VirtualClientMultiRoleComponent + { + private Task serverProcess; + private bool disposed; + private IFileSystem fileSystem; + private IPackageManager packageManager; + private IStateManager stateManager; + private ISystemManagement systemManagement; + + private string dotnetExePath; + private string aspnetBenchDirectory; + private string aspnetBenchDllPath; + + /// + /// Initializes a new instance of the class. + /// + /// Provides all of the required dependencies to the Virtual Client component. + /// An enumeration of key-value pairs that can control the execution of the component./param> + public AspNetServerExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + this.systemManagement = this.Dependencies.GetService(); + this.packageManager = this.systemManagement.PackageManager; + this.stateManager = this.systemManagement.StateManager; + this.fileSystem = this.systemManagement.FileSystem; + this.ServerRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(3, (retries) => TimeSpan.FromSeconds(retries)); + } + + /// + /// The name of the package where the AspNetBench package is downloaded. + /// + public string TargetFramework + { + get + { + return this.Parameters.GetValue(nameof(AspNetServerExecutor.TargetFramework)).ToLower(); + } + } + + /// + /// The port for ASPNET to run. + /// + public string ServerPort + { + get + { + return this.Parameters.GetValue(nameof(AspNetServerExecutor.ServerPort), "9876"); + } + } + + /// + /// API Client that is used to communicate with server-hosted instance of the Virtual Client Server. + /// + public IApiClient ServerApi { get; set; } + + /// + /// The name of the package where the DotNetSDK package is downloaded. + /// + public string DotNetSdkPackageName + { + get + { + return this.Parameters.GetValue(nameof(AspNetServerExecutor.DotNetSdkPackageName), "dotnetsdk"); + } + } + + /// + /// ASPNETCORE_threadCount + /// + public string AspNetCoreThreadCount + { + get + { + return this.Parameters.GetValue(nameof(AspNetServerExecutor.AspNetCoreThreadCount), "1"); + } + } + + /// + /// DOTNET_SYSTEM_NET_SOCKETS_THREAD_COUNT + /// + public string DotNetSystemNetSocketsThreadCount + { + get + { + return this.Parameters.GetValue(nameof(AspNetServerExecutor.DotNetSystemNetSocketsThreadCount), "1"); + } + } + + /// + /// Gets or sets whether to bind the workload to specific CPU cores. + /// + public bool BindToCores + { + get + { + return this.Parameters.GetValue(nameof(this.BindToCores), defaultValue: false); + } + } + + /// + /// Gets the CPU core affinity specification (e.g., "0-3", "0,2,4,6"). + /// + public string CoreAffinity + { + get + { + this.Parameters.TryGetValue(nameof(this.CoreAffinity), out IConvertible value); + return value?.ToString(); + } + } + + /// + /// A retry policy to apply to the server when starting to handle transient issues that + /// would otherwise prevent it from starting successfully. + /// + protected IAsyncPolicy ServerRetryPolicy { get; set; } + + /// + /// Disposes of resources used by the executor including shutting down any + /// instances of ASP.NET server running. + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + if (!this.disposed) + { + this.KillServerInstancesAsync(null, CancellationToken.None) + .GetAwaiter().GetResult(); + this.disposed = true; + } + } + } + + /// + /// Initializes the environment for execution of the AspNetBench workload. + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, cancellationToken) + .ConfigureAwait(false); + + if (workloadPackage != null) + { + this.aspnetBenchDirectory = this.Combine(workloadPackage.Path, "src", "Benchmarks"); + } + else + { + throw new DependencyException( + $"The expected workload package '{this.PackageName}' does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } + + DependencyPath dotnetSdkPackage = await this.packageManager.GetPackageAsync(this.DotNetSdkPackageName, cancellationToken) + .ConfigureAwait(false); + + if (dotnetSdkPackage == null) + { + throw new DependencyException( + $"The expected DotNet SDK package does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } + + this.dotnetExePath = this.Combine(dotnetSdkPackage.Path, this.Platform == PlatformID.Unix ? "dotnet" : "dotnet.exe"); + + this.InitializeApiClients(); + } + + /// + /// Validates the component parameters. + /// + protected override void Validate() + { + base.Validate(); + + if (this.BindToCores) + { + this.ThrowIfParameterNotDefined(nameof(this.CoreAffinity)); + } + } + + /// + /// Initializes API client. + /// + protected void InitializeApiClients() + { + IApiClientManager clientManager = this.Dependencies.GetService(); + bool isSingleVM = !this.IsMultiRoleLayout(); + + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + } + + /// + /// + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + protected override Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return this.Logger.LogMessageAsync($"{this.TypeName}.ExecuteServer", telemetryContext, async () => + { + try + { + this.SetServerOnline(false); + + await this.ServerApi.PollForHeartbeatAsync(TimeSpan.FromMinutes(5), cancellationToken); + + await this.DeleteStateAsync(telemetryContext, cancellationToken); + await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + await this.BuildAspNetBenchAsync(telemetryContext, cancellationToken); + this.StartServerInstances(telemetryContext, cancellationToken); + + await this.SaveStateAsync(telemetryContext, cancellationToken); + this.SetServerOnline(true); + + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) + { + await Task.WhenAny(this.serverProcess); + + if (cancellationToken.IsCancellationRequested) + { + await Task.WhenAll(this.serverProcess); + } + } + } + catch + { + this.SetServerOnline(false); + await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + throw; + } + }); + } + + /// + /// Builds the ASP.NET Benchmark application + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + protected async Task BuildAspNetBenchAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + string buildArgument = $"build -c Release -p:BenchmarksTargetFramework={this.TargetFramework}"; + await this.ExecuteCommandAsync(this.dotnetExePath, buildArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken) + .ConfigureAwait(false); + + this.aspnetBenchDllPath = this.Combine( + this.aspnetBenchDirectory, + "bin", + "Release", + this.TargetFramework, + "Benchmarks.dll"); + } + + private Task DeleteStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + return this.Logger.LogMessageAsync($"{this.TypeName}.DeleteState", relatedContext, async () => + { + using (HttpResponseMessage response = await this.ServerApi.DeleteStateAsync(nameof(State), cancellationToken)) + { + relatedContext.AddResponseContext(response); + if (response.StatusCode != HttpStatusCode.NoContent) + { + response.ThrowOnError(ErrorReason.HttpNonSuccessResponse); + } + } + }); + } + + private Task KillServerInstancesAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + this.Logger.LogTraceMessage($"{this.TypeName}.KillServerInstances"); + + this.ExecuteCommandAsync("pkill", "dotnet", this.aspnetBenchDirectory, telemetryContext, cancellationToken); + + this.ExecuteCommandAsync("fuser", $"-n tcp -k {this.ServerPort}", this.aspnetBenchDirectory, telemetryContext, cancellationToken); + + return this.WaitAsync(TimeSpan.FromSeconds(3), cancellationToken); + } + + private void StartServerInstances(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + + this.Logger.LogMessage($"{this.TypeName}.StartServerInstances", relatedContext, () => + { + try + { + string options = $"--nonInteractive true --scenarios json --urls http://*:{this.ServerPort} --server Kestrel --kestrelTransport Sockets --protocol http"; + string headers = @"--header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive"""; + string commandArguments = $"{this.aspnetBenchDllPath} {options} {headers}"; + string workingDirectory = this.aspnetBenchDirectory; + + relatedContext.AddContext("command", this.dotnetExePath); + relatedContext.AddContext("commandArguments", commandArguments); + relatedContext.AddContext("workingDir", workingDirectory); + relatedContext.AddContext("bindToCores", this.BindToCores); + relatedContext.AddContext("coreAffinity", this.CoreAffinity); + + this.serverProcess = this.StartServerInstanceAsync(this.dotnetExePath, commandArguments, workingDirectory, relatedContext, cancellationToken); + } + catch (OperationCanceledException) + { + // Expected whenever certain operations (e.g. Task.Delay) are cancelled. + } + }); + } + + private Task StartServerInstanceAsync(string command, string commandArguments, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken) + { + return (this.ServerRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => + { + try + { + IProcessProxy process = null; + ProcessAffinityConfiguration affinityConfig = null; + + if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) + { + affinityConfig = ProcessAffinityConfiguration.Create( + this.Platform, + this.CoreAffinity); + + telemetryContext.AddContext("affinityMask", affinityConfig.ToString()); + + if (this.Platform == PlatformID.Win32NT) + { + process = this.systemManagement.ProcessManager.CreateProcess( + command, + commandArguments, + workingDirectory); + } + else + { + string fullCommandLine = $"{command} {commandArguments}"; + + LinuxProcessAffinityConfiguration linuxConfig = (LinuxProcessAffinityConfiguration)affinityConfig; + string wrappedCommand = linuxConfig.GetCommandWithAffinity(null, fullCommandLine); + + process = this.systemManagement.ProcessManager.CreateProcess( + "/bin/bash", + $"-c {wrappedCommand}", + workingDirectory); + } + } + else + { + process = await this.ExecuteCommandAsync(command, commandArguments, workingDirectory, telemetryContext, cancellationToken); + } + + using (process) + { + if (affinityConfig != null) + { + if (this.Platform == PlatformID.Win32NT) + { + process.Start(); + process.ApplyAffinity((WindowsProcessAffinityConfiguration)affinityConfig); + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + } + else + { + await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "AspNetBenchmarks"); + process.ThrowIfWorkloadFailed(successCodes: new int[] { 0 }); + } + } + } + catch (OperationCanceledException) + { + // Expected whenever certain operations (e.g. Task.Delay) are cancelled. + } + catch (Exception exc) + { + this.Logger.LogMessage( + $"{this.TypeName}.StartServerInstanceError", + LogLevel.Error, + telemetryContext.Clone().AddError(exc)); + + throw; + } + }); + } + + private Task SaveStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + return this.Logger.LogMessageAsync($"{this.TypeName}.SaveState", relatedContext, async () => + { + Item serverState = new Item(nameof(State), new State()); + serverState.Definition.Online(true); + using (HttpResponseMessage response = await this.ServerApi.UpdateStateAsync(serverState.Id, serverState, cancellationToken)) + { + relatedContext.AddResponseContext(response); + response.ThrowOnError(ErrorReason.HttpNonSuccessResponse); + } + }); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs new file mode 100644 index 0000000000..b4be5dcdf3 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs @@ -0,0 +1,529 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.IO.Abstractions; + using System.Linq; + using System.Net.Http; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + using VirtualClient.Contracts.Metadata; + + /// + /// The Bombardier Client executor. + /// + [SupportedPlatforms("linux-arm64,linux-x64,win-x64,win-arm64")] + public class BombardierExecutor : VirtualClientMultiRoleComponent + { + /// + /// Constructor + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public BombardierExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + this.ClientFlowRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(2, (retries) => TimeSpan.FromSeconds(retries * 2)); + + this.ClientRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(2, (retries) => TimeSpan.FromSeconds(retries)); + + this.SystemManagement = this.Dependencies.GetService(); + } + + /// + /// API Client that is used to communicate with server-hosted instance of the Virtual Client Server. + /// + public IApiClient ServerApi { get; set; } + + /// + /// API Client that is used to communicate with ReverseProxy instance of the Virtual Client Server. + /// + public IApiClient ReverseProxyApi { get; set; } + + /// + /// Provides components and services necessary for interacting with the local system and environment. + /// + public ISystemManagement SystemManagement { get; } + + /// + /// Option for testing webserver (default), reverse-proxy (rp), api-gateway (apigw) + /// + public string TargetService + { + get + { + switch (this.Parameters.GetValue(nameof(this.TargetService), string.Empty).ToLower()) + { + case "reverse-proxy": + case "rp": + return "rp"; + case "apiwg": + case "apigw": + case "api-gateway": + return "apigw"; + case "server": + return "server"; + default: + IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); + if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + { + return "server"; + } + else + { + return "rp"; + } + } + } + } + + /// + /// The command line argument defined in the profile. + /// + public string CommandArguments + { + get + { + return this.Parameters.GetValue(nameof(this.CommandArguments)); + } + } + + /// + /// Gets or sets whether to bind the workload to specific CPU cores. + /// + public bool BindToCores + { + get + { + return this.Parameters.GetValue(nameof(this.BindToCores), defaultValue: false); + } + } + + /// + /// Gets the CPU core affinity specification (e.g., "0-3", "0,2,4,6"). + /// + public string CoreAffinity + { + get + { + this.Parameters.TryGetValue(nameof(this.CoreAffinity), out IConvertible value); + return value?.ToString(); + } + } + + /// + /// Polling Timeout + /// + public TimeSpan Timeout + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.Timeout), TimeSpan.FromMinutes(5)); + } + } + + /// + /// Parameter defines true/false whether the action is meant to warm up the server. + /// We do not capture metrics on warm up operations. + /// + public bool WarmUp + { + get + { + return this.Parameters.GetValue(nameof(this.WarmUp), false); + } + } + + /// + /// The path to the Bombardier package. + /// + public string PackageDirectory { get; set; } + + /// + /// True/false whether the server instance has been warmed up. + /// + protected bool IsServerWarmedUp { get; set; } + + /// + /// The retry policy to apply to the client-side execution workflow. + /// + protected IAsyncPolicy ClientFlowRetryPolicy { get; set; } + + /// + /// The retry policy to apply to each Bombardier workload instance when trying to startup + /// against a target server. + /// + protected IAsyncPolicy ClientRetryPolicy { get; set; } + + /// + /// Initializes the executor dependencies, package locations, server api, etc... + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + DependencyPath bombardierPackage = await this.GetPlatformSpecificPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); + + this.PackageDirectory = bombardierPackage.Path; + + // Make bombardier executable on Unix systems + if (this.Platform == PlatformID.Unix) + { + string bombardierPath = this.Combine(this.PackageDirectory, "bombardier"); + await this.SystemManagement.MakeFileExecutableAsync(bombardierPath, this.Platform, cancellationToken).ConfigureAwait(false); + } + + this.InitializeApiClients(); + } + + /// + /// Validates the component parameters. + /// + protected override void Validate() + { + base.Validate(); + + if (this.BindToCores) + { + this.ThrowIfParameterNotDefined(nameof(this.CoreAffinity)); + } + } + + /// + /// Executes component logic + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + if (!this.WarmUp || !this.IsServerWarmedUp) + { + Task clientWorkloadTask; + + clientWorkloadTask = this.ClientFlowRetryPolicy.ExecuteAsync(async () => + { + if (!cancellationToken.IsCancellationRequested) + { + this.Logger.LogTraceMessage("Synchronization: Poll server API for heartbeat..."); + await this.ServerApi.PollForHeartbeatAsync(this.PollingTimeout, cancellationToken); + this.Logger.LogTraceMessage("Synchronization: Poll server for online signal..."); + + await this.ServerApi.PollForExpectedStateAsync(nameof(State), (state => state.Online()), this.Timeout, cancellationToken).ConfigureAwait(false); + + this.Logger.LogTraceMessage("Synchronization: Server online signal confirmed..."); + + // verify ReverseProxy is online + if ((this.ReverseProxyApi != null) && ((this.TargetService == "rp") || (this.TargetService == "apigw"))) + { + this.Logger.LogTraceMessage("Synchronization: Poll ReverseProxy for online signal..."); + await this.ReverseProxyApi.PollForExpectedStateAsync(nameof(State), (state => state.Online()), TimeSpan.FromMinutes(10), cancellationToken).ConfigureAwait(false); + this.Logger.LogTraceMessage("Synchronization: ReverseProxy online signal confirmed..."); + Item reverseProxyState = await this.ReverseProxyApi.GetStateAsync(nameof(State), cancellationToken).ConfigureAwait(false); + telemetryContext.AddContext(nameof(reverseProxyState), reverseProxyState); + + HttpResponseMessage reverseProxyHttpResponseMessage = await this.ReverseProxyApi.GetStateAsync("version", cancellationToken).ConfigureAwait(continueOnCapturedContext: false); + telemetryContext.AddResponseContext(reverseProxyHttpResponseMessage); + + if (reverseProxyHttpResponseMessage.IsSuccessStatusCode) + { + Item reverseProxyVersion = await reverseProxyHttpResponseMessage.FromContentAsync>(); + telemetryContext.AddContext(nameof(reverseProxyVersion), reverseProxyVersion.Definition.Properties); + } + } + + this.Logger.LogTraceMessage("Synchronization: Start client workload..."); + + string commandArguments = this.GetCommandLineArguments(cancellationToken); + await this.ExecuteWorkloadAsync(commandArguments, this.PackageDirectory, telemetryContext, cancellationToken); + } + }); + + await Task.WhenAll(clientWorkloadTask); + + if (this.WarmUp) + { + this.IsServerWarmedUp = true; + } + } + } + + /// + /// Initializes API client. + /// + protected void InitializeApiClients() + { + IApiClientManager clientManager = this.Dependencies.GetService(); + + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ServerApi); + + IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); + if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + { + this.ReverseProxyApi = null; + } + else + { + ClientInstance reverseProxyInstance = reverseProxyInstanceEnumerable.FirstOrDefault(); + this.ReverseProxyApi = clientManager.GetOrCreateApiClient(reverseProxyInstance.Name, reverseProxyInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ReverseProxyApi); + } + } + + /// + /// Gets Command Line Argument to start workload. + /// + protected string GetCommandLineArguments(CancellationToken cancellationToken) + { + string result = this.CommandArguments; + + Dictionary roleAndRegexKvp = new Dictionary() + { + { ClientRole.Server, new Regex(@"\{ServerIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) }, + { ClientRole.ReverseProxy, new Regex(@"\{ReverseProxyIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) }, + { ClientRole.Client, new Regex(@"\{ClientIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) } + }; + + foreach (KeyValuePair kvp in roleAndRegexKvp) + { + MatchCollection matches = kvp.Value.Matches(this.CommandArguments); + + if (matches?.Any() == true) + { + foreach (Match match in matches) + { + ClientInstance roleIP = this.GetLayoutClientInstances(kvp.Key).FirstOrDefault(); + result = Regex.Replace(result, match.Value, roleIP.IPAddress); + } + } + } + + return result; + } + + /// + /// Execute Bombardier Executor + /// + /// Command argument to execute on workload + /// Working Directory + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected async Task ExecuteWorkloadAsync(string commandArguments, string workingDir, EventContext telemetryContext, CancellationToken cancellationToken) + { + commandArguments.ThrowIfNullOrEmpty(nameof(commandArguments)); + + string bombardierPath = this.Combine(this.PackageDirectory, this.Platform == PlatformID.Unix ? "bombardier" : "bombardier.exe"); + + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(bombardierPath); + + EventContext relatedContext = telemetryContext.Clone() + .AddContext(nameof(bombardierPath), bombardierPath) + .AddContext(nameof(commandArguments), commandArguments) + .AddContext("bindToCores", this.BindToCores) + .AddContext("coreAffinity", this.CoreAffinity); + + try + { + await (this.ClientRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => + { + try + { + DateTime startTime = DateTime.UtcNow; + IProcessProxy process = null; + ProcessAffinityConfiguration affinityConfig = null; + + if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) + { + affinityConfig = ProcessAffinityConfiguration.Create( + this.Platform, + this.CoreAffinity); + + relatedContext.AddContext("affinityMask", affinityConfig.ToString()); + + if (this.Platform == PlatformID.Win32NT) + { + process = this.SystemManagement.ProcessManager.CreateProcess( + bombardierPath, + commandArguments, + workingDir); + } + else + { + LinuxProcessAffinityConfiguration linuxConfig = (LinuxProcessAffinityConfiguration)affinityConfig; + string fullCommandLine = $"{bombardierPath} {commandArguments}"; + string wrappedCommand = linuxConfig.GetCommandWithAffinity(null, fullCommandLine); + + process = this.SystemManagement.ProcessManager.CreateProcess( + "/bin/bash", + $"-c {wrappedCommand}", + workingDir); + } + } + else + { + process = await this.ExecuteCommandAsync(bombardierPath, commandArguments, workingDir, telemetryContext, cancellationToken, runElevated: true); + } + + using (process) + { + if (affinityConfig != null) + { + if (this.Platform == PlatformID.Win32NT) + { + process.Start(); + process.ApplyAffinity((WindowsProcessAffinityConfiguration)affinityConfig); + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + } + else + { + await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "Bombardier", logToFile: true); + process.ThrowIfWorkloadFailed(); + + if (process.StandardOutput.Length == 0) + { + throw new WorkloadException($"{this.PackageName} did not write metrics to console.", ErrorReason.CriticalWorkloadFailure); + } + + if (!this.WarmUp) + { + this.CaptureMetrics(process, commandArguments, relatedContext, cancellationToken); + } + } + } + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.WorkloadStartError", LogLevel.Warning, telemetryContext.Clone().AddError(exc)); + throw; + } + }); + } + catch (OperationCanceledException) + { + this.Logger.LogMessage($"{this.TypeName}.OperationCanceledException", LogLevel.Warning, telemetryContext.Clone()); + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.ExecuteWorkloadError", LogLevel.Error, telemetryContext.Clone().AddError(exc)); + throw; + } + } + + /// + /// Get Bombardier Version + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// Bombardier Version + protected string GetBombardierVersion(EventContext telemetryContext, CancellationToken cancellationToken) + { + string bombardierPath = this.Combine(this.PackageDirectory, this.Platform == PlatformID.Unix ? "bombardier" : "bombardier.exe"); + + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(bombardierPath); + + string commandArguments = "--version"; + string versionPattern = @"bombardier\s+version\s+v?(\d+\.\d+\.\d+)"; + Regex versionRegex = new Regex(versionPattern, RegexOptions.Compiled | RegexOptions.IgnoreCase); + string bombardierVersion = null; + + try + { + using (IProcessProxy process = this.ExecuteCommandAsync(bombardierPath, commandArguments, workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).Result) + { + if (!cancellationToken.IsCancellationRequested) + { + this.LogProcessDetailsAsync(process, telemetryContext, "BombardierVersion", logToFile: true).Wait(); + string output = process.StandardOutput.ToString(); + Match match = versionRegex.Match(output); + + if (match.Success) + { + bombardierVersion = match.Groups[1].Value; + telemetryContext.AddContext("BombardierVersion", bombardierVersion); + this.Logger.LogMessage($"{this.TypeName}.BombardierVersionCaptured", LogLevel.Information, telemetryContext); + } + else + { + throw new WorkloadException("Failed to parse bombardier version from output.", ErrorReason.CriticalWorkloadFailure); + } + } + } + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.BombardierVersionCaptureError", LogLevel.Error, telemetryContext.Clone().AddError(exc)); + throw; + } + + return bombardierVersion; + } + + private void CaptureMetrics(IProcessProxy workloadProcess, string commandArguments, EventContext context, CancellationToken cancellationToken) + { + if (!cancellationToken.IsCancellationRequested) + { + if (workloadProcess.ExitCode == 0) + { + EventContext telemetryContext = context.Clone(); + telemetryContext.AddContext(nameof(this.MetricScenario), this.MetricScenario); + telemetryContext.AddContext(nameof(this.Scenario), this.Scenario); + + BombardierMetricsParser resultsParser = new BombardierMetricsParser(workloadProcess.StandardOutput.ToString()); + IList metrics = resultsParser.Parse(); + + foreach (var metric in metrics) + { + metric.Metadata.Add("bindToCores", this.BindToCores.ToString()); + if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) + { + metric.Metadata.Add("coreAffinity", this.CoreAffinity); + } + } + + string bombardierVersion = this.GetBombardierVersion(telemetryContext, cancellationToken); + + this.MetadataContract.AddForScenario( + toolName: this.PackageName, + toolArguments: commandArguments, + toolVersion: bombardierVersion, + packageName: this.PackageName, + packageVersion: null, + additionalMetadata: null); + this.MetadataContract.Apply(telemetryContext); + + this.Logger.LogMetrics( + toolName: this.PackageName, + scenarioName: this.MetricScenario ?? this.Scenario, + workloadProcess.StartTime, + workloadProcess.ExitTime, + metrics: metrics, + metricCategorization: null, + scenarioArguments: commandArguments, + tags: this.Tags, + eventContext: telemetryContext); + } + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierMetricsParser.cs similarity index 100% rename from src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs rename to src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierMetricsParser.cs diff --git a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxCommand.cs b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxCommand.cs new file mode 100644 index 0000000000..c09743d927 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxCommand.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + /// + /// Common Nginx Commands + /// Explore more here: https://nginx.org/en/docs/switches.html + /// + public enum NginxCommand + { + /// + /// "service nginx start" + /// + Start, + + /// + /// "service nginx stop" + /// shut down quickly + /// + Stop, + + /// + /// "nginx -T"; + /// -V: print nginx version, compiler version, and configure parameters. + /// -T: Test configuration file. And dump it. + /// Use standard output to read + /// + GetVersion, + + /// + /// "nginx -T"; + /// -T: Test configuration file. And dump it. + /// Use standard output to read + /// + GetConfig + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs new file mode 100644 index 0000000000..5981ce65f4 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using VirtualClient.Common.Extensions; + using VirtualClient.Contracts; + + /// + /// Extension method for Nginx Workload. + /// + public static class NginxExtensions + { + /// + /// Checks if workload has expired from the state's "Timeout" property + /// + /// + /// + public static bool IsExpired(this State state) + { + return state.Timeout() < DateTime.UtcNow; + } + + /// + /// Gets or sets the 'Timeout' property value from the state. + /// + public static DateTime Timeout(this State state, DateTime? value = null) + { + state.ThrowIfNull("state"); + if (value != null) + { + state.Properties["Timeout"] = value.Value; + } + + return state.Properties.GetValue("Timeout"); + } + + /// + /// Returns nginx commands to start the process. + /// + /// + /// + /// + public static string ConvertToCommandArgs(this NginxCommand command) + { + switch (command) + { + case NginxCommand.Start: + return "systemctl restart nginx"; + + case NginxCommand.Stop: + return "systemctl disable nginx"; + + case NginxCommand.GetVersion: + return "nginx -V"; + + case NginxCommand.GetConfig: + return "nginx -T"; + + default: + throw new WorkloadException($"Unable to convert {nameof(NginxCommand)} enum into string value. Value: {Enum.GetName(command)} - {command}"); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs new file mode 100644 index 0000000000..52173b12c6 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs @@ -0,0 +1,355 @@ +// Copyright (c) Microsoft Corporation. +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.IO.Abstractions; + using System.Linq; + using System.Net; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.CodeAnalysis; + using Microsoft.Extensions.DependencyInjection; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.Platform; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// The NGINX Server Executor + /// + [UnixCompatible] + public class NginxServerExecutor : VirtualClientComponent + { + private TimeSpan pollingInterval = TimeSpan.FromSeconds(120); + + /// + /// Constructor + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public NginxServerExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + this.SystemManagement = this.Dependencies.GetService(); + this.pollingInterval = parameters.GetTimeSpanValue(nameof(this.pollingInterval), TimeSpan.FromSeconds(60)); + } + + /// + /// Number workers used. Number cores nginx can use. Set to null or 0 to use all cores. + /// + public int Workers + { + get + { + return this.Parameters.GetValue(nameof(this.Workers), 0); + } + } + + /// + /// Polling Timeout + /// + public TimeSpan Timeout + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.Timeout), TimeSpan.FromMinutes(30)); + } + } + + /// + /// File size to transport between Client and Server + /// + public int FileSizeInKB + { + get + { + return this.Parameters.GetValue(nameof(this.FileSizeInKB), 1); + } + } + + /// + /// The role of current instance + /// + public string Role + { + get + { + return this.Parameters.GetValue(nameof(this.Role)); + } + } + + /// + /// Provides components and services necessary for interacting with the local system and environment. + /// + protected ISystemManagement SystemManagement { get; } + + /// + /// API Client that is used to communicate with self-hosted instance of the Virtual Client. + /// + protected IApiClient ServerApi { get; set; } + + /// + /// API Client that is used to communicate with client-hosted instance of the Virtual Client Client. + /// + protected IApiClient ClientApi { get; set; } + + /// + /// The path to the Nginx package. + /// + protected string PackageDirectory { get; set; } + + /// + /// Initializes the API dependencies for running Nginx Server + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + this.Logger.LogTraceMessage($"Role: {this.Roles}; Layout: {this.GetLayoutClientInstances}"); + + PlatformSpecifics.ThrowIfNotSupported(this.CpuArchitecture); + if (this.Platform != PlatformID.Unix) + { + this.Logger.LogNotSupported(this.PackageName, this.Platform, this.CpuArchitecture, EventContext.Persisted()); + throw new NotSupportedException($"The OS/system platform '{this.Platform}' is not supported."); + } + + IApiClientManager clientManager = this.Dependencies.GetService(); + + ClientInstance serverInstance = this.GetLayoutClientInstances(this.Role).First(); + IPAddress.TryParse(serverInstance.IPAddress, out IPAddress serverIPAddress); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + + ClientInstance clientInstance = this.GetLayoutClientInstances(ClientRole.Client).First(); + IPAddress.TryParse(clientInstance.IPAddress, out IPAddress clientIPAddress); + this.ClientApi = clientManager.GetOrCreateApiClient(clientInstance.Name, clientInstance); + + DependencyPath workloadPackage = await this.SystemManagement.PackageManager.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); + if (workloadPackage == null) + { + throw new DependencyException($"{this.TypeName} did not find package ({this.PackageName}) in the packages directory.", ErrorReason.WorkloadDependencyMissing); + } + + workloadPackage = this.PlatformSpecifics.ToPlatformSpecificPath(workloadPackage, this.Platform, this.CpuArchitecture); + this.PackageDirectory = workloadPackage.Path; + + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.PackageDirectory, "setup-reset.sh")); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.PackageDirectory, "setup-config.sh")); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.PackageDirectory, "setup-content.sh")); + + string resetFilePath = this.PlatformSpecifics.Combine(this.PackageDirectory, "reset.sh"); + + if (!this.SystemManagement.FileSystem.File.Exists(resetFilePath)) + { + IProcessProxy process1 = await this.ExecuteCommandAsync(command: "bash", commandArguments: "setup-reset.sh", workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true); + await this.LogProcessDetailsAsync(process1, telemetryContext, this.PackageDirectory, logToFile: true); + process1.ThrowIfWorkloadFailed(); + telemetryContext.AddContext("resetContent", process1.StandardOutput); + + using (FileSystemStream fileStream = this.SystemManagement.FileSystem.FileStream.New(resetFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite)) + { + byte[] bytedata = Encoding.Default.GetBytes(process1.StandardOutput.ToString()); + fileStream.Write(bytedata, 0, bytedata.Length); + await fileStream.FlushAsync().ConfigureAwait(false); + fileStream.Close(); + fileStream.Dispose(); + this.Logger.LogTraceMessage($"File Created...{resetFilePath}"); + } + } + + IProcessProxy process2 = await this.ExecuteCommandAsync(command: "bash", commandArguments: $"setup-content.sh {this.FileSizeInKB}", workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process2, telemetryContext, this.PackageDirectory, logToFile: true); + process2.ThrowIfWorkloadFailed(); + + ClientInstance backendInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + IPAddress.TryParse(backendInstance.IPAddress, out IPAddress backendIPAddress); + IProcessProxy process3 = await this.ExecuteCommandAsync(command: "bash", commandArguments: $"setup-config.sh {((this.Workers != 0) ? this.Workers : "auto")} {this.Role} {backendIPAddress}", workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process3, telemetryContext, this.PackageDirectory, logToFile: true); + process3.ThrowIfWorkloadFailed(); + + await this.ServerApi.DeleteStateAsync("version", cancellationToken).ConfigureAwait(false); + await this.ServerApi.DeleteStateAsync(nameof(State), cancellationToken).ConfigureAwait(false); + this.SetServerOnline(true); + this.Logger.LogTraceMessage($"{this.TypeName} Initialize Complete."); + } + + /// + /// Executes component logic + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + telemetryContext + .AddContext("currentDirectory", Environment.CurrentDirectory) + .AddContext("toolName", "nginx") + .AddContext("timeout", this.Timeout); + + if (!cancellationToken.IsCancellationRequested) + { + try + { + Dictionary nginxVersion = await this.GetNginxVersionAsync(telemetryContext, cancellationToken).ConfigureAwait(false); + State nginxState = new State(); + nginxVersion.Any(x => nginxState.Properties.TryAdd(x.Key, x.Value)); + await this.ServerApi.CreateStateAsync("version", nginxState, cancellationToken).ConfigureAwait(false); + telemetryContext.AddContext(nameof(nginxVersion), nginxVersion); + + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) + { + await this.ExecuteNginxCommandAsync(NginxCommand.Start, workingDirectory: null, telemetryContext, cancellationToken).ConfigureAwait(false); + + Item serverState = new Item(nameof(State), new State()); + serverState.Definition.Timeout(DateTime.UtcNow.Add(this.Timeout)); + serverState.Definition.Online(true); + await this.ServerApi.CreateStateAsync(nameof(State), serverState.Definition, cancellationToken).ConfigureAwait(false); + + while (!(serverState.Definition.Timeout() < DateTime.UtcNow)) + { + EventContext relatedContext = telemetryContext + .Clone() + .AddContext(nameof(serverState), serverState); + + await this.ClientApi.PollForExpectedStateAsync(nameof(State), (state => state.Online() == true), this.Timeout, cancellationToken, this.pollingInterval).ConfigureAwait(false); + Item clientState = await this.ClientApi.GetStateAsync(nameof(State), cancellationToken).ConfigureAwait(false); + relatedContext.AddContext(nameof(clientState), clientState); + + await Task.Delay(this.pollingInterval, cancellationToken); + DateTime timeout = clientState.Definition.Properties.GetValue("Timeout", serverState.Definition.Timeout()); + + serverState.Definition.Timeout(timeout); + await this.ServerApi.UpdateStateAsync(nameof(State), serverState, cancellationToken).ConfigureAwait(false); + relatedContext.AddContext(nameof(serverState), serverState); + } + } + } + catch (Exception exc) + { + this.Logger.LogErrorMessage(exc, telemetryContext); + throw; + } + finally + { + await this.ResetNginxAsync(telemetryContext, cancellationToken).ConfigureAwait(false); + } + } + } + + /// + /// Disposes of resources used by the executor including resetting server. + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + Task.Run((Func)(async () => + { + await this.ResetNginxAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + })).Wait(); + } + } + + /// + /// Reset Server for Nginx + /// + /// Provides context information to include with telemetry events. + /// A token that can be used to cancel the operation. + protected async Task ResetNginxAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + try + { + IProcessProxy process1 = await this.ExecuteCommandAsync(command: "bash", commandArguments: "reset.sh", workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process1, telemetryContext, this.PackageName, logToFile: true); + + await this.ExecuteNginxCommandAsync(NginxCommand.Stop, workingDirectory: null, telemetryContext, cancellationToken).ConfigureAwait(false); + + State serverState = new State(); + serverState.Online(false); + serverState.Properties["ResetTime"] = DateTime.UtcNow.ToString(); + await this.ServerApi.UpdateStateAsync(nameof(State), new Item(nameof(State), serverState), cancellationToken).ConfigureAwait(false); + } + catch + { + this.Logger.LogTraceMessage("Failed to reset server."); + } + } + + /// + /// Executes a command to get Nginx Version installed from the system. + /// + /// Provides context information to include with telemetry events. + /// A token that can be used to cancel the process execution. + /// NginxVersion + protected async Task> GetNginxVersionAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + IProcessProxy process = await this.ExecuteNginxCommandAsync(NginxCommand.GetVersion, null, telemetryContext, cancellationToken).ConfigureAwait(false); + string standardErr = process.StandardError.ToString(); + + return this.TransformNginxVersionToDictionary(standardErr); + } + + private Dictionary TransformNginxVersionToDictionary(string processOutput) + { + processOutput.ThrowIfNullOrEmpty(nameof(processOutput)); + string[] standardOutputSections = processOutput.Split(Environment.NewLine, StringSplitOptions.TrimEntries); + + string nginxVersion = + standardOutputSections + .Where(x => x.Contains("nginx version", StringComparison.OrdinalIgnoreCase)).FirstOrDefault() + .Split(":").Last().Trim(); + + nginxVersion.ThrowIfNullOrEmpty(nameof(nginxVersion), $"{nameof(processOutput)} does not contain nginx version. {nameof(processOutput)}: {processOutput}"); + + string sslVersion = + standardOutputSections + .Where(x => x.Contains("OpenSSL", StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + + string serverNameIndicationSupport = + standardOutputSections + .Where(x => x.Contains("TLS", StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + + string arguments = + standardOutputSections + .Where(x => x.Contains("configure arguments", StringComparison.OrdinalIgnoreCase)).FirstOrDefault() + ?.Split(":").Last().Trim(); + + nginxVersion.ThrowIfNullOrEmpty(nameof(nginxVersion), $"{nameof(processOutput)} does not contain nginx version. {nameof(processOutput)}: {processOutput}"); + return new Dictionary + { + { nameof(nginxVersion), nginxVersion }, + { nameof(sslVersion), sslVersion }, + { nameof(serverNameIndicationSupport), serverNameIndicationSupport }, + { nameof(arguments), arguments } + }; + } + + private Task ExecuteNginxCommandAsync(NginxCommand nginxCommand, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken) + { + nginxCommand.ThrowIfNull("nginxCommand"); + telemetryContext.ThrowIfNull("telemetryContext"); + + telemetryContext.AddContext(nameof(nginxCommand), nginxCommand); + string commandArgs = nginxCommand.ConvertToCommandArgs(); + telemetryContext.AddContext(nameof(commandArgs), $"{commandArgs}"); + + if (this.Platform != PlatformID.Unix) + { + throw new NotSupportedException($"Nginx command is not supported on '{this.Platform}' platform/architecture systems."); + } + + return this.Logger.LogMessageAsync($"{nameof(this.TypeName)}.ExecuteNginxCommand", telemetryContext, async () => + { + IProcessProxy process = await this.ExecuteCommandAsync(command: "sudo", commandArguments: commandArgs, workingDirectory: workingDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process, telemetryContext, logToFile: true); + process.ThrowIfWorkloadFailed(); + return process; + }); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/VirtualClient.Actions.csproj b/src/VirtualClient/VirtualClient.Actions/VirtualClient.Actions.csproj index 038749f5b7..31975ac868 100644 --- a/src/VirtualClient/VirtualClient.Actions/VirtualClient.Actions.csproj +++ b/src/VirtualClient/VirtualClient.Actions/VirtualClient.Actions.csproj @@ -29,6 +29,7 @@ + diff --git a/src/VirtualClient/VirtualClient.Actions/Wrk/Wrk2Executor.cs b/src/VirtualClient/VirtualClient.Actions/Wrk/Wrk2Executor.cs new file mode 100644 index 0000000000..38769028e7 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Wrk/Wrk2Executor.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// The Wrk Client executor. + /// + [SupportedPlatforms("linux-arm64,linux-x64")] + public class Wrk2Executor : WrkExecutor + { + /// + /// Constructor + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public Wrk2Executor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + } + + /// + /// Initializes the executor dependencies, package locations, server api, etc... + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + DependencyPath workloadPackage = await this.SystemManagement.PackageManager.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); + DependencyPath scriptPackage = await this.SystemManagement.PackageManager.GetPackageAsync(Wrk2Executor.WrkConfiguration, cancellationToken).ConfigureAwait(false); + + if (workloadPackage == null || this.PackageName != "wrk2") + { + throw new DependencyException($"{this.TypeName} did not find correct package in the directory. Supported Package: wrk2. Package Provided: {this.PackageName}", ErrorReason.WorkloadDependencyMissing); + } + + if (scriptPackage == null) + { + throw new DependencyException($"{this.TypeName} did not find package ({WrkExecutor.WrkConfiguration}) in the packages directory.", ErrorReason.WorkloadDependencyMissing); + } + + this.PackageDirectory = workloadPackage.Path; + this.ScriptDirectory = this.PlatformSpecifics.ToPlatformSpecificPath(scriptPackage, this.Platform, this.CpuArchitecture).Path; + + this.InitializeApiClients(); + await this.SetupWrkClient(telemetryContext, cancellationToken).ConfigureAwait(false); + } + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs new file mode 100644 index 0000000000..dcc894fd34 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs @@ -0,0 +1,578 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.IO.Abstractions; + using System.Linq; + using System.Net.Http; + using System.Text; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.CodeAnalysis; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + using VirtualClient.Contracts.Metadata; + + /// + /// The Wrk Client executor. + /// + [SupportedPlatforms("linux-arm64,linux-x64")] + public class WrkExecutor : VirtualClientMultiRoleComponent + { + internal const string WrkConfiguration = "wrkconfiguration"; + internal const string WrkRunShell = "runwrk.sh"; + + /// + /// Constructor + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public WrkExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + this.ClientFlowRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(2, (retries) => TimeSpan.FromSeconds(retries * 2)); + + this.ClientRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(2, (retries) => TimeSpan.FromSeconds(retries)); + + this.SystemManagement = this.Dependencies.GetService(); + } + + /// + /// API Client that is used to communicate with server-hosted instance of the Virtual Client Server. + /// + public IApiClient ServerApi { get; set; } + + /// + /// API Client that is used to communicate with ReverseProxy instance of the Virtual Client Server. + /// + public IApiClient ReverseProxyApi { get; set; } + + /// + /// Provides components and services necessary for interacting with the local system and environment. + /// + public ISystemManagement SystemManagement { get; } + + /// + /// Option for testing webserver (default), reverse-proxy (rp), api-gateway (apigw) + /// + public string TargetService + { + get + { + switch (this.Parameters.GetValue(nameof(this.TargetService), string.Empty).ToLower()) + { + case "reverse-proxy": + case "rp": + return "rp"; + case "apiwg": + case "apigw": + case "api-gateway": + return "apigw"; + case "server": + return "server"; + default: + IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); + if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + { + return "server"; + } + else + { + return "rp"; + } + } + } + } + + /// + /// The command line argument defined in the profile. + /// + public string CommandArguments + { + get + { + return this.Parameters.GetValue(nameof(this.CommandArguments)); + } + } + + /// + /// Gets or sets whether to bind the workload to specific CPU cores. + /// + public bool BindToCores + { + get + { + return this.Parameters.GetValue(nameof(this.BindToCores), defaultValue: false); + } + } + + /// + /// Gets the CPU core affinity specification (e.g., "0-3", "0,2,4,6"). + /// + public string CoreAffinity + { + get + { + this.Parameters.TryGetValue(nameof(this.CoreAffinity), out IConvertible value); + return value?.ToString(); + } + } + + /// + /// Emit Latency Spectrum as additional metrics + /// + public bool EmitLatencySpectrum + { + get + { + return this.Parameters.GetValue(nameof(this.EmitLatencySpectrum), false); + } + } + + /// + /// Polling Timeout + /// + public TimeSpan Timeout + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.Timeout), TimeSpan.FromMinutes(5)); + } + } + + /// + /// Parameter defines true/false whether the action is meant to warm up the server. + /// We do not capture metrics on warm up operations. + /// + public bool WarmUp + { + get + { + return this.Parameters.GetValue(nameof(this.WarmUp), false); + } + } + + /// + /// The path to the Wrk package. + /// + public string PackageDirectory { get; set; } + + /// + /// The path to wrk scripts. + /// + public string ScriptDirectory { get; set; } + + /// + /// True/false whether the server instance has been warmed up. + /// + protected bool IsServerWarmedUp { get; set; } + + /// + /// The retry policy to apply to the client-side execution workflow. + /// + protected IAsyncPolicy ClientFlowRetryPolicy { get; set; } + + /// + /// The retry policy to apply to each Memtier workload instance when trying to startup + /// against a target server. + /// + protected IAsyncPolicy ClientRetryPolicy { get; set; } + + /// + /// Initializes the executor dependencies, package locations, server api, etc... + /// + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + DependencyPath workloadPackage = await this.SystemManagement.PackageManager.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); + DependencyPath scriptPackage = await this.SystemManagement.PackageManager.GetPackageAsync(WrkExecutor.WrkConfiguration, cancellationToken).ConfigureAwait(false); + + if (this.PackageName != "wrk" || workloadPackage == null) + { + throw new DependencyException($"{this.TypeName} did not find correct package in the directory. Supported Package: wrk. Package Provided: {this.PackageName}", ErrorReason.WorkloadDependencyMissing); + } + + if (scriptPackage == null) + { + throw new DependencyException($"{this.TypeName} did not find package ({WrkExecutor.WrkConfiguration}) in the packages directory.", ErrorReason.WorkloadDependencyMissing); + } + + this.PackageDirectory = workloadPackage.Path; + this.ScriptDirectory = this.PlatformSpecifics.ToPlatformSpecificPath(scriptPackage, this.Platform, this.CpuArchitecture).Path; + + this.InitializeApiClients(); + await this.SetupWrkClient(telemetryContext, cancellationToken).ConfigureAwait(false); + } + + /// + /// Validates the component parameters. + /// + protected override void Validate() + { + base.Validate(); + + if (this.BindToCores) + { + this.ThrowIfParameterNotDefined(nameof(this.CoreAffinity)); + } + } + + /// + /// Executes component logic + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + if (!this.WarmUp || !this.IsServerWarmedUp) + { + Task clientWorkloadTask; + + clientWorkloadTask = this.ClientFlowRetryPolicy.ExecuteAsync(async () => + { + if (!cancellationToken.IsCancellationRequested) + { + this.Logger.LogTraceMessage("Synchronization: Poll server API for heartbeat..."); + await this.ServerApi.PollForHeartbeatAsync(this.PollingTimeout, cancellationToken); + this.Logger.LogTraceMessage("Synchronization: Poll server for online signal..."); + + await this.ServerApi.PollForExpectedStateAsync(nameof(State), (state => state.Online()), this.Timeout, cancellationToken).ConfigureAwait(false); + + this.Logger.LogTraceMessage("Synchronization: Server online signal confirmed..."); + + // verify ReverseProxy is online + if ((this.ReverseProxyApi != null) && ((this.TargetService == "rp") || (this.TargetService == "apigw"))) + { + this.Logger.LogTraceMessage("Synchronization: Poll ReverseProxy for online signal..."); + await this.ReverseProxyApi.PollForExpectedStateAsync(nameof(State), (state => state.Online()), TimeSpan.FromMinutes(10), cancellationToken).ConfigureAwait(false); + this.Logger.LogTraceMessage("Synchronization: ReverseProxy online signal confirmed..."); + Item reverseProxyState = await this.ReverseProxyApi.GetStateAsync(nameof(State), cancellationToken).ConfigureAwait(false); + telemetryContext.AddContext(nameof(reverseProxyState), reverseProxyState); + + HttpResponseMessage reverseProxyHttpResponseMessage = await this.ReverseProxyApi.GetStateAsync("version", cancellationToken).ConfigureAwait(continueOnCapturedContext: false); + telemetryContext.AddResponseContext(reverseProxyHttpResponseMessage); + + if (reverseProxyHttpResponseMessage.IsSuccessStatusCode) + { + Item reverseProxyVersion = await reverseProxyHttpResponseMessage.FromContentAsync>(); + telemetryContext.AddContext(nameof(reverseProxyVersion), reverseProxyVersion.Definition.Properties); + } + } + + this.Logger.LogTraceMessage("Synchronization: Start client workload..."); + + string commandArguments = this.GetCommandLineArguments(cancellationToken); + await this.ExecuteWorkloadAsync(commandArguments, this.PackageDirectory, telemetryContext, cancellationToken); + } + }); + + await Task.WhenAll(clientWorkloadTask); + + if (this.WarmUp) + { + this.IsServerWarmedUp = true; + } + } + } + + /// + /// Setup Wrk Executor + /// + protected async Task SetupWrkClient(EventContext telemetryContext, CancellationToken cancellationToken) + { + this.PackageDirectory.ThrowIfNullOrWhiteSpace(nameof(this.PackageDirectory)); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.PackageDirectory, "wrk")); + await this.SystemManagement.MakeFileExecutableAsync(this.PlatformSpecifics.Combine(this.PackageDirectory, "wrk"), this.Platform, cancellationToken).ConfigureAwait(false); + + string shellScriptPath = this.PlatformSpecifics.GetScriptPath("wrk", WrkExecutor.WrkRunShell); + this.SystemManagement.FileSystem.File.Copy(shellScriptPath, this.Combine(this.PackageDirectory, WrkExecutor.WrkRunShell), true); + + this.ScriptDirectory.ThrowIfNullOrWhiteSpace(nameof(this.ScriptDirectory)); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.ScriptDirectory, "setup-reset.sh")); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.ScriptDirectory, "setup-config.sh")); + + string resetFilePath = this.PlatformSpecifics.Combine(this.ScriptDirectory, "reset.sh"); + if (!this.SystemManagement.FileSystem.File.Exists(resetFilePath)) + { + IProcessProxy process1 = await this.ExecuteCommandAsync(command: "bash", commandArguments: "setup-reset.sh", workingDirectory: this.ScriptDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process1, telemetryContext, WrkExecutor.WrkConfiguration, logToFile: true); + process1.ThrowIfWorkloadFailed(); + + using (FileSystemStream fileStream = this.SystemManagement.FileSystem.FileStream.New(resetFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite)) + { + byte[] bytedata = Encoding.Default.GetBytes(process1.StandardOutput.ToString()); + fileStream.Write(bytedata, 0, bytedata.Length); + await fileStream.FlushAsync().ConfigureAwait(false); + this.Logger.LogTraceMessage($"File Created...{resetFilePath}"); + } + } + + // set up Config + IProcessProxy process2 = await this.ExecuteCommandAsync(command: "bash", commandArguments: "setup-config.sh", workingDirectory: this.ScriptDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process2, telemetryContext, WrkExecutor.WrkConfiguration, logToFile: true); + process2.ThrowIfWorkloadFailed(); + } + + /// + /// Initializes API client. + /// + protected void InitializeApiClients() + { + IApiClientManager clientManager = this.Dependencies.GetService(); + + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ServerApi); + + IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); + if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + { + this.ReverseProxyApi = null; + } + else + { + ClientInstance reverseProxyInstance = reverseProxyInstanceEnumerable.FirstOrDefault(); + this.ReverseProxyApi = clientManager.GetOrCreateApiClient(reverseProxyInstance.Name, reverseProxyInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ReverseProxyApi); + } + } + + /// + /// Gets Command Line Argument to start workload. + /// + protected string GetCommandLineArguments(CancellationToken cancellationToken) + { + string result = this.CommandArguments; + + Dictionary roleAndRegexKvp = new Dictionary() + { + { ClientRole.Server, new Regex(@"\{ServerIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) }, + { ClientRole.ReverseProxy, new Regex(@"\{ReverseProxyIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) }, + { ClientRole.Client, new Regex(@"\{ClientIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) } + }; + + foreach (KeyValuePair kvp in roleAndRegexKvp) + { + MatchCollection matches = kvp.Value.Matches(this.CommandArguments); + + if (matches?.Any() == true) + { + foreach (Match match in matches) + { + ClientInstance roleIP = this.GetLayoutClientInstances(kvp.Key).FirstOrDefault(); + result = Regex.Replace(result, match.Value, roleIP.IPAddress); + } + } + } + + return result; + } + + /// + /// Execute Wrk Executor + /// + /// Command argument ti execute on workload + /// Working Directory + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected async Task ExecuteWorkloadAsync(string commandArguments, string workingDir, EventContext telemetryContext, CancellationToken cancellationToken) + { + commandArguments.ThrowIfNullOrEmpty(nameof(commandArguments)); + string scriptPath = this.Combine(this.PackageDirectory, WrkExecutor.WrkRunShell); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(scriptPath); + string command = $"bash {scriptPath}"; + + EventContext relatedContext = telemetryContext.Clone() + .AddContext(nameof(command), command) + .AddContext(nameof(commandArguments), commandArguments) + .AddContext("bindToCores", this.BindToCores) + .AddContext("coreAffinity", this.CoreAffinity); + + try + { + await (this.ClientRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => + { + try + { + DateTime startTime = DateTime.UtcNow; + IProcessProxy process = null; + ProcessAffinityConfiguration affinityConfig = null; + + if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) + { + affinityConfig = ProcessAffinityConfiguration.Create( + this.Platform, + this.CoreAffinity); + + relatedContext.AddContext("affinityMask", affinityConfig.ToString()); + + string fullCommandLine = $"{command} \"{commandArguments}\""; + LinuxProcessAffinityConfiguration linuxConfig = (LinuxProcessAffinityConfiguration)affinityConfig; + string wrappedCommand = linuxConfig.GetCommandWithAffinity(null, fullCommandLine); + + process = this.SystemManagement.ProcessManager.CreateProcess( + "/bin/bash", + $"-c {wrappedCommand}", + workingDir); + + process.RedirectStandardInput = true; + } + else + { + process = await this.ExecuteCommandAsync(command, commandArguments: $"\"{commandArguments}\"", workingDir, telemetryContext, cancellationToken, runElevated: true); + } + + using (process) + { + if (affinityConfig != null) + { + await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); + } + + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "Wrk", logToFile: true); + process.ThrowIfWorkloadFailed(); + + if (process.StandardOutput.Length == 0) + { + throw new WorkloadException($"{this.PackageName} did not write metrics to console.", ErrorReason.CriticalWorkloadFailure); + } + + if (!this.WarmUp) + { + this.CaptureMetrics(process, commandArguments, this.EmitLatencySpectrum, relatedContext, cancellationToken); + } + } + } + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.WorkloadStartError", LogLevel.Warning, telemetryContext.Clone().AddError(exc)); + throw; + } + }); + } + catch (OperationCanceledException) + { + this.Logger.LogMessage($"{this.TypeName}.OperationCanceledException", LogLevel.Warning, telemetryContext.Clone()); + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.ExecuteWorkloadError", LogLevel.Error, telemetryContext.Clone().AddError(exc)); + throw; + } + } + + /// + /// Get Wrk Version + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// Wrk Version + protected string GetWrkVersion(EventContext telemetryContext, CancellationToken cancellationToken) + { + string scriptPath = this.Combine(this.PackageDirectory, WrkExecutor.WrkRunShell); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(scriptPath); + + string command = $"bash {scriptPath}"; + string commandArguments = "--version"; + string versionPattern = @"wrk\s(\d+\.\d+\.\d+)"; + Regex versionRegex = new Regex(versionPattern, RegexOptions.Compiled); + string wrkVersion = null; + + try + { + using (IProcessProxy process = this.ExecuteCommandAsync(command, commandArguments, workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).Result) + { + if (!cancellationToken.IsCancellationRequested) + { + this.LogProcessDetailsAsync(process, telemetryContext, "WrkVersion", logToFile: true).Wait(); + string output = process.StandardOutput.ToString(); + Match match = versionRegex.Match(output); + + if (match.Success) + { + wrkVersion = match.Groups[1].Value; + telemetryContext.AddContext("WrkVersion", wrkVersion); + this.Logger.LogMessage($"{this.TypeName}.WrkVersionCaptured", LogLevel.Information, telemetryContext); + } + else + { + throw new WorkloadException("Failed to parse wrk version from output.", ErrorReason.CriticalWorkloadFailure); + } + } + } + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.WrkVersionCaptureError", LogLevel.Error, telemetryContext.Clone().AddError(exc)); + throw; + } + + return wrkVersion; + } + + private void CaptureMetrics(IProcessProxy workloadProcess, string commandArguments, bool emitLatencySpectrum, EventContext context, CancellationToken cancellationToken) + { + if (!cancellationToken.IsCancellationRequested) + { + if (workloadProcess.ExitCode == 0) + { + EventContext telemetryContext = context.Clone(); + telemetryContext.AddContext(nameof(this.MetricScenario), this.MetricScenario); + telemetryContext.AddContext(nameof(this.Scenario), this.Scenario); + + WrkMetricParser resultsParser = new WrkMetricParser(workloadProcess.StandardOutput.ToString()); + IList metrics = resultsParser.Parse(emitLatencySpectrum); + + foreach (var metric in metrics) + { + metric.Metadata.Add("bindToCores", this.BindToCores.ToString()); + if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) + { + metric.Metadata.Add("coreAffinity", this.CoreAffinity); + } + } + + string wrkVersion = this.GetWrkVersion(telemetryContext, cancellationToken); + + this.MetadataContract.AddForScenario( + toolName: this.PackageName, + toolArguments: commandArguments, + toolVersion: wrkVersion, + packageName: this.PackageName, + packageVersion: null, + additionalMetadata: null); + telemetryContext.AddContext("TestConfig", resultsParser.GetTestConfig()); + this.MetadataContract.Apply(telemetryContext); + + this.Logger.LogMetrics( + toolName: this.PackageName, + scenarioName: this.MetricScenario ?? this.Scenario, + workloadProcess.StartTime, + workloadProcess.ExitTime, + metrics: metrics, + metricCategorization: null, + scenarioArguments: commandArguments, + tags: this.Tags, + eventContext: telemetryContext); + } + } + } + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/WrkMetricParser.cs b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkMetricParser.cs similarity index 100% rename from src/VirtualClient/VirtualClient.Actions/ASPNET/WrkMetricParser.cs rename to src/VirtualClient/VirtualClient.Actions/Wrk/WrkMetricParser.cs diff --git a/src/VirtualClient/VirtualClient.Actions/Wrk/runwrk.sh b/src/VirtualClient/VirtualClient.Actions/Wrk/runwrk.sh new file mode 100644 index 0000000000..d7c59a5c11 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Wrk/runwrk.sh @@ -0,0 +1,2 @@ +ulimit -n 65535 +./wrk $1 \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-ORCHARD-WRK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-ORCHARD-WRK.json new file mode 100644 index 0000000000..84d24b06dc --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-ORCHARD-WRK.json @@ -0,0 +1,161 @@ +{ + "Description": ".NET benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedOperatingSystems": "CBL-Mariner,Ubuntu" + }, + "Parameters": { + "DotNetVersion": "9.0.203", + "TargetFramework": "net9.0", + "OrchardCoreAdminValue": "Compete@CRC1", + "ServerPort": 5014, + "TestDuration": "00:00:20", + "EmitLatencySpectrum": false, + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s http://{serverip}:{ServerPort}/about --header \"Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\" --header \"Connection:keep-alive\"" + }, + "Actions": [ + { + "Type": "AspNetOrchardServerExecutor", + "Parameters": { + "Role": "Server", + "Scenario": "OrchardCore", + "PackageName": "orchardcore", + "DotNetSdkPackageName": "dotnetsdk", + "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "$.Parameters.ServerPort" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Warmup", + "MetricScenario": "Warmup_Orchard-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "ServerPort": "$.Parameters.ServerPort", + "ThreadCount": "64", + "Connection": 128, + "TestDuration": "00:02:00", + "WarmUp": "true", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Tags": "Orchard,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Benchmark_Measurement", + "MetricScenario": "Orchard-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "ServerPort": "$.Parameters.ServerPort", + "ThreadCount": "64", + "Connection": 128, + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Tags": "Orchard,WRK" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "build-essential,libevent-dev,pkg-config,zlib1g-dev,libssl-dev,autoconf,automake,make,libpcre3-dev,gcc,unzip,openssl", + "Packages-Yum": "zlib-devel,pcre-devel,libevent-devel,openssl-devel,git,gcc-c++,make,autoconf,automake", + "Packages-Dnf": "zlib-devel,libevent-devel,openssl-devel,git,gcc,gcc-c++,make,autoconf,icu,automake,unzip,binutils,libstdc++-devel,kernel-headers,glibc-devel,perl,perl-Module-CoreList" + } + }, + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "wget,build-essential,tcl-dev,numactl", + "Packages-Yum": "wget,numactl,tcl-devel", + "Packages-Dnf": "wget,numactl,tcl-devel,iptables" + } + }, + { + "Type": "ChocolateyInstallation", + "Parameters": { + "Scenario": "InstallChocolatey", + "PackageName": "chocolatey" + } + }, + { + "Type": "ChocolateyPackageInstallation", + "Parameters": { + "Scenario": "InstallGit", + "PackageName": "chocolatey", + "Packages": "git" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneOrchardCoreRepo", + "RepoUri": "https://github.com/orchardcms/orchardcore.git", + "Commit": "4e7c47c", + "PackageName": "orchardcore", + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "DotNetInstallation", + "Parameters": { + "Scenario": "InstallDotNetSdk", + "DotNetVersion": "$.Parameters.DotNetVersion", + "PackageName": "dotnetsdk", + "Role": "Server" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + }, + { + "Type": "SetEnvironmentVariable", + "Parameters": { + "Scenario": "SetEnvironmentVariableForAspNet", + "EnvironmentVariables": "OrchardCore__OrchardCore_AutoSetup__Tenants__0__ShellName=Default;OrchardCore__OrchardCore_AutoSetup__Tenants__0__SiteName=Benchmark;OrchardCore__OrchardCore_AutoSetup__Tenants__0__SiteTimeZone=Europe/Amsterdam;OrchardCore__OrchardCore_AutoSetup__Tenants__0__AdminUsername=admin;OrchardCore__OrchardCore_AutoSetup__Tenants__0__AdminEmail=info@orchardproject.net;OrchardCore__OrchardCore_AutoSetup__Tenants__0__AdminPassword={OrchardCoreAdminValue};OrchardCore__OrchardCore_AutoSetup__Tenants__0__DatabaseProvider=Sqlite;OrchardCore__OrchardCore_AutoSetup__Tenants__0__RecipeName=Blog;DOTNET_GCDynamicAdaptationMode=0;DOTNET_HillClimbing_Disable=1", + "OrchardCoreAdminValue": "$.Parameters.OrchardCoreAdminValue", + "Role": "Server" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-TEJSON-WRK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-TEJSON-WRK.json new file mode 100644 index 0000000000..ea2bad74f1 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-TEJSON-WRK.json @@ -0,0 +1,150 @@ +{ + "Description": ".NET benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" + }, + "Parameters": { + "DotNetVersion": "9.0.101", + "TargetFramework": "net9.0", + "ServerPort": 9876, + "TestDuration": "00:00:15", + "Timeout": "00:10:00", + "EmitLatencySpectrum": false, + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s http://{serverip}:{ServerPort}/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\"" + }, + "Actions": [ + { + "Type": "AspNetServerExecutor", + "Parameters": { + "Role": "Server", + "Scenario": "ExecuteJsonSerializationBenchmark", + "PackageName": "aspnetbenchmarks", + "DotNetSdkPackageName": "dotnetsdk", + "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "$.Parameters.ServerPort" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Warmup", + "MetricScenario": "Warmup_ASP.NET-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "ServerPort": "$.Parameters.ServerPort", + "ThreadCount": "64", + "Connection": 4096, + "TestDuration": "00:00:45", + "WarmUp": "true", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "ASP.NET,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Benchmark Measurement", + "MetricScenario": "ASP.NET-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "ServerPort": "$.Parameters.ServerPort", + "ThreadCount": "64", + "Connection": 4096, + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "ASP.NET,WRK" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip", + "Packages-Yum": "zlib-devel,pcre-devel,libevent-devel,openssl-devel,git,gcc-c++,make,autoconf,automake", + "Packages-Dnf": "git,make,gcc,zlib-devel,openssl-devel,icu,glibc-devel,kernel-headers,binutils,perl,unzip" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "DisableFirewall", + "Command": "sudo iptables -P INPUT ACCEPT" + } + }, + { + "Type": "ChocolateyInstallation", + "Parameters": { + "Scenario": "InstallChocolatey", + "PackageName": "chocolatey" + } + }, + { + "Type": "ChocolateyPackageInstallation", + "Parameters": { + "Scenario": "InstallGit", + "PackageName": "chocolatey", + "Packages": "git" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneAspNetBenchmarksRepo", + "RepoUri": "https://github.com/aspnet/Benchmarks.git", + "PackageName": "aspnetbenchmarks", + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "DotNetInstallation", + "Parameters": { + "Scenario": "InstallDotNetSdk", + "DotNetVersion": "$.Parameters.DotNetVersion", + "PackageName": "dotnetsdk" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-AFFINITY.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-AFFINITY.json new file mode 100644 index 0000000000..b244f863da --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-AFFINITY.json @@ -0,0 +1,93 @@ +{ + "Description": ".NET benchmarking Workload with CPU Core Affinity", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:05:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" + }, + "Parameters": { + "DotNetVersion": "8.0.204", + "TargetFramework": "net8.0", + "ServerPort": 9876, + "ServerCoreAffinity": "0-7", + "ClientCoreAffinity": "8-15" + }, + "Actions": [ + { + "Type": "AspNetServerExecutor", + "Parameters": { + "Role": "Server", + "Scenario": "ExecuteJsonSerializationBenchmarkWithAffinity", + "PackageName": "aspnetbenchmarks", + "DotNetSdkPackageName": "dotnetsdk", + "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "$.Parameters.ServerPort", + "BindToCores": true, + "CoreAffinity": "$.Parameters.ServerCoreAffinity" + } + }, + { + "Type": "BombardierExecutor", + "Parameters": { + "Role": "Client", + "Scenario": "ExecuteJsonSerializationBenchmarkWithAffinity", + "PackageName": "bombardier", + "CommandArguments": "--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ServerIp}:9876/json --print r --format json", + "WarmUp": false, + "Timeout": "00:05:00", + "BindToCores": true, + "CoreAffinity": "$.Parameters.ClientCoreAffinity" + } + } + ], + "Dependencies": [ + { + "Type": "ChocolateyInstallation", + "Parameters": { + "Scenario": "InstallChocolatey", + "PackageName": "chocolatey" + } + }, + { + "Type": "ChocolateyPackageInstallation", + "Parameters": { + "Scenario": "InstallGit", + "PackageName": "chocolatey", + "Packages": "git" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneAspNetBenchmarksRepo", + "RepoUri": "https://github.com/aspnet/Benchmarks.git", + "Commit": "cf5b6ee", + "PackageName": "aspnetbenchmarks" + } + }, + { + "Type": "DotNetInstallation", + "Parameters": { + "Scenario": "InstallDotNetSdk", + "DotNetVersion": "$.Parameters.DotNetVersion", + "PackageName": "dotnetsdk" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallBombardierPackage", + "BlobContainer": "packages", + "BlobName": "bombardier.1.2.5.zip", + "PackageName": "bombardier", + "Extract": true + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json index 44f2ef4970..ab5d00b09e 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json @@ -14,40 +14,38 @@ }, "Actions": [ { - "Type": "AspNetBenchServerExecutor", + "Type": "AspNetServerExecutor", "Parameters": { "Role": "Server", "Scenario": "ExecuteJsonSerializationBenchmark", "PackageName": "aspnetbenchmarks", - "BombardierPackageName": "bombardier", "DotNetSdkPackageName": "dotnetsdk", "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "9876", "AspNetCoreThreadCount": "$.Parameters.AspNetCoreThreadCount", "DotNetSystemNetSocketsThreadCount": "$.Parameters.DotNetSystemNetSocketsThreadCount" } }, { - "Type": "AspNetBenchClientExecutor", + "Type": "WrkExecutor", "Parameters": { "Role": "Client", "Scenario": "ExecuteJsonSerializationBenchmarkWarmUp", - "PackageName": "aspnetbenchmarks", - "BombardierPackageName": "bombardier", - "DotNetSdkPackageName": "dotnetsdk", - "TargetFramework": "$.Parameters.TargetFramework", - "WrkCommandLine": "-t 256 -c 256 -d 45s --timeout 10s http://{ipAddress}:{port}/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q = 0.9,application/xml;q=0.8,*/*;q=0.7\"" + "PackageName": "wrk", + "CommandArguments": "-t 256 -c 256 -d 45s --timeout 10s http://{ServerIp}:9876/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\"", + "WarmUp": true, + "Timeout": "00:05:00" } }, { - "Type": "AspNetBenchClientExecutor", + "Type": "WrkExecutor", "Parameters": { "Role": "Client", "Scenario": "ExecuteJsonSerializationBenchmark", - "PackageName": "aspnetbenchmarks", - "BombardierPackageName": "bombardier", - "DotNetSdkPackageName": "dotnetsdk", - "TargetFramework": "$.Parameters.TargetFramework", - "WrkCommandLine": "-t 256 -c 256 -d 15s --timeout 10s http://{ipAddress}:{port}/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q = 0.9,application/xml;q=0.8,*/*;q=0.7\"" + "PackageName": "wrk", + "CommandArguments": "-t 256 -c 256 -d 15s --timeout 10s http://{ServerIp}:9876/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\"", + "WarmUp": false, + "Timeout": "00:05:00" } } @@ -85,6 +83,17 @@ "Role": "Client" } }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, { "Type": "GitRepoClone", "Parameters": { diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json index f4baf54582..d8e1606d0b 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json @@ -7,17 +7,30 @@ }, "Parameters": { "DotNetVersion": "8.0.204", - "TargetFramework": "net8.0" + "TargetFramework": "net8.0", + "ServerPort": 9876 }, "Actions": [ { - "Type": "AspNetBenchExecutor", + "Type": "AspNetServerExecutor", "Parameters": { + "Role": "Server", "Scenario": "ExecuteJsonSerializationBenchmark", "PackageName": "aspnetbenchmarks", - "BombardierPackageName": "bombardier", "DotNetSdkPackageName": "dotnetsdk", - "TargetFramework": "$.Parameters.TargetFramework" + "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "$.Parameters.ServerPort" + } + }, + { + "Type": "BombardierExecutor", + "Parameters": { + "Role": "Client", + "Scenario": "ExecuteJsonSerializationBenchmark", + "PackageName": "bombardier", + "CommandArguments": "--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ServerIp}:9876/json --print r --format json", + "WarmUp": false, + "Timeout": "00:05:00" } } ], @@ -63,6 +76,12 @@ "PackageName": "bombardier", "Extract": true } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } } ] } \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK-RP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK-RP.json new file mode 100644 index 0000000000..2f930f5299 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK-RP.json @@ -0,0 +1,404 @@ +{ + "Description": "Networking benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "01:00:00", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu", + "Note": "Reverse Proxy config requires three nodes for the roles Client, Reverse Proxy, and Server." + }, + "Parameters": { + "TestDuration": "00:02:30", + "Timeout": "00:30:00", + "FileSizeInKB": 1, + "EmitLatencySpectrum": false, + "Workers": null, + "TargetService": "reverse-proxy", + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s https://{reverseproxyip}/{FileSizeInKB}kb" + }, + "Actions": [ + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "Server", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "ReverseProxy", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip nginx" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "StopNginxAutostart", + "Command": "sudo systemctl disable nginx" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "ReverseProxy" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK.json new file mode 100644 index 0000000000..182783d5a6 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK.json @@ -0,0 +1,382 @@ +{ + "Description": "Networking benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "01:00:00", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu" + }, + "Parameters": { + "TestDuration": "00:02:30", + "Timeout": "00:30:00", + "FileSizeInKB": 1, + "EmitLatencySpectrum": false, + "Workers": null, + "TargetService": "server", + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s https://{serverip}/api_new/{FileSizeInKB}kb" + }, + "Actions": [ + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "Server", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip nginx" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "StopNginxAutostart", + "Command": "sudo systemctl disable nginx" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2-RP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2-RP.json new file mode 100644 index 0000000000..f82c68a429 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2-RP.json @@ -0,0 +1,421 @@ +{ + "Description": "Networking benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "01:00:00", + "SupportedPlatforms": "linux-x64", + "SupportedOperatingSystems": "Ubuntu 22", + "Note": "Reverse Proxy config requires three nodes for the roles Client, Reverse Proxy, and Server." + }, + "Parameters": { + "TestDuration": "00:02:30", + "Timeout": "00:20:00", + "FileSizeInKB": 1, + "EmitLatencySpectrum": false, + "Workers": null, + "TargetService": "reverse-proxy", + "Rate": 1000, + "CommandArguments": "--rate {Rate} --latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s https://{reverseproxyip}/{FileSizeInKB}kb" + }, + "Actions": [ + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "Server", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "ReverseProxy", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip nginx" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "StopNginxAutostart", + "Command": "sudo systemctl disable nginx" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "ReverseProxy" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk2", + "RepoUri": "https://github.com/giltene/wrk2", + "PackageName": "wrk2", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk2", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk2}", + "Role": "Client" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2.json new file mode 100644 index 0000000000..e319436f1f --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2.json @@ -0,0 +1,399 @@ +{ + "Description": "Networking benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "01:00:00", + "SupportedPlatforms": "linux-x64", + "SupportedOperatingSystems": "Ubuntu 22" + }, + "Parameters": { + "TestDuration": "00:02:30", + "Timeout": "00:20:00", + "FileSizeInKB": 1, + "EmitLatencySpectrum": false, + "Workers": null, + "TargetService": "server", + "Rate": 1000, + "CommandArguments": "--rate {Rate} --latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s https://{serverip}/api_new/{FileSizeInKB}kb" + }, + "Actions": [ + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "Server", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip nginx" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "StopNginxAutostart", + "Command": "sudo systemctl disable nginx" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk2", + "RepoUri": "https://github.com/giltene/wrk2", + "PackageName": "wrk2", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk2", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk2}", + "Role": "Client" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} \ No newline at end of file From de9213640ccc7792034c458a7bba4a1a520af570 Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Thu, 9 Apr 2026 22:42:09 -0700 Subject: [PATCH 02/12] commit changes and Profile Updates. --- .../AspNetOrchardServerExecutorTests.cs | 5 + .../AspNetBench/AspNetServerExecutorTests.cs | 5 + .../AspNetBench/BombardierExecutorTests.cs | 8 +- .../Wrk/WrkExecutorTest.cs | 6 +- .../ASPNET/AspNetOrchardServerExecutor.cs | 71 ++++++- .../ASPNET/AspNetServerExecutor.cs | 71 ++++++- .../Bombardier/BombardierExecutor.cs | 58 +++--- .../Nginx/NginxServerExecutor.cs | 1 - .../VirtualClient.Actions/Wrk/WrkExecutor.cs | 62 ++++--- .../profiles/PERF-ASPNET-TEJSON-WRK.json | 150 --------------- .../profiles/PERF-ASPNETBENCH-AFFINITY.json | 93 ---------- ....json => PERF-WEB-ASPNET-ORCHARD-WRK.json} | 0 .../PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json | 172 +++++++++++++++++ .../profiles/PERF-WEB-ASPNET-TEJSON-WRK.json | 164 +++++++++++++++++ website/docs/overview/overview.md | 2 +- .../aspnet-benchmarks-profiles.md | 134 ++++++++++++++ .../aspnet-benchmarks/aspnet-benchmarks.md | 173 ++++++++++++++++++ .../aspnetbench/aspnetbench-profiles.md | 49 ----- .../docs/workloads/aspnetbench/aspnetbench.md | 56 ------ .../docs/workloads/bombardier/bombardier.md | 90 +++++++++ website/docs/workloads/nginx/nginx.md | 86 +++++++++ website/docs/workloads/wrk/wrk.md | 166 +++++++++++++++++ 22 files changed, 1199 insertions(+), 423 deletions(-) delete mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-TEJSON-WRK.json delete mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-AFFINITY.json rename src/VirtualClient/VirtualClient.Main/profiles/{PERF-ASPNET-ORCHARD-WRK.json => PERF-WEB-ASPNET-ORCHARD-WRK.json} (100%) create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK.json create mode 100644 website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md create mode 100644 website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks.md delete mode 100644 website/docs/workloads/aspnetbench/aspnetbench-profiles.md delete mode 100644 website/docs/workloads/aspnetbench/aspnetbench.md create mode 100644 website/docs/workloads/bombardier/bombardier.md create mode 100644 website/docs/workloads/nginx/nginx.md create mode 100644 website/docs/workloads/wrk/wrk.md diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs index 194d862db6..0387e76c1e 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs @@ -204,6 +204,11 @@ public string DotNetExePath { base.Validate(); } + + protected override Task WaitForPortReadyAsync(EventContext context, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } } private void SetupDefaultMockBehaviors(PlatformID platform) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs index 6d655c0b75..2969a77e4b 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs @@ -235,6 +235,11 @@ public string DotNetExePath { base.Validate(); } + + protected override Task WaitForPortReadyAsync(EventContext context, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } } private void SetupDefaultMockBehaviors(PlatformID platform) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs index 5e3facd268..c252a6b4ed 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs @@ -72,17 +72,15 @@ public void BombardierExecutorGetBombardierVersionParsesVersionWithoutVPrefix() } [Test] - public void BombardierExecutorGetBombardierVersionThrowsOnUnparsableOutput() + public void BombardierExecutorGetBombardierVersionReturnsNullOnUnparsableOutput() { this.mockFixture.SetupProcessOutput(".*--version.*", "unrecognized output"); using (TestBombardierExecutor executor = new TestBombardierExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) { executor.PackageDirectory = this.mockPackage.Path; - WorkloadException exception = Assert.Throws( - () => executor.GetBombardierVersion(EventContext.None, CancellationToken.None)); - - Assert.AreEqual(ErrorReason.CriticalWorkloadFailure, exception.Reason); + string version = executor.GetBombardierVersion(EventContext.None, CancellationToken.None); + Assert.IsNull(version); } } diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs index ef1c179329..f16d69bb3d 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs @@ -593,7 +593,7 @@ public void GetWrkVersionReturnsCorrectVersion() } [Test] - public void GetWrkVersion_ThrowsException_WhenVersionCannotBeParsed() + public void GetWrkVersion_ReturnsNull_WhenVersionCannotBeParsed() { this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); @@ -606,8 +606,8 @@ public void GetWrkVersion_ThrowsException_WhenVersionCannotBeParsed() .TrackProcesses() .SetupProcessOutput("--version", "Invalid output without version"); - WorkloadException exception = Assert.Throws(() => executor.GetWrkVersion()); - Assert.AreEqual("Failed to parse wrk version from output.", exception.Message); + string version = executor.GetWrkVersion(); + Assert.IsNull(version); this.mockFixture.Tracking.AssertCommandsExecuted(true, "sudo bash .* --version"); } diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs index 3877a86d40..4af2eb0e6c 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs @@ -9,6 +9,7 @@ namespace VirtualClient.Actions using System.Linq; using System.Net; using System.Net.Http; + using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -25,6 +26,7 @@ namespace VirtualClient.Actions /// /// AspNet Orchard Server Executor. /// + [SupportedPlatforms("linux-arm64,linux-x64,win-arm64,win-x64")] public class AspNetOrchardServerExecutor : VirtualClientMultiRoleComponent { private Task serverProcess; @@ -134,7 +136,7 @@ public string CoreAffinity /// /// Disposes of resources used by the executor including shutting down any - /// instances of Redis server running. + /// instances of server running. /// protected override void Dispose(bool disposing) { @@ -200,9 +202,16 @@ protected override void Validate() protected void InitializeApiClients() { IApiClientManager clientManager = this.Dependencies.GetService(); - ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); - this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + if (!this.IsMultiRoleLayout()) + { + this.ServerApi = clientManager.GetOrCreateApiClient(IPAddress.Loopback.ToString(), IPAddress.Loopback); + } + else + { + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + } } /// @@ -248,16 +257,28 @@ protected override Task ExecuteAsync(EventContext telemetryContext, Cancellation await this.KillServerInstancesAsync(telemetryContext, cancellationToken); await this.BuildAspNetOrchardAsync(telemetryContext, cancellationToken); this.StartServerInstances(telemetryContext, cancellationToken); + await this.WaitForPortReadyAsync(telemetryContext, cancellationToken); await this.SaveStateAsync(telemetryContext, cancellationToken); this.SetServerOnline(true); - using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) + if (!this.IsMultiRoleLayout()) + { + // In single-VM mode, clear cleanup tasks to prevent the base class + // CleanupAsync from killing the server process. The server must stay + // alive for the subsequent client action. It will be killed by + // KillServerInstancesAsync on the next iteration or by Dispose. + this.CleanupTasks.Clear(); + } + else { - await Task.WhenAny(this.serverProcess); - if (cancellationToken.IsCancellationRequested) + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) { - await Task.WhenAll(this.serverProcess); + await Task.WhenAny(this.serverProcess); + if (cancellationToken.IsCancellationRequested) + { + await Task.WhenAll(this.serverProcess); + } } } } @@ -270,6 +291,42 @@ protected override Task ExecuteAsync(EventContext telemetryContext, Cancellation }); } + /// + /// Waits for the configured port to start accepting TCP connections. + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected virtual async Task WaitForPortReadyAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + int port = int.Parse(this.ServerPort); + TimeSpan timeout = TimeSpan.FromMinutes(5); + DateTime deadline = DateTime.UtcNow.Add(timeout); + + this.Logger.LogTraceMessage($"{this.TypeName}: Waiting for server to accept connections on port {port}..."); + + while (DateTime.UtcNow < deadline && !cancellationToken.IsCancellationRequested) + { + try + { + using (TcpClient client = new TcpClient()) + { + await client.ConnectAsync(IPAddress.Loopback, port).ConfigureAwait(false); + this.Logger.LogTraceMessage($"{this.TypeName}: Server is accepting connections on port {port}."); + return; + } + } + catch (SocketException) + { + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false); + } + } + + cancellationToken.ThrowIfCancellationRequested(); + throw new WorkloadException( + $"The server did not start accepting connections on port {port} within {timeout}.", + ErrorReason.WorkloadFailed); + } + private Task DeleteStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) { EventContext relatedContext = telemetryContext.Clone(); diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs index 7ba8cbd6ed..25bd98a18d 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs @@ -9,6 +9,7 @@ namespace VirtualClient.Actions using System.Linq; using System.Net; using System.Net.Http; + using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -214,10 +215,16 @@ protected override void Validate() protected void InitializeApiClients() { IApiClientManager clientManager = this.Dependencies.GetService(); - bool isSingleVM = !this.IsMultiRoleLayout(); - ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); - this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + if (!this.IsMultiRoleLayout()) + { + this.ServerApi = clientManager.GetOrCreateApiClient(IPAddress.Loopback.ToString(), IPAddress.Loopback); + } + else + { + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + } } /// @@ -240,17 +247,29 @@ protected override Task ExecuteAsync(EventContext telemetryContext, Cancellation await this.KillServerInstancesAsync(telemetryContext, cancellationToken); await this.BuildAspNetBenchAsync(telemetryContext, cancellationToken); this.StartServerInstances(telemetryContext, cancellationToken); + await this.WaitForPortReadyAsync(telemetryContext, cancellationToken); await this.SaveStateAsync(telemetryContext, cancellationToken); this.SetServerOnline(true); - using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) + if (!this.IsMultiRoleLayout()) { - await Task.WhenAny(this.serverProcess); - - if (cancellationToken.IsCancellationRequested) + // In single-VM mode, clear cleanup tasks to prevent the base class + // CleanupAsync from killing the server process. The server must stay + // alive for the subsequent client action. It will be killed by + // KillServerInstancesAsync on the next iteration or by Dispose. + this.CleanupTasks.Clear(); + } + else + { + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) { - await Task.WhenAll(this.serverProcess); + await Task.WhenAny(this.serverProcess); + + if (cancellationToken.IsCancellationRequested) + { + await Task.WhenAll(this.serverProcess); + } } } } @@ -283,6 +302,42 @@ await this.ExecuteCommandAsync(this.dotnetExePath, buildArgument, this.aspnetBen "Benchmarks.dll"); } + /// + /// Waits for the configured port to start accepting TCP connections. + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected virtual async Task WaitForPortReadyAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + int port = int.Parse(this.ServerPort); + TimeSpan timeout = TimeSpan.FromMinutes(5); + DateTime deadline = DateTime.UtcNow.Add(timeout); + + this.Logger.LogTraceMessage($"{this.TypeName}: Waiting for server to accept connections on port {port}..."); + + while (DateTime.UtcNow < deadline && !cancellationToken.IsCancellationRequested) + { + try + { + using (TcpClient client = new TcpClient()) + { + await client.ConnectAsync(IPAddress.Loopback, port).ConfigureAwait(false); + this.Logger.LogTraceMessage($"{this.TypeName}: Server is accepting connections on port {port}."); + return; + } + } + catch (SocketException) + { + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false); + } + } + + cancellationToken.ThrowIfCancellationRequested(); + throw new WorkloadException( + $"The server did not start accepting connections on port {port} within {timeout}.", + ErrorReason.WorkloadFailed); + } + private Task DeleteStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) { EventContext relatedContext = telemetryContext.Clone(); diff --git a/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs index b4be5dcdf3..942c711c68 100644 --- a/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs @@ -8,6 +8,7 @@ namespace VirtualClient.Actions using System.IO; using System.IO.Abstractions; using System.Linq; + using System.Net; using System.Net.Http; using System.Text.RegularExpressions; using System.Threading; @@ -268,20 +269,27 @@ protected void InitializeApiClients() { IApiClientManager clientManager = this.Dependencies.GetService(); - ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); - this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); - this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ServerApi); - - IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); - if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + if (!this.IsMultiRoleLayout()) { - this.ReverseProxyApi = null; + this.ServerApi = clientManager.GetOrCreateApiClient(IPAddress.Loopback.ToString(), IPAddress.Loopback); } else { - ClientInstance reverseProxyInstance = reverseProxyInstanceEnumerable.FirstOrDefault(); - this.ReverseProxyApi = clientManager.GetOrCreateApiClient(reverseProxyInstance.Name, reverseProxyInstance); - this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ReverseProxyApi); + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ServerApi); + + IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); + if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + { + this.ReverseProxyApi = null; + } + else + { + ClientInstance reverseProxyInstance = reverseProxyInstanceEnumerable.FirstOrDefault(); + this.ReverseProxyApi = clientManager.GetOrCreateApiClient(reverseProxyInstance.Name, reverseProxyInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ReverseProxyApi); + } } } @@ -307,8 +315,9 @@ protected string GetCommandLineArguments(CancellationToken cancellationToken) { foreach (Match match in matches) { - ClientInstance roleIP = this.GetLayoutClientInstances(kvp.Key).FirstOrDefault(); - result = Regex.Replace(result, match.Value, roleIP.IPAddress); + IEnumerable instances = this.GetLayoutClientInstances(kvp.Key, throwIfNotExists: false); + string ipAddress = instances?.FirstOrDefault()?.IPAddress ?? IPAddress.Loopback.ToString(); + result = Regex.Replace(result, match.Value, ipAddress); } } } @@ -439,16 +448,16 @@ protected async Task ExecuteWorkloadAsync(string commandArguments, string workin protected string GetBombardierVersion(EventContext telemetryContext, CancellationToken cancellationToken) { string bombardierPath = this.Combine(this.PackageDirectory, this.Platform == PlatformID.Unix ? "bombardier" : "bombardier.exe"); - - this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(bombardierPath); - - string commandArguments = "--version"; - string versionPattern = @"bombardier\s+version\s+v?(\d+\.\d+\.\d+)"; - Regex versionRegex = new Regex(versionPattern, RegexOptions.Compiled | RegexOptions.IgnoreCase); string bombardierVersion = null; try { + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(bombardierPath); + + string commandArguments = "--version"; + string versionPattern = @"bombardier\s+(?:version\s+)?v?(\d+\.\d+\.\d+)"; + Regex versionRegex = new Regex(versionPattern, RegexOptions.Compiled | RegexOptions.IgnoreCase); + using (IProcessProxy process = this.ExecuteCommandAsync(bombardierPath, commandArguments, workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).Result) { if (!cancellationToken.IsCancellationRequested) @@ -457,23 +466,24 @@ protected string GetBombardierVersion(EventContext telemetryContext, Cancellatio string output = process.StandardOutput.ToString(); Match match = versionRegex.Match(output); + if (!match.Success) + { + output = process.StandardError.ToString(); + match = versionRegex.Match(output); + } + if (match.Success) { bombardierVersion = match.Groups[1].Value; telemetryContext.AddContext("BombardierVersion", bombardierVersion); this.Logger.LogMessage($"{this.TypeName}.BombardierVersionCaptured", LogLevel.Information, telemetryContext); } - else - { - throw new WorkloadException("Failed to parse bombardier version from output.", ErrorReason.CriticalWorkloadFailure); - } } } } catch (Exception exc) { - this.Logger.LogMessage($"{this.TypeName}.BombardierVersionCaptureError", LogLevel.Error, telemetryContext.Clone().AddError(exc)); - throw; + this.Logger.LogErrorMessage(exc, telemetryContext); } return bombardierVersion; diff --git a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs index 52173b12c6..b3b2daeb78 100644 --- a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs @@ -10,7 +10,6 @@ namespace VirtualClient.Actions using System.Text; using System.Threading; using System.Threading.Tasks; - using Microsoft.CodeAnalysis; using Microsoft.Extensions.DependencyInjection; using VirtualClient.Common; using VirtualClient.Common.Contracts; diff --git a/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs index dcc894fd34..9821657e6b 100644 --- a/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs @@ -8,12 +8,12 @@ namespace VirtualClient.Actions using System.IO; using System.IO.Abstractions; using System.Linq; + using System.Net; using System.Net.Http; using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; - using Microsoft.CodeAnalysis; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Polly; @@ -334,20 +334,27 @@ protected void InitializeApiClients() { IApiClientManager clientManager = this.Dependencies.GetService(); - ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); - this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); - this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ServerApi); - - IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); - if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + if (!this.IsMultiRoleLayout()) { - this.ReverseProxyApi = null; + this.ServerApi = clientManager.GetOrCreateApiClient(IPAddress.Loopback.ToString(), IPAddress.Loopback); } else { - ClientInstance reverseProxyInstance = reverseProxyInstanceEnumerable.FirstOrDefault(); - this.ReverseProxyApi = clientManager.GetOrCreateApiClient(reverseProxyInstance.Name, reverseProxyInstance); - this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ReverseProxyApi); + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ServerApi); + + IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); + if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + { + this.ReverseProxyApi = null; + } + else + { + ClientInstance reverseProxyInstance = reverseProxyInstanceEnumerable.FirstOrDefault(); + this.ReverseProxyApi = clientManager.GetOrCreateApiClient(reverseProxyInstance.Name, reverseProxyInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ReverseProxyApi); + } } } @@ -373,8 +380,9 @@ protected string GetCommandLineArguments(CancellationToken cancellationToken) { foreach (Match match in matches) { - ClientInstance roleIP = this.GetLayoutClientInstances(kvp.Key).FirstOrDefault(); - result = Regex.Replace(result, match.Value, roleIP.IPAddress); + IEnumerable instances = this.GetLayoutClientInstances(kvp.Key, throwIfNotExists: false); + string ipAddress = instances?.FirstOrDefault()?.IPAddress ?? IPAddress.Loopback.ToString(); + result = Regex.Replace(result, match.Value, ipAddress); } } } @@ -486,17 +494,18 @@ protected async Task ExecuteWorkloadAsync(string commandArguments, string workin /// Wrk Version protected string GetWrkVersion(EventContext telemetryContext, CancellationToken cancellationToken) { - string scriptPath = this.Combine(this.PackageDirectory, WrkExecutor.WrkRunShell); - this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(scriptPath); - - string command = $"bash {scriptPath}"; - string commandArguments = "--version"; - string versionPattern = @"wrk\s(\d+\.\d+\.\d+)"; - Regex versionRegex = new Regex(versionPattern, RegexOptions.Compiled); string wrkVersion = null; try { + string scriptPath = this.Combine(this.PackageDirectory, WrkExecutor.WrkRunShell); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(scriptPath); + + string command = $"bash {scriptPath}"; + string commandArguments = "--version"; + string versionPattern = @"wrk\s(\d+\.\d+\.\d+)"; + Regex versionRegex = new Regex(versionPattern, RegexOptions.Compiled); + using (IProcessProxy process = this.ExecuteCommandAsync(command, commandArguments, workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).Result) { if (!cancellationToken.IsCancellationRequested) @@ -505,23 +514,24 @@ protected string GetWrkVersion(EventContext telemetryContext, CancellationToken string output = process.StandardOutput.ToString(); Match match = versionRegex.Match(output); + if (!match.Success) + { + output = process.StandardError.ToString(); + match = versionRegex.Match(output); + } + if (match.Success) { wrkVersion = match.Groups[1].Value; telemetryContext.AddContext("WrkVersion", wrkVersion); this.Logger.LogMessage($"{this.TypeName}.WrkVersionCaptured", LogLevel.Information, telemetryContext); } - else - { - throw new WorkloadException("Failed to parse wrk version from output.", ErrorReason.CriticalWorkloadFailure); - } } } } catch (Exception exc) { - this.Logger.LogMessage($"{this.TypeName}.WrkVersionCaptureError", LogLevel.Error, telemetryContext.Clone().AddError(exc)); - throw; + this.Logger.LogErrorMessage(exc, telemetryContext); } return wrkVersion; diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-TEJSON-WRK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-TEJSON-WRK.json deleted file mode 100644 index ea2bad74f1..0000000000 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-TEJSON-WRK.json +++ /dev/null @@ -1,150 +0,0 @@ -{ - "Description": ".NET benchmarking Workload", - "Metadata": { - "RecommendedMinimumExecutionTime": "00:10:00", - "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", - "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" - }, - "Parameters": { - "DotNetVersion": "9.0.101", - "TargetFramework": "net9.0", - "ServerPort": 9876, - "TestDuration": "00:00:15", - "Timeout": "00:10:00", - "EmitLatencySpectrum": false, - "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s http://{serverip}:{ServerPort}/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\"" - }, - "Actions": [ - { - "Type": "AspNetServerExecutor", - "Parameters": { - "Role": "Server", - "Scenario": "ExecuteJsonSerializationBenchmark", - "PackageName": "aspnetbenchmarks", - "DotNetSdkPackageName": "dotnetsdk", - "TargetFramework": "$.Parameters.TargetFramework", - "ServerPort": "$.Parameters.ServerPort" - } - }, - { - "Type": "WrkExecutor", - "Parameters": { - "PackageName": "wrk", - "Scenario": "Warmup", - "MetricScenario": "Warmup_ASP.NET-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", - "CommandArguments": "$.Parameters.CommandArguments", - "ServerPort": "$.Parameters.ServerPort", - "ThreadCount": "64", - "Connection": 4096, - "TestDuration": "00:00:45", - "WarmUp": "true", - "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", - "Role": "Client", - "Timeout": "$.Parameters.Timeout", - "Tags": "ASP.NET,WRK" - } - }, - { - "Type": "WrkExecutor", - "Parameters": { - "PackageName": "wrk", - "Scenario": "Benchmark Measurement", - "MetricScenario": "ASP.NET-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", - "CommandArguments": "$.Parameters.CommandArguments", - "ServerPort": "$.Parameters.ServerPort", - "ThreadCount": "64", - "Connection": 4096, - "TestDuration": "$.Parameters.TestDuration", - "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", - "Role": "Client", - "Timeout": "$.Parameters.Timeout", - "Tags": "ASP.NET,WRK" - } - } - ], - "Dependencies": [ - { - "Type": "LinuxPackageInstallation", - "Parameters": { - "Scenario": "InstallLinuxPackages", - "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip", - "Packages-Yum": "zlib-devel,pcre-devel,libevent-devel,openssl-devel,git,gcc-c++,make,autoconf,automake", - "Packages-Dnf": "git,make,gcc,zlib-devel,openssl-devel,icu,glibc-devel,kernel-headers,binutils,perl,unzip" - } - }, - { - "Type": "ExecuteCommand", - "Parameters": { - "Scenario": "DisableFirewall", - "Command": "sudo iptables -P INPUT ACCEPT" - } - }, - { - "Type": "ChocolateyInstallation", - "Parameters": { - "Scenario": "InstallChocolatey", - "PackageName": "chocolatey" - } - }, - { - "Type": "ChocolateyPackageInstallation", - "Parameters": { - "Scenario": "InstallGit", - "PackageName": "chocolatey", - "Packages": "git" - } - }, - { - "Type": "GitRepoClone", - "Parameters": { - "Scenario": "CloneAspNetBenchmarksRepo", - "RepoUri": "https://github.com/aspnet/Benchmarks.git", - "PackageName": "aspnetbenchmarks", - "Role": "Server" - } - }, - { - "Type": "DependencyPackageInstallation", - "Parameters": { - "Scenario": "InstallWrkConfiguration", - "BlobContainer": "packages", - "BlobName": "wrkconfiguration.1.0.0.zip", - "PackageName": "wrkconfiguration", - "Extract": true, - "Role": "Client" - } - }, - { - "Type": "GitRepoClone", - "Parameters": { - "Scenario": "CloneWrk", - "RepoUri": "https://github.com/wg/wrk", - "PackageName": "wrk", - "Role": "Client" - } - }, - { - "Type": "ExecuteCommand", - "Parameters": { - "Scenario": "CompileWrk", - "Command": "make", - "WorkingDirectory": "{PackagePath:wrk}", - "Role": "Client" - } - }, - { - "Type": "DotNetInstallation", - "Parameters": { - "Scenario": "InstallDotNetSdk", - "DotNetVersion": "$.Parameters.DotNetVersion", - "PackageName": "dotnetsdk" - } - }, - { - "Type": "ApiServer", - "Parameters": { - "Scenario": "StartAPIServer" - } - } - ] -} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-AFFINITY.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-AFFINITY.json deleted file mode 100644 index b244f863da..0000000000 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-AFFINITY.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "Description": ".NET benchmarking Workload with CPU Core Affinity", - "Metadata": { - "RecommendedMinimumExecutionTime": "00:05:00", - "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", - "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" - }, - "Parameters": { - "DotNetVersion": "8.0.204", - "TargetFramework": "net8.0", - "ServerPort": 9876, - "ServerCoreAffinity": "0-7", - "ClientCoreAffinity": "8-15" - }, - "Actions": [ - { - "Type": "AspNetServerExecutor", - "Parameters": { - "Role": "Server", - "Scenario": "ExecuteJsonSerializationBenchmarkWithAffinity", - "PackageName": "aspnetbenchmarks", - "DotNetSdkPackageName": "dotnetsdk", - "TargetFramework": "$.Parameters.TargetFramework", - "ServerPort": "$.Parameters.ServerPort", - "BindToCores": true, - "CoreAffinity": "$.Parameters.ServerCoreAffinity" - } - }, - { - "Type": "BombardierExecutor", - "Parameters": { - "Role": "Client", - "Scenario": "ExecuteJsonSerializationBenchmarkWithAffinity", - "PackageName": "bombardier", - "CommandArguments": "--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ServerIp}:9876/json --print r --format json", - "WarmUp": false, - "Timeout": "00:05:00", - "BindToCores": true, - "CoreAffinity": "$.Parameters.ClientCoreAffinity" - } - } - ], - "Dependencies": [ - { - "Type": "ChocolateyInstallation", - "Parameters": { - "Scenario": "InstallChocolatey", - "PackageName": "chocolatey" - } - }, - { - "Type": "ChocolateyPackageInstallation", - "Parameters": { - "Scenario": "InstallGit", - "PackageName": "chocolatey", - "Packages": "git" - } - }, - { - "Type": "GitRepoClone", - "Parameters": { - "Scenario": "CloneAspNetBenchmarksRepo", - "RepoUri": "https://github.com/aspnet/Benchmarks.git", - "Commit": "cf5b6ee", - "PackageName": "aspnetbenchmarks" - } - }, - { - "Type": "DotNetInstallation", - "Parameters": { - "Scenario": "InstallDotNetSdk", - "DotNetVersion": "$.Parameters.DotNetVersion", - "PackageName": "dotnetsdk" - } - }, - { - "Type": "DependencyPackageInstallation", - "Parameters": { - "Scenario": "InstallBombardierPackage", - "BlobContainer": "packages", - "BlobName": "bombardier.1.2.5.zip", - "PackageName": "bombardier", - "Extract": true - } - }, - { - "Type": "ApiServer", - "Parameters": { - "Scenario": "StartAPIServer" - } - } - ] -} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-ORCHARD-WRK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-ORCHARD-WRK.json similarity index 100% rename from src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-ORCHARD-WRK.json rename to src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-ORCHARD-WRK.json diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json new file mode 100644 index 0000000000..9cf49811f8 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json @@ -0,0 +1,172 @@ +{ + "Description": ".NET benchmarking Workload with CPU Core Affinity (supports .NET 9 and .NET 10)", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" + }, + "Parameters": { + "DotNetVersion": "9", + "TargetFramework": "net9.0", + "ServerPort": 9876, + "ServerCoreAffinity": "0-7", + "ClientCoreAffinity": "8-15", + "TestDuration": "00:00:15", + "ServerOnlineTimeout": "00:10:00", + "EmitLatencySpectrum": false, + "WaitForServer": false, + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s http://{serverip}:{ServerPort}/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\"" + }, + "ParametersOn": [ + { + "Condition": "{calculate(\"{DotNetVersion}\" == \"9\")}", + "DotNetVersion": "9.0.111", + "TargetFramework": "net9.0" + }, + { + "Condition": "{calculate(\"{DotNetVersion}\" == \"10\")}", + "DotNetVersion": "10.0.101", + "TargetFramework": "net10.0" + } + ], + "Actions": [ + { + "Type": "AspNetServerExecutor", + "Parameters": { + "Role": "Server", + "Scenario": "ExecuteJsonSerializationBenchmarkWithAffinity", + "PackageName": "aspnetbenchmarks", + "DotNetSdkPackageName": "dotnetsdk", + "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "$.Parameters.ServerPort", + "BindToCores": true, + "CoreAffinity": "$.Parameters.ServerCoreAffinity" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "warmup", + "MetricScenario": "Warmup_ASP.NET-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "ServerPort": "$.Parameters.ServerPort", + "ThreadCount": "64", + "Connection": 4096, + "TestDuration": "00:00:45", + "WaitForServer": "$.Parameters.WaitForServer", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Timeout": "$.Parameters.ServerOnlineTimeout", + "Tags": "ASP.NET,WRK", + "BindToCores": true, + "CoreAffinity": "$.Parameters.ClientCoreAffinity" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "benchmark_measurement", + "MetricScenario": "ASP.NET-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "ServerPort": "$.Parameters.ServerPort", + "ThreadCount": "64", + "Connection": 4096, + "TestDuration": "$.Parameters.TestDuration", + "WaitForServer": "$.Parameters.WaitForServer", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Timeout": "$.Parameters.ServerOnlineTimeout", + "Tags": "ASP.NET,WRK", + "BindToCores": true, + "CoreAffinity": "$.Parameters.ClientCoreAffinity" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip", + "Packages-Yum": "zlib-devel,pcre-devel,libevent-devel,openssl-devel,git,gcc-c++,make,autoconf,automake", + "Packages-Dnf": "git,make,gcc,zlib-devel,openssl-devel,icu,glibc-devel,kernel-headers,binutils,perl,unzip" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "DisableFirewall", + "Command": "sudo iptables -P INPUT ACCEPT" + } + }, + { + "Type": "ChocolateyInstallation", + "Parameters": { + "Scenario": "InstallChocolatey", + "PackageName": "chocolatey" + } + }, + { + "Type": "ChocolateyPackageInstallation", + "Parameters": { + "Scenario": "InstallGit", + "PackageName": "chocolatey", + "Packages": "git" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneAspNetBenchmarksRepo", + "RepoUri": "https://github.com/aspnet/Benchmarks.git", + "Commit": "5643540", + "PackageName": "aspnetbenchmarks" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "DotNetInstallation", + "Parameters": { + "Scenario": "InstallDotNetSdk", + "DotNetVersion": "$.Parameters.DotNetVersion", + "PackageName": "dotnetsdk" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK.json new file mode 100644 index 0000000000..2f8850c7e9 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK.json @@ -0,0 +1,164 @@ +{ + "Description": ".NET benchmarking Workload (supports .NET 9 and .NET 10)", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" + }, + "Parameters": { + "DotNetVersion": "9", + "TargetFramework": "net9.0", + "Port": 9876, + "TestDuration": "00:00:15", + "ServerOnlineTimeout": "00:10:00", + "EmitLatencySpectrum": false, + "WaitForServer": false, + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s http://{serverip}:{Port}/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\"" + }, + "ParametersOn": [ + { + "Condition": "{calculate(\"{DotNetVersion}\" == \"9\")}", + "DotNetVersion": "9.0.111", + "TargetFramework": "net9.0" + }, + { + "Condition": "{calculate(\"{DotNetVersion}\" == \"10\")}", + "DotNetVersion": "10.0.101", + "TargetFramework": "net10.0" + } + ], + "Actions": [ + { + "Type": "AspNetServerExecutor", + "Parameters": { + "Role": "Server", + "Scenario": "ExecuteJsonSerializationBenchmark", + "PackageName": "aspnetbenchmarks", + "DotNetSdkPackageName": "dotnetsdk", + "TargetFramework": "$.Parameters.TargetFramework", + "Port": "$.Parameters.Port" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "warmup", + "MetricScenario": "Warmup_ASP.NET-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "Port": "$.Parameters.Port", + "ThreadCount": "64", + "Connection": 4096, + "TestDuration": "00:00:45", + "WaitForServer": "$.Parameters.WaitForServer", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Timeout": "$.Parameters.ServerOnlineTimeout", + "Tags": "ASP.NET,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "benchmark_measurement", + "MetricScenario": "ASP.NET-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "Port": "$.Parameters.Port", + "ThreadCount": "64", + "Connection": 4096, + "TestDuration": "$.Parameters.TestDuration", + "WaitForServer": "$.Parameters.WaitForServer", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Timeout": "$.Parameters.ServerOnlineTimeout", + "Tags": "ASP.NET,WRK" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip", + "Packages-Yum": "zlib-devel,pcre-devel,libevent-devel,openssl-devel,git,gcc-c++,make,autoconf,automake", + "Packages-Dnf": "git,make,gcc,zlib-devel,openssl-devel,icu,glibc-devel,kernel-headers,binutils,perl,unzip" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "DisableFirewall", + "Command": "sudo iptables -P INPUT ACCEPT" + } + }, + { + "Type": "ChocolateyInstallation", + "Parameters": { + "Scenario": "InstallChocolatey", + "PackageName": "chocolatey" + } + }, + { + "Type": "ChocolateyPackageInstallation", + "Parameters": { + "Scenario": "InstallGit", + "PackageName": "chocolatey", + "Packages": "git" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneAspNetBenchmarksRepo", + "RepoUri": "https://github.com/aspnet/Benchmarks.git", + "Commit": "5643540", + "PackageName": "aspnetbenchmarks" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "DotNetInstallation", + "Parameters": { + "Scenario": "InstallDotNetSdk", + "DotNetVersion": "$.Parameters.DotNetVersion", + "PackageName": "dotnetsdk" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} \ No newline at end of file diff --git a/website/docs/overview/overview.md b/website/docs/overview/overview.md index 56bc735d95..e4d9124079 100644 --- a/website/docs/overview/overview.md +++ b/website/docs/overview/overview.md @@ -50,7 +50,7 @@ for using them. | **Workload/Benchmark** | **Specialization** | **Supported Platforms/Architectures** | **License(s)** | |------------------------|--------------------|---------------------------------------|----------------| | [7zip](https://microsoft.github.io/VirtualClient/docs/workloads/compression/7zip) | Compression | linux-x64, linux-arm64 | [GNU LGPL](https://www.7-zip.org/faq.html) | -| [AspNetBench](https://microsoft.github.io/VirtualClient/docs/workloads/aspnetbench/aspnetbench) | ASP.NET Web Server. | linux-x64, linux-arm64, win-x64, win-arm64 | [MIT (ASP.NET)](https://github.com/dotnet/aspnetcore/blob/main/LICENSE.txt)
[MIT (Bombardier)](https://github.com/codesenberg/bombardier/blob/master/LICENSE) | +| [ASP.NET Benchmarks](https://microsoft.github.io/VirtualClient/docs/workloads/aspnet-benchmarks/aspnet-benchmarks) | ASP.NET Kestrel web server throughput and latency. | linux-x64, linux-arm64, win-x64, win-arm64 | [MIT (ASP.NET)](https://github.com/dotnet/aspnetcore/blob/main/LICENSE.txt)
[MIT (Bombardier)](https://github.com/codesenberg/bombardier/blob/master/LICENSE) | | [BlenderBenchmark](https://microsoft.github.io/VirtualClient/docs/workloads/blenderbenchmark) | GPU/Graphics Rendering Performance | win-x64 | [GNU LGPL](https://projects.blender.org/infrastructure/blender-open-data/src/branch/main/LICENSE) | | [CoreMark](https://microsoft.github.io/VirtualClient/docs/workloads/coremark/coremark) | CPU Performance | linux-x64, linux-arm64 | [Apache+Custom](https://github.com/eembc/coremark/blob/main/LICENSE.md) | | [CoreMark Pro](https://microsoft.github.io/VirtualClient/docs/workloads/coremark) | Precision CPU | linux-x64, linux-arm64, win-x64, win-arm64 | [Apache+Custom](https://github.com/eembc/coremark-pro/blob/main/LICENSE.md) | diff --git a/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md b/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md new file mode 100644 index 0000000000..08fb5aa4de --- /dev/null +++ b/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md @@ -0,0 +1,134 @@ +# ASP.NET Benchmarks Workload Profiles +The following profiles run customer-representative or benchmarking scenarios using ASP.NET server workloads. + +* [Workload Details](./aspnet-benchmarks.md) + +## PERF-WEB-ASPNET-TEJSON-WRK.json +Runs the ASP.NET TechEmpower JSON serialization benchmark using Wrk. Includes a warm-up pass before the benchmark measurement. +Supports .NET 9 and .NET 10 via the `ParametersOn` conditional parameter system. + +* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK.json) + +* **Supported Platform/Architectures** + * linux-x64 + * linux-arm64 + * win-x64 + * win-arm64 + +* **Supports Disconnected Scenarios** + * No. Internet connection required. + +* **Dependencies** + The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. + * Internet connection. + + Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: + * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) + +* **Profile Parameters** + + | Parameter | Purpose | Default Value | + |---------------------------|-------------------------------------------------------------------|---------------| + | DotNetVersion | The major .NET version to use. Triggers `ParametersOn` conditional resolution. | 9 | + | TargetFramework | The .NET target framework to run. | net9.0 | + | TestDuration | Duration of each test run. | 00:00:15 | + | EmitLatencySpectrum | When `true`, emits fine-grained latency spectrum data. | false | + +* **Profile Runtimes** + See the 'Metadata' section of the profile for estimated runtimes. + +* **Usage Examples** + + ```bash + # Run with .NET 9 (default) + ./VirtualClient --profile=PERF-WEB-ASPNET-TEJSON-WRK.json --system=Demo --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" + + # Run with .NET 10 + ./VirtualClient --profile=PERF-WEB-ASPNET-TEJSON-WRK.json --system=Demo --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" --parameters="DotNetVersion=10" + ``` + +## PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json +Runs the ASP.NET TechEmpower JSON serialization benchmark using Wrk with CPU core affinity for both server and client. +Includes a warm-up pass before the benchmark measurement. Supports .NET 9 and .NET 10. + +* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json) + +* **Supported Platform/Architectures** + * linux-x64 + * linux-arm64 + * win-x64 + * win-arm64 + +* **Supports Disconnected Scenarios** + * No. Internet connection required. + +* **Dependencies** + The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. + * Internet connection. + + Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: + * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) + +* **Profile Parameters** + + | Parameter | Purpose | Default Value | + |---------------------------|-------------------------------------------------------------------|---------------| + | DotNetVersion | The major .NET version to use. Triggers `ParametersOn` conditional resolution. | 9 | + | TargetFramework | The .NET target framework to run. | net9.0 | + | ServerCoreAffinity | CPU core affinity for the server process. | 0-7 | + | ClientCoreAffinity | CPU core affinity for the client process. | 8-15 | + | TestDuration | Duration of each test run. | 00:00:15 | + | EmitLatencySpectrum | When `true`, emits fine-grained latency spectrum data. | false | + +* **Profile Runtimes** + See the 'Metadata' section of the profile for estimated runtimes. + +* **Usage Examples** + + ```bash + # Run with default affinity and .NET 9 + ./VirtualClient --profile=PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json --system=Demo --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" + + # Run with .NET 10 and custom core affinity + ./VirtualClient --profile=PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json --system=Demo --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" --parameters="DotNetVersion=10,,,ServerCoreAffinity=0-3,,,ClientCoreAffinity=4-7" + ``` + +## PERF-WEB-ASPNET-ORCHARD-WRK.json +Runs the ASP.NET OrchardCore CMS benchmark using Wrk. The server runs the OrchardCore.Cms.Web application +and Wrk benchmarks the `/about` endpoint. Includes a warm-up pass before the benchmark measurement. + +* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-ORCHARD-WRK.json) + +* **Supported Platform/Architectures** + * linux-x64 + * linux-arm64 + +* **Supports Disconnected Scenarios** + * No. Internet connection required. + +* **Dependencies** + The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. + * Internet connection. + + Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: + * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) + +* **Profile Parameters** + + | Parameter | Purpose | Default Value | + |---------------------------|-------------------------------------------------------------------|---------------| + | DotNetVersion | The version of the .NET SDK to download and install. | 9.0.203 | + | TargetFramework | The .NET target framework to run. | net9.0 | + | ServerPort | The port the OrchardCore server listens on. | 5014 | + | TestDuration | Duration of each test run. | 00:00:20 | + | EmitLatencySpectrum | When `true`, emits fine-grained latency spectrum data. | false | + +* **Profile Runtimes** + See the 'Metadata' section of the profile for estimated runtimes. + +* **Usage Examples** + + ```bash + # Execute the workload profile + ./VirtualClient --profile=PERF-WEB-ASPNET-ORCHARD-WRK.json --system=Demo --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" + ``` diff --git a/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks.md b/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks.md new file mode 100644 index 0000000000..8337681595 --- /dev/null +++ b/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks.md @@ -0,0 +1,173 @@ +# ASP.NET Benchmarks +The ASP.NET benchmarks measure the throughput and latency of ASP.NET Kestrel web applications under sustained HTTP load. +The workloads use a client-server architecture where the server runs an ASP.NET application and the client generates +HTTP requests using either [Bombardier](../bombardier/bombardier.md) or [Wrk](../wrk/wrk.md). + +Two server workloads are supported: + +- **TechEmpower JSON Serialization** — Based on the [ASP.NET Benchmarks](https://github.com/aspnet/benchmarks) project + (derived from the [TechEmpower framework](https://www.techempower.com/benchmarks/)), the server exposes a `/json` + endpoint that serializes a simple JSON object. Supports Bombardier and Wrk as client load generators. +- **OrchardCore CMS** — Runs the [OrchardCore](https://github.com/orchardcms/orchardcore) content management system + with a Blog recipe, benchmarked with Wrk against the `/about` endpoint. + +Both server configurations support **CPU core affinity** via the `BindToCores` and `CoreAffinity` parameters, allowing the +server process to be pinned to specific CPU cores for controlled performance measurement. + +* [ASP.NET Benchmarks GitHub](https://github.com/aspnet/benchmarks) +* [OrchardCore GitHub](https://github.com/orchardcms/orchardcore) +* [TechEmpower Framework Benchmarks](https://www.techempower.com/benchmarks/) +* [Bombardier Documentation](../bombardier/bombardier.md) +* [Wrk/Wrk2 Documentation](../wrk/wrk.md) + +## Deployment Modes +The ASP.NET benchmark workloads support two deployment modes: + +- **Multi-VM (Client-Server)** — The server and client run on separate machines connected via an + [environment layout](../../guides/0020-client-server.md) file. This is the recommended mode for production benchmarking + as it isolates server and client resource consumption. +- **Single-VM** — When no layout file is provided, both server and client actions run sequentially on the same machine. + The client connects to the server via the loopback address (`127.0.0.1`). This mode is useful for development, + validation, and quick smoke testing. + +CPU core affinity profiles (e.g., `PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json`) work in +both modes. In single-VM mode, core affinity is especially useful to prevent the server and client from contending +for the same CPU cores. + +## What is Being Measured? +The client tools (Bombardier or Wrk) generate concurrent HTTP requests against the ASP.NET server and capture latency +percentile distributions and throughput statistics. + +### TechEmpower JSON Serialization +The server exposes a `/json` endpoint that serializes a simple JSON object. This scenario measures raw HTTP request processing +performance of the Kestrel server with minimal application logic overhead. + +### OrchardCore CMS +The server runs a full OrchardCore CMS application with a Blog recipe. The `/about` endpoint exercises the full CMS rendering +pipeline including routing, middleware, and content rendering. + +## Workload Metrics +The following metrics are examples of those captured by the Virtual Client when running the ASP.NET benchmark workloads. + +### Bombardier Metrics +When Bombardier is used as the client: + +| Name | Example Value | Unit | Description | +|------------------------|--------------------|-------------|-----------------------------------------------------| +| Latency Average | 133.3698688 | milliseconds| Average HTTP response latency | +| Latency Max | 7123.06304 | milliseconds| Maximum HTTP response latency | +| Latency P50 | 83.39392 | milliseconds| HTTP response latency (50th percentile) | +| Latency P75 | 160.14336 | milliseconds| HTTP response latency (75th percentile) | +| Latency P90 | 286.4128 | milliseconds| HTTP response latency (90th percentile) | +| Latency P95 | 367.4112 | milliseconds| HTTP response latency (95th percentile) | +| Latency P99 | 637.534208 | milliseconds| HTTP response latency (99th percentile) | +| RequestPerSecond Avg | 32768.449018 | Reqs/sec | ASP.NET Web Requests per second (average) | +| RequestPerSecond Stddev| 6446.822354105378 | Reqs/sec | ASP.NET Web Requests per second (standard deviation)| +| RequestPerSecond P50 | 31049.462844 | Reqs/sec | ASP.NET Web Requests per second (P50) | +| RequestPerSecond P75 | 35597.436614 | Reqs/sec | ASP.NET Web Requests per second (P75) | +| RequestPerSecond P90 | 39826.205746 | Reqs/sec | ASP.NET Web Requests per second (P90) | +| RequestPerSecond P95 | 41662.542962 | Reqs/sec | ASP.NET Web Requests per second (P95) | +| RequestPerSecond P99 | 48600.556224 | Reqs/sec | ASP.NET Web Requests per second (P99) | + +### Wrk Metrics +When Wrk is used as the client (e.g., `PERF-WEB-ASPNET-TEJSON-WRK.json`, `PERF-WEB-ASPNET-ORCHARD-WRK.json`): + +| Name | Example Value | Unit | Description | +|--------------------|---------------|---------------|-----------------------------------------------| +| latency_p50 | 1.427 | milliseconds | HTTP response latency (50th percentile) | +| latency_p75 | 1.982 | milliseconds | HTTP response latency (75th percentile) | +| latency_p90 | 2.683 | milliseconds | HTTP response latency (90th percentile) | +| latency_p99 | 3.960 | milliseconds | HTTP response latency (99th percentile) | +| latency_p99_9 | 6.930 | milliseconds | HTTP response latency (99.9th percentile) | +| latency_p100 | 9.770 | milliseconds | HTTP response latency (100th percentile) | +| requests/sec | 16305.17 | requests/sec | Aggregate throughput | +| transfers/sec | 20.01 | megabytes/sec | Data transfer rate | + +See the [Wrk/Wrk2 documentation](../wrk/wrk.md) and [Bombardier documentation](../bombardier/bombardier.md) for the complete list of client metrics. + +## Profiles +The following profiles are available for the ASP.NET benchmark workloads. See the [profile details](./aspnet-benchmarks-profiles.md) +page for per-profile parameters, dependencies, and usage examples. + +| Profile Name | Description | Client Tool | Server | Platforms | +|------------------------------------------|------------------------------------------------------------------------------------|--------------------|-------------------------------|--------------------------------------------| +| PERF-WEB-ASPNET-TEJSON-WRK.json | TechEmpower JSON serialization benchmark using Wrk with warm-up pass. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json | TechEmpower JSON serialization benchmark using Wrk with CPU core affinity. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-ORCHARD-WRK.json | OrchardCore CMS benchmark using Wrk with warm-up pass. | WrkExecutor | AspNetOrchardServerExecutor | linux-x64, linux-arm64 | + +## Server Parameters +The following tables describe the key parameters supported by the ASP.NET server executors. + +### AspNetServerExecutor + +| Parameter | Description | Default | +|----------------------------------|-----------------------------------------------------------------------------|------------| +| PackageName | The name of the ASP.NET Benchmarks dependency package. | *required* | +| DotNetSdkPackageName | The name of the .NET SDK dependency package. | `dotnetsdk`| +| TargetFramework | The .NET target framework (e.g., `net8.0`, `net9.0`). | *required* | +| ServerPort | The port the Kestrel server listens on. | `9876` | +| AspNetCoreThreadCount | The ASPNETCORE thread count environment variable. | `1` | +| DotNetSystemNetSocketsThreadCount| The DOTNET_SYSTEM_NET_SOCKETS_THREAD_COUNT environment variable. | `1` | +| BindToCores | When `true`, the server process is pinned to specific CPU cores. | `false` | +| CoreAffinity | CPU core affinity specification (e.g., `0-7`). Required when `BindToCores` is `true`. | *none* | + +### AspNetOrchardServerExecutor + +| Parameter | Description | Default | +|--------------------|-----------------------------------------------------------------------------|------------| +| PackageName | The name of the OrchardCore dependency package. | *required* | +| DotNetSdkPackageName | The name of the .NET SDK dependency package. | `dotnetsdk`| +| TargetFramework | The .NET target framework (e.g., `net9.0`). | *required* | +| ServerPort | The port the OrchardCore server listens on. | `5014` | +| BindToCores | When `true`, the server process is pinned to specific CPU cores. | `false` | +| CoreAffinity | CPU core affinity specification (e.g., `0-7`). Required when `BindToCores` is `true`. | *none* | + +## Packaging and Setup +The following section covers how to create the custom Virtual Client dependency packages required to execute the workload +and toolset(s). This section is meant to provide guidance for users who would like to create their own packages with the +software for use with the Virtual Client. For example, users may want to bring in new versions of the software. +See the documentation on [Dependency Packages](https://microsoft.github.io/VirtualClient/docs/developing/0040-vc-packages/) +for more information on the concepts. + +### TechEmpower JSON Serialization Setup +1. Virtual Client installs the .NET SDK via the `DotNetInstallation` dependency. +2. Virtual Client clones the [ASP.NET Benchmarks](https://github.com/aspnet/Benchmarks) GitHub repository. +3. The `src/Benchmarks` project is built using `dotnet build`. +4. The server is started using `dotnet run`: + +```bash +dotnet /Benchmarks.dll \ + --nonInteractive true \ + --scenarios json \ + --urls http://*:9876 \ + --server Kestrel \ + --kestrelTransport Sockets \ + --protocol http \ + --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" \ + --header "Connection: keep-alive" +``` +5. The client tool (Bombardier or Wrk) generates HTTP load against the server: + +```bash +# Bombardier example +bombardier -d 15s -c 256 -t 2s --fasthttp --insecure -l http://:9876/json --print r --format json + +# Wrk example +wrk --latency --threads 64 --connections 4096 --duration 15s --timeout 10s http://:9876/json +``` + +### OrchardCore CMS Setup +1. Virtual Client installs the .NET SDK via the `DotNetInstallation` dependency. +2. Virtual Client clones the [OrchardCore](https://github.com/OrchardCMS/OrchardCore) GitHub repository. +3. The `OrchardCore.Cms.Web` project is published using `dotnet publish`. +4. The server is started: + +```bash +nohup /OrchardCore.Cms.Web --urls http://*:5014 +``` + +5. Wrk generates HTTP load against the `/about` endpoint: + +```bash +wrk -t 64 -c 128 -d 20s http://:5014/about +``` diff --git a/website/docs/workloads/aspnetbench/aspnetbench-profiles.md b/website/docs/workloads/aspnetbench/aspnetbench-profiles.md deleted file mode 100644 index d44fcc0010..0000000000 --- a/website/docs/workloads/aspnetbench/aspnetbench-profiles.md +++ /dev/null @@ -1,49 +0,0 @@ -# AspNetBench Workload Profiles -The following profiles run customer-representative or benchmarking scenarios using the AspNetBench workload. - -* [Workload Details](./aspnetbench.md) - -## PERF-ASPNETBENCH.json -Runs the AspNetBench benchmark workload to assess the performance of an ASP.NET Server. - -* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json) - -* **Supported Platform/Architectures** - * linux-x64 - * linux-arm64 - * win-x64 - * win-arm64 - -* **Supports Disconnected Scenarios** - * No. Internet connection required. - -* **Dependencies** - The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. - * Internet connection. - - Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: - * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) - -* **Profile Parameters** - The following parameters can be optionally supplied on the command line to modify the behaviors of the workload. - - | Parameter | Purpose | Default Value | - |---------------------------|-------------------------------------------------------------------|---------------| - | DotNetVersion | Optional. The version of the [.NET SDK to download and install](https://dotnet.microsoft.com/en-us/download/visual-studio-sdks). | 7.0.100 | - | TargetFramework | Optional. The [.NET target framework](https://learn.microsoft.com/en-us/dotnet/standard/frameworks) to run (e.g. net6.0, net7.0). | net7.0 | - -* **Profile Runtimes** - See the 'Metadata' section of the profile for estimated runtimes. These timings represent the length of time required to run a single round of profile - actions. These timings can be used to determine minimum required runtimes for the Virtual Client in order to get results. These are often estimates based on the - number of system cores. - -* **Usage Examples** - The following section provides a few basic examples of how to use the workload profile. - - ``` bash - # Execute the workload profile - VirtualClient.exe --profile=PERF-ASPNETBENCH.json --system=Demo --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" - - # Override the profile default parameters to use a different .NET SDK version - VirtualClient.exe --profile=PERF-ASPNETBENCH.json --system=Demo --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" --parameters="DotNetVersion=7.0.307" - ``` \ No newline at end of file diff --git a/website/docs/workloads/aspnetbench/aspnetbench.md b/website/docs/workloads/aspnetbench/aspnetbench.md deleted file mode 100644 index 9f89941e05..0000000000 --- a/website/docs/workloads/aspnetbench/aspnetbench.md +++ /dev/null @@ -1,56 +0,0 @@ -# AspNetBenchmark -AspNetBenchmark is a benchmark developed by MSFT ASPNET team, based on open source benchmark TechEmpower. -This workload has server and client part, on the same test machine. The server part is started as a ASPNET service. The client calls server using open source bombardier binaries. -Bombardier binaries could be downloaded from Github release, or directly compile from source using "go build ." - -* [AspNetBenchmarks Github](https://github.com/aspnet/benchmarks) -* [Bombardier Github](https://github.com/codesenberg/bombardier) -* [Bombardier Release](https://github.com/codesenberg/bombardier/releases/tag/v1.2.5) - -## Workload Metrics -The following metrics are examples of those captured by the Virtual Client when running the AspNetBenchmark workload. - -[Bombardier output example](https://github.com/codesenberg/bombardier#examples) - -The following metrics are examples of those captured during the operations of the AspNetBench workload. - -| Name | Example | Unit | Description | -|--------------------------|--------------------|-------------|----------------------------------------| -| Latency Max | 178703 | microsecond | ASP.NET Web Request latency (max) | -| Latency Average | 8270.807963429836 | microsecond | ASP.NET Web Request latency (avg) | -| Latency Stddev | 6124.356473307014 | microsecond | ASP.NET Web Request latency (standard deviation) | -| Latency P50 | 6058 | microsecond | ASP.NET Web Request latency (P50) | -| Latency P75 | 10913 | microsecond | ASP.NET Web Request latency (P75) | -| Latency P90 | 17949 | microsecond | ASP.NET Web Request latency (P90) | -| Latency P95 | 23318 | microsecond | ASP.NET Web Request latency (P95) | -| Latency P99 | 35856 | microsecond | ASP.NET Web Request latency (P99) | -| RequestPerSecond Max | 61221.282458945345 | Reqs/sec | ASP.NET Web Request per second (max) | -| RequestPerSecond Average | 31211.609987720527 | Reqs/sec | ASP.NET Web Request per second (avg) | -| RequestPerSecond Stddev | 6446.822354105378 | Reqs/sec | ASP.NET Web Request per second (standard deviation) | -| RequestPerSecond P50 | 31049.462844 | Reqs/sec | ASP.NET Web Request per second (P50) | -| RequestPerSecond P75 | 35597.436614 | Reqs/sec | ASP.NET Web Request per second (P75) | -| RequestPerSecond P90 | 39826.205746 | Reqs/sec | ASP.NET Web Request per second (P90) | -| RequestPerSecond P95 | 41662.542962 | Reqs/sec | ASP.NET Web Request per second (P95) | -| RequestPerSecond P99 | 48600.556224 | Reqs/sec | ASP.NET Web Request per second (P99) | - -## Packaging and Setup -The following section covers how to create the custom Virtual Client dependency packages required to execute the workload and toolset(s). This section -is meant to provide guidance for users that would like to create their own packages with the software for use with the Virtual Client. For example, users -may want to bring in new versions of the software. See the documentation on '[Dependency Packages](https://microsoft.github.io/VirtualClient/docs/developing/0040-vc-packages/)' -for more information on the concepts. - -1. VC installs dotnet SDK -2. VC clones AspNetBenchmarks github repo -3. dotnet build src/benchmarks project in AspNetBenchmarks repo -4. Use dotnet to start server - -``` -dotnet \Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:5000 --server Kestrel --kestrelTransport Sockets --protocol http --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" --header "Connection: keep-alive" -``` - -5. Use bombardier to start client -``` -bombardier-windows-amd64.exe -d 15s -c 256 -t 2s --fasthttp --insecure -l http://localhost:5000/json --print r --format json -``` - - diff --git a/website/docs/workloads/bombardier/bombardier.md b/website/docs/workloads/bombardier/bombardier.md new file mode 100644 index 0000000000..0b96422478 --- /dev/null +++ b/website/docs/workloads/bombardier/bombardier.md @@ -0,0 +1,90 @@ +# Bombardier HTTP Benchmarking +Bombardier is a fast, cross-platform HTTP(S) benchmarking tool written in Go. It uses the fasthttp library for high-performance +HTTP client operations and supports configurable concurrency, request duration, timeouts, and output in JSON format for automated +metric collection. + +In Virtual Client, Bombardier serves as the HTTP client load generator for ASP.NET server workloads. It runs on a dedicated +client machine (or the same machine in single-VM mode) and sends requests to the ASP.NET Kestrel server, producing latency +and throughput statistics in JSON format that are parsed into standardized metrics. + +* [Bombardier GitHub](https://github.com/codesenberg/bombardier) +* [Bombardier Releases](https://github.com/codesenberg/bombardier/releases) + +## Deployment Modes +The Bombardier executor supports two deployment modes: + +- **Multi-VM (Client-Server)** — The client and server run on separate machines connected via a layout file. + This isolates load generation from server processing for accurate benchmarking. +- **Single-VM** — When no layout file is provided, Bombardier connects to the server via the loopback address + (`127.0.0.1`). Both server and client actions run sequentially on the same machine. + +## What is Being Measured? +Bombardier generates sustained HTTP request traffic against a target server endpoint and captures the following: + +- **Latency** — average, standard deviation, maximum, and percentile distributions (P50, P75, P90, P95, P99) of response latency in microseconds. +- **Requests per second** — average, standard deviation, maximum, and percentile distributions (P50, P75, P90, P95, P99) of throughput. + +Bombardier outputs results in JSON format, which is parsed by the `BombardierMetricsParser` to extract structured metrics. + +## Workload Metrics +The following metrics are examples of those captured by the Virtual Client when running the Bombardier workload against an +ASP.NET server. + +### Latency Metrics + +| Metric Name | Example Value | Unit | Description | +|--------------------|---------------------|-------------|--------------------------------------------| +| Latency Max | 178703 | microsecond | HTTP response latency (maximum) | +| Latency Average | 8270.807963429836 | microsecond | HTTP response latency (average) | +| Latency Stddev | 6124.356473307014 | microsecond | HTTP response latency (standard deviation) | +| Latency P50 | 6058 | microsecond | HTTP response latency (50th percentile) | +| Latency P75 | 10913 | microsecond | HTTP response latency (75th percentile) | +| Latency P90 | 17949 | microsecond | HTTP response latency (90th percentile) | +| Latency P95 | 23318 | microsecond | HTTP response latency (95th percentile) | +| Latency P99 | 35856 | microsecond | HTTP response latency (99th percentile) | + +### Throughput Metrics + +| Metric Name | Example Value | Unit | Description | +|------------------------|----------------------|----------|----------------------------------------------------| +| RequestPerSecond Max | 67321.282458945348 | Reqs/sec | HTTP requests per second (maximum) | +| RequestPerSecond Average| 31211.609987720527 | Reqs/sec | HTTP requests per second (average) | +| RequestPerSecond Stddev | 6446.822354105378 | Reqs/sec | HTTP requests per second (standard deviation) | +| RequestPerSecond P50 | 31049.462844 | Reqs/sec | HTTP requests per second (50th percentile) | +| RequestPerSecond P75 | 35597.436614 | Reqs/sec | HTTP requests per second (75th percentile) | +| RequestPerSecond P90 | 39826.205746 | Reqs/sec | HTTP requests per second (90th percentile) | +| RequestPerSecond P95 | 41662.542962 | Reqs/sec | HTTP requests per second (95th percentile) | +| RequestPerSecond P99 | 49625.656227 | Reqs/sec | HTTP requests per second (99th percentile) | + +## Profiles +The following profiles use Bombardier as the client load generator. + +| Profile Name | Description | Server | Platforms | +|------------------------------|-------------------------------------------------------------------------------------------------|------------------------|----------------------------------------------| +| PERF-ASPNETBENCH.json | ASP.NET JSON serialization benchmark using Bombardier with 256 connections. | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | + +## Parameters +The following table describes the key parameters supported by the Bombardier executor. + +| Parameter | Description | Default | +|------------------|--------------------------------------------------------------------------------------------------|---------------| +| PackageName | The name of the Bombardier dependency package. | *required* | +| CommandArguments | The Bombardier command-line arguments (duration, connections, timeout, URL, output format, etc.). | *required* | +| TargetService | The target service type: `server`, `rp` (reverse-proxy), or `apigw` (API gateway). | auto-detected | +| Timeout | Maximum time to wait for server availability. | 5 minutes | +| WarmUp | When `true`, the run is a warm-up pass and metrics are not captured. | `false` | +| BindToCores | When `true`, the Bombardier process is pinned to specific CPU cores. | `false` | +| CoreAffinity | CPU core affinity specification (e.g., `8-15`). Required when `BindToCores` is `true`. | *none* | + +## Packaging and Setup +Virtual Client handles installation of Bombardier automatically through profile dependencies. The typical setup flow is: + +1. VC installs Bombardier from the registered dependency package. +2. On Unix systems, VC makes the `bombardier` binary executable. +3. VC waits for the server (e.g., ASP.NET Kestrel) to come online. +4. Bombardier is invoked with the configured parameters against the target URL. + +Example Bombardier command: +``` +bombardier -d 15s -c 256 -t 10s --fasthttp --insecure -l http://localhost:9876/json --print r --format json +``` diff --git a/website/docs/workloads/nginx/nginx.md b/website/docs/workloads/nginx/nginx.md new file mode 100644 index 0000000000..b7bd08d147 --- /dev/null +++ b/website/docs/workloads/nginx/nginx.md @@ -0,0 +1,86 @@ +# Nginx +Nginx is a high-performance open-source web server, reverse proxy, and load balancer. In Virtual Client, the Nginx workload measures web server performance by serving static content of configurable sizes over HTTPS and measuring throughput and latency under sustained load. The HTTP load is generated by the Wrk or Wrk2 client tool. + +The workload supports two deployment topologies: + +- **Client-Server (two-node)** — A dedicated client machine runs wrk/wrk2 against a dedicated server machine running Nginx. This separates resource consumption between the web server and the load generator for accurate measurements. +- **Reverse Proxy (three-node)** — A client sends requests to an Nginx reverse-proxy instance, which forwards them to a backend Nginx server. This topology is used to benchmark Nginx reverse-proxy performance. + +The `NginxServerExecutor` manages the Nginx server lifecycle on the server (and reverse-proxy) instances. It handles configuration generation, SSL setup, static content creation of configurable file sizes, and server start/stop/reset operations via the `NginxCommand` enum (`Start`, `Stop`, `GetVersion`, `GetConfig`). + +* [Nginx Official Site](https://nginx.org/en/) +* [Nginx GitHub](https://github.com/nginx/nginx) +* [Wrk GitHub](https://github.com/wg/wrk) +* [Wrk2 GitHub](https://github.com/giltene/wrk2) + +## Deployment Modes +The Nginx workload requires a **multi-VM layout** and does not support single-VM mode. This is because the +`NginxServerExecutor` depends on separate client and server instances connected via a layout file. + +- **Client-Server (two-node)** — A dedicated client machine runs wrk/wrk2, and a dedicated server machine runs Nginx. +- **Reverse Proxy (three-node)** — A client machine, a reverse-proxy machine, and a backend server machine. + +## What is Being Measured? +The Nginx workload measures the throughput and latency of an Nginx web server serving static files of a configurable size. The Wrk (or Wrk2) client tool generates concurrent HTTP/HTTPS requests across different connection counts and thread configurations to characterize server performance at varying load levels. + +The following scenarios are included in the standard profile: + +| Scenario | Connections | Description | +|---------------------------------------|-------------|-----------------------------------------------------------------------| +| Latency_100_Connections | 100 | Low-concurrency baseline measurement at full thread count. | +| Latency_1K_Connections | 1,000 | Medium-concurrency measurement at full thread count. | +| Latency_5K_Connections | 5,000 | High-concurrency measurement at full thread count. | +| Latency_10K_Connections | 10,000 | Very high-concurrency measurement at full thread count. | +| Latency_100_Connections_Thread/2 | 100 | Low-concurrency measurement at half the logical core count. | +| Latency_1K_Connections_Thread/2 | 1,000 | Medium-concurrency measurement at half the logical core count. | +| Latency_5K_Connections_Thread/2 | 5,000 | High-concurrency measurement at half the logical core count. | +| Latency_10K_Connections_Thread/2 | 10,000 | Very high-concurrency at half the logical core count. | +| Latency_100_Connections_Thread/4 | 100 | Low-concurrency measurement at one quarter of the logical core count. | +| Latency_1K_Connections_Thread/4 | 1,000 | Medium-concurrency measurement at one quarter of the logical core count. | +| Latency_5K_Connections_Thread/4 | 5,000 | High-concurrency measurement at one quarter of the logical core count. | +| Latency_10K_Connections_Thread/4 | 10,000 | Very high-concurrency at one quarter of the logical core count. | +| Latency_100_Connections_Thread/8 | 100 | Low-concurrency measurement at one eighth of the logical core count. | +| Latency_1K_Connections_Thread/8 | 1,000 | Medium-concurrency measurement at one eighth of the logical core count. | +| Latency_5K_Connections_Thread/8 | 5,000 | High-concurrency measurement at one eighth of the logical core count. | +| Latency_10K_Connections_Thread/8 | 10,000 | Very high-concurrency at one eighth of the logical core count. | + +## Workload Metrics +The following metrics are examples of those captured by the Virtual Client when running the Nginx workload. Latency values are normalized to milliseconds by the parser regardless of the unit reported by Wrk (nanoseconds, microseconds, milliseconds, or seconds). + +| Tool Name | Metric Name | Example Value | Unit | +|-----------|-------------------|---------------|---------------| +| Wrk | latency_p50 | 1.234 | milliseconds | +| Wrk | latency_p75 | 2.456 | milliseconds | +| Wrk | latency_p90 | 3.678 | milliseconds | +| Wrk | latency_p99 | 8.901 | milliseconds | +| Wrk | latency_p99_9 | 15.432 | milliseconds | +| Wrk | latency_p99_99 | 20.987 | milliseconds | +| Wrk | latency_p99_999 | 30.123 | milliseconds | +| Wrk | latency_p100 | 45.678 | milliseconds | +| Wrk | requests/sec | 25432.56 | requests/sec | +| Wrk | transfers/sec | 312.45 | megabytes/sec | + +When wrk2 is used, an additional set of uncorrected latency metrics is emitted (e.g., `uncorrected_latency_p50`, `uncorrected_latency_p99`). +See the [Wrk/Wrk2 documentation](../wrk/wrk.md) for the complete list of wrk metrics. + +## Profiles +The following profiles are available for the Nginx workload. + +| Profile Name | Description | Client Tool | Topology | Platforms | +|----------------------------|------------------------------------------------------------------------------------------|--------------|-------------------|------------------------| +| PERF-WEB-NGINX-WRK.json | Nginx web server benchmark using wrk across multiple connection and thread counts. | WrkExecutor | Client → Server | linux-x64, linux-arm64 | +| PERF-WEB-NGINX-WRK2.json | Nginx web server benchmark using wrk2 with constant request rate and corrected latency. | Wrk2Executor | Client → Server | linux-x64 | +| PERF-WEB-NGINX-WRK-RP.json | Nginx reverse-proxy benchmark using wrk across a three-node layout. | WrkExecutor | Client → RP → Server | linux-x64, linux-arm64 | +| PERF-WEB-NGINX-WRK2-RP.json | Nginx reverse-proxy benchmark using wrk2 across a three-node layout. | Wrk2Executor | Client → RP → Server | linux-x64 | + +## Server Parameters +The following table describes the key parameters supported by the `NginxServerExecutor`. + +| Parameter | Description | Default | +|-----------------|-----------------------------------------------------------------------------|-------------| +| PackageName | The name of the Nginx dependency package. | *required* | +| Role | The role of the current instance: `Server` or `ReverseProxy`. | *required* | +| Workers | Number of Nginx worker processes. Set to `0` or omit to use all cores. | `0` (auto) | +| FileSizeInKB | Size of the static file to serve (in kilobytes). | `1` | +| Timeout | Maximum time to keep the server online before resetting. | 30 minutes | +| pollingInterval | Interval between server state polling cycles. | 60 seconds | diff --git a/website/docs/workloads/wrk/wrk.md b/website/docs/workloads/wrk/wrk.md new file mode 100644 index 0000000000..50987502c6 --- /dev/null +++ b/website/docs/workloads/wrk/wrk.md @@ -0,0 +1,166 @@ +# Wrk/Wrk2 HTTP Benchmarking +Wrk is a modern HTTP benchmarking tool capable of generating significant load when run on a single multi-core CPU. It combines a multithreaded +design with scalable event notification systems such as epoll and kqueue to produce high request throughput with low resource consumption. + +Wrk2 is a variant of wrk that adds constant-throughput, correct-latency recording using a modified version of Gil Tene's wrk2 rate-limiting +approach. Unlike wrk (which measures only coordinated-omission-free throughput), wrk2 takes a target request rate and produces an +HdrHistogram-corrected latency distribution, making it suitable for latency-sensitive benchmarks. + +In Virtual Client, wrk and wrk2 serve as the HTTP client load generators for Nginx and ASP.NET web server workloads. +They run on a dedicated client machine and send requests to a server machine running the target web server. + +* [Wrk GitHub](https://github.com/wg/wrk) +* [Wrk2 GitHub](https://github.com/giltene/wrk2) +* [How NOT to Measure Latency — Gil Tene (video)](https://www.youtube.com/watch?v=lJ8ydIuPFeU) +* [On Coordinated Omission — ScyllaDB](https://www.scylladb.com/2021/04/22/on-coordinated-omission/) + +## Deployment Modes +The Wrk/Wrk2 executors support two deployment modes when used with ASP.NET server workloads: + +- **Multi-VM (Client-Server)** — The client and server run on separate machines connected via a layout file. + This isolates load generation from server processing for accurate benchmarking. +- **Single-VM** — When no layout file is provided, wrk connects to the server via the loopback address + (`127.0.0.1`). Both server and client actions run sequentially on the same machine. + +Note: Nginx workloads (`PERF-WEB-NGINX-*.json`) require a multi-VM layout and do not support single-VM mode. + +## What is Being Measured? +The Wrk/Wrk2 toolset generates sustained HTTP/HTTPS request traffic against a target web server and captures the following: + +- **Latency percentile distribution** — full HdrHistogram percentiles (P50 through P100) of response latency. Values are normalized to milliseconds. +- **Requests per second** — the aggregate throughput achieved during the test run. +- **Transfer rate** — the data transfer rate achieved during the test run (megabytes/sec). + +Wrk2 additionally captures an *uncorrected latency distribution* that records raw measured latency without accounting for coordinated omission. +Both tools optionally emit a detailed percentile spectrum for fine-grained latency analysis (controlled by the `EmitLatencySpectrum` parameter). + +## Workload Metrics +The following metrics are examples of those captured by the Virtual Client when running the Wrk workload. +Latency values are normalized to milliseconds by the parser regardless of the unit reported by wrk (nanoseconds, microseconds, milliseconds, or seconds). + +### Latency Distribution Metrics + +| Tool Name | Metric Name | Example Value | Unit | +|-----------|------------------------|---------------|--------------| +| Wrk | latency_p50 | 1.427 | milliseconds | +| Wrk | latency_p75 | 1.982 | milliseconds | +| Wrk | latency_p90 | 2.683 | milliseconds | +| Wrk | latency_p99 | 3.960 | milliseconds | +| Wrk | latency_p99_9 | 6.930 | milliseconds | +| Wrk | latency_p99_99 | 8.990 | milliseconds | +| Wrk | latency_p99_999 | 9.770 | milliseconds | +| Wrk | latency_p100 | 9.770 | milliseconds | + +When wrk2 is used, an additional set of uncorrected latency metrics is emitted: + +| Tool Name | Metric Name | Example Value | Unit | +|-----------|---------------------------------|---------------|--------------| +| Wrk2 | uncorrected_latency_p50 | 0.483 | milliseconds | +| Wrk2 | uncorrected_latency_p75 | 1.120 | milliseconds | +| Wrk2 | uncorrected_latency_p90 | 1.710 | milliseconds | +| Wrk2 | uncorrected_latency_p99 | 2.870 | milliseconds | +| Wrk2 | uncorrected_latency_p99_9 | 5.760 | milliseconds | +| Wrk2 | uncorrected_latency_p99_99 | 8.020 | milliseconds | +| Wrk2 | uncorrected_latency_p99_999 | 8.410 | milliseconds | +| Wrk2 | uncorrected_latency_p100 | 8.410 | milliseconds | + +### Throughput Metrics + +| Tool Name | Metric Name | Example Value | Unit | +|-----------|----------------|---------------|---------------| +| Wrk | requests/sec | 16305.17 | requests/sec | +| Wrk | transfers/sec | 20.01 | megabytes/sec | + +### Latency Spectrum Metrics (Optional) +When the `EmitLatencySpectrum` parameter is set to `true`, the parser emits fine-grained percentile spectrum data points. +These are useful for visualizing full HdrHistogram latency distributions. + +| Tool Name | Metric Name | Example Value | Unit | Description | +|-----------|--------------------------------|---------------|------|---------------------------------| +| Wrk | latency_spectrum_p0_000000 | 0.175 | | TotalCount:1 | +| Wrk | latency_spectrum_p0_100000 | 0.566 | | TotalCount:3954 | +| Wrk | latency_spectrum_p0_500000 | 1.427 | | TotalCount:19773 | +| Wrk | latency_spectrum_p0_900000 | 2.683 | | TotalCount:35553 | +| Wrk | latency_spectrum_p0_990000 | 3.960 | | TotalCount:39130 | +| Wrk | latency_spectrum_p0_999000 | 7.011 | | TotalCount:39462 | +| Wrk | latency_spectrum_p1_000000 | 9.767 | | TotalCount:39500 | + +### Error Metrics + +| Tool Name | Metric Name | Example Value | Unit | +|-----------|--------------------------|---------------|------| +| Wrk | Non-2xx or 3xx responses | 58902 | | + +If socket errors are detected, the parser raises a `WorkloadException` with a `Socket Error` metric. + +## Profiles +The following profiles are available for the Wrk/Wrk2 workloads. + +| Profile Name | Description | Client Tool | Server | Platforms | +|---------------------------|---------------------------------------------------------------------------------------------|-------------|----------------------------|----------------------| +| PERF-WEB-NGINX-WRK.json | Nginx web server benchmark using wrk. Tests 100 to 10K connections at multiple thread counts. | WrkExecutor | NginxServerExecutor | linux-x64, linux-arm64 | +| PERF-WEB-NGINX-WRK2.json | Nginx web server benchmark using wrk2 with constant request rate and corrected latency. | Wrk2Executor | NginxServerExecutor | linux-x64 | +| PERF-WEB-NGINX-WRK-RP.json | Nginx reverse-proxy benchmark using wrk. Uses a three-node layout (Client → Reverse Proxy → Server). | WrkExecutor | NginxServerExecutor (×2) | linux-x64, linux-arm64 | +| PERF-WEB-NGINX-WRK2-RP.json | Nginx reverse-proxy benchmark using wrk2. Uses a three-node layout (Client → Reverse Proxy → Server). | Wrk2Executor | NginxServerExecutor (×2) | linux-x64 | +| PERF-WEB-ASPNET-TEJSON-WRK.json | ASP.NET TechEmpower JSON serialization benchmark using wrk. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json | ASP.NET TechEmpower JSON serialization benchmark using wrk with CPU core affinity. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-ORCHARD-WRK.json | ASP.NET OrchardCore CMS benchmark using wrk. | WrkExecutor | AspNetOrchardServerExecutor | linux-x64, linux-arm64 | + +## Parameters +The following table describes the key parameters supported by the Wrk/Wrk2 executors. + +| Parameter | Description | Default | +|----------------------|-----------------------------------------------------------------------------------------------|-----------------| +| PackageName | The name of the wrk or wrk2 dependency package. | *required* | +| CommandArguments | The wrk/wrk2 command-line arguments (threads, connections, duration, URL, etc.). | *required* | +| TargetService | The target service type: `server`, `rp` (reverse-proxy), or `apigw` (API gateway). | auto-detected | +| TestDuration | Duration of the test run (e.g., `00:02:30`). | profile-defined | +| Timeout | Maximum time to wait for server availability. | 5 minutes | +| WarmUp | When `true`, the run is a warm-up pass and metrics are not captured. | `false` | +| EmitLatencySpectrum | When `true`, the fine-grained latency spectrum is emitted as additional metrics. | `false` | +| BindToCores | When `true`, the wrk process is pinned to specific CPU cores. | `false` | +| CoreAffinity | CPU core affinity specification (e.g., `0-3`, `0,2,4,6`). Required when `BindToCores` is `true`. | *none* | + +## Wrk Command Line Options +The following are the key command-line options for wrk and wrk2. These are referenced in the `CommandArguments` +parameter in Virtual Client profiles. + +| Option | Description | +|--------------------|--------------------------------------------------------------------------------------------------| +| `-t, --threads` | Total number of threads to use. | +| `-c, --connections` | Total number of HTTP connections to keep open (each thread handles N = connections/threads). | +| `-d, --duration` | Duration of the test (e.g., `2s`, `2m`, `2h`). | +| `-R, --rate` | Total requests per second (wrk2 only). Enables constant-throughput, corrected-latency recording. | +| `-L, --latency` | Print detailed latency statistics (HdrHistogram percentile distribution). | +| `--timeout` | Record a timeout if a response is not received within this amount of time. | + +## Translating a Profile to a Command +Virtual Client profiles define wrk parameters declaratively. The executor translates them into a wrk command line +at runtime. For example, the following profile action: + +`json +{ + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "benchmark_measurement", + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s http://{serverip}:{ServerPort}/json", + "ThreadCount": "64", + "Connection": 4096, + "TestDuration": "00:00:15", + "ServerPort": 9876, + "Role": "Client" + } +} +` + +Translates to the following wrk command (assuming server IP `10.0.0.5`): + +` +wrk --latency --threads 64 --connections 4096 --duration 15s --timeout 10s http://10.0.0.5:9876/json +` + +## Performance Notes +Generally, the more concurrent connections you configure, the higher the load on the server. At some point, increasing +connections further will result in diminishing returns in requests per second and increased latency, as the server becomes +saturated. This saturation point is useful for characterizing the maximum throughput capacity of the server under test. From fde4b630a795e6d73fe7e1e98093c2dff2bb9223 Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Thu, 9 Apr 2026 23:08:06 -0700 Subject: [PATCH 03/12] Fix wrk.md: replace single-backtick code fences with triple backticks for MDX compatibility --- website/docs/workloads/wrk/wrk.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/docs/workloads/wrk/wrk.md b/website/docs/workloads/wrk/wrk.md index 50987502c6..a3bb91f56f 100644 --- a/website/docs/workloads/wrk/wrk.md +++ b/website/docs/workloads/wrk/wrk.md @@ -138,7 +138,7 @@ parameter in Virtual Client profiles. Virtual Client profiles define wrk parameters declaratively. The executor translates them into a wrk command line at runtime. For example, the following profile action: -`json +```json { "Type": "WrkExecutor", "Parameters": { @@ -152,13 +152,13 @@ at runtime. For example, the following profile action: "Role": "Client" } } -` +``` Translates to the following wrk command (assuming server IP `10.0.0.5`): -` +``` wrk --latency --threads 64 --connections 4096 --duration 15s --timeout 10s http://10.0.0.5:9876/json -` +``` ## Performance Notes Generally, the more concurrent connections you configure, the higher the load on the server. At some point, increasing From ef367514cf8a2a80c5b1966f479e7436723210ed Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Fri, 10 Apr 2026 08:03:15 -0700 Subject: [PATCH 04/12] commit changes --- .../AspNetBench/BombardierExecutorTests.cs | 4 ++-- .../Wrk/WrkExecutorTest.cs | 12 ++++++------ .../ASPNET/AspNetOrchardServerExecutor.cs | 8 ++++---- .../ASPNET/AspNetServerExecutor.cs | 8 ++++---- .../Bombardier/BombardierExecutor.cs | 12 ++++++------ .../Nginx/NginxServerExecutor.cs | 13 ++++++++++--- .../VirtualClient.Actions/Wrk/WrkExecutor.cs | 12 ++++++------ .../VirtualClient.Actions/Wrk/WrkMetricParser.cs | 2 +- .../profiles/PERF-WEB-ASPNET-TEJSON-WRK.json | 2 +- 9 files changed, 40 insertions(+), 33 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs index c252a6b4ed..c419243d3e 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs @@ -97,9 +97,9 @@ public TestBombardierExecutor(IServiceCollection dependencies, IDictionary base.PackageDirectory = value; } - public new string GetBombardierVersion(EventContext telemetryContext, CancellationToken cancellationToken) + public new async Task GetBombardierVersionAsync(EventContext telemetryContext, CancellationToken cancellationToken) { - return base.GetBombardierVersion(telemetryContext, cancellationToken); + return await base.GetBombardierVersionAsync(telemetryContext, cancellationToken); } } } diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs index f16d69bb3d..7d272e474a 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs @@ -569,7 +569,7 @@ public async Task WrkClientExecutorExecutesAsyncAsExpected(PlatformID platform, } [Test] - public void GetWrkVersionReturnsCorrectVersion() + public async Task GetWrkVersionReturnsCorrectVersion() { this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); @@ -584,7 +584,7 @@ public void GetWrkVersionReturnsCorrectVersion() .TrackProcesses() .SetupProcessOutput("--version", wrkOutput); - string actualVersion = executor.GetWrkVersion(); + string actualVersion = await executor.GetWrkVersionAsync(); Assert.AreEqual(expectedVersion, actualVersion); this.mockFixture.Tracking.AssertCommandsExecuted(true, @@ -593,7 +593,7 @@ public void GetWrkVersionReturnsCorrectVersion() } [Test] - public void GetWrkVersion_ReturnsNull_WhenVersionCannotBeParsed() + public async Task GetWrkVersion_ReturnsNull_WhenVersionCannotBeParsed() { this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); @@ -606,7 +606,7 @@ public void GetWrkVersion_ReturnsNull_WhenVersionCannotBeParsed() .TrackProcesses() .SetupProcessOutput("--version", "Invalid output without version"); - string version = executor.GetWrkVersion(); + string version = await executor.GetWrkVersionAsync(); Assert.IsNull(version); this.mockFixture.Tracking.AssertCommandsExecuted(true, "sudo bash .* --version"); @@ -648,9 +648,9 @@ public bool GetIsServerWarmedUp() return base.IsServerWarmedUp; } - public string GetWrkVersion() + public async Task GetWrkVersionAsync() { - return base.GetWrkVersion(EventContext.None, CancellationToken.None); + return await base.GetWrkVersionAsync(EventContext.None, CancellationToken.None); } public void SetIsServerWarmedUp(bool value) diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs index 4af2eb0e6c..7003731451 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs @@ -343,13 +343,13 @@ private Task DeleteStateAsync(EventContext telemetryContext, CancellationToken c }); } - private Task KillServerInstancesAsync(EventContext telemetryContext, CancellationToken cancellationToken) + private async Task KillServerInstancesAsync(EventContext telemetryContext, CancellationToken cancellationToken) { this.Logger.LogTraceMessage($"{this.TypeName}.KillServerInstances"); - this.ExecuteCommandAsync("pkill", "OrchardCore", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); - this.ExecuteCommandAsync("fuser", $"-n tcp -k {this.ServerPort}", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); + await this.ExecuteCommandAsync("pkill", "OrchardCore", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); + await this.ExecuteCommandAsync("fuser", $"-n tcp -k {this.ServerPort}", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); - return this.WaitAsync(TimeSpan.FromSeconds(3), cancellationToken); + await this.WaitAsync(TimeSpan.FromSeconds(3), cancellationToken); } private void StartServerInstances(EventContext telemetryContext, CancellationToken cancellationToken) diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs index 25bd98a18d..067a690cc6 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs @@ -354,15 +354,15 @@ private Task DeleteStateAsync(EventContext telemetryContext, CancellationToken c }); } - private Task KillServerInstancesAsync(EventContext telemetryContext, CancellationToken cancellationToken) + private async Task KillServerInstancesAsync(EventContext telemetryContext, CancellationToken cancellationToken) { this.Logger.LogTraceMessage($"{this.TypeName}.KillServerInstances"); - this.ExecuteCommandAsync("pkill", "dotnet", this.aspnetBenchDirectory, telemetryContext, cancellationToken); + await this.ExecuteCommandAsync("pkill", "dotnet", this.aspnetBenchDirectory, telemetryContext, cancellationToken); - this.ExecuteCommandAsync("fuser", $"-n tcp -k {this.ServerPort}", this.aspnetBenchDirectory, telemetryContext, cancellationToken); + await this.ExecuteCommandAsync("fuser", $"-n tcp -k {this.ServerPort}", this.aspnetBenchDirectory, telemetryContext, cancellationToken); - return this.WaitAsync(TimeSpan.FromSeconds(3), cancellationToken); + await this.WaitAsync(TimeSpan.FromSeconds(3), cancellationToken); } private void StartServerInstances(EventContext telemetryContext, CancellationToken cancellationToken) diff --git a/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs index 942c711c68..71d13fe134 100644 --- a/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs @@ -416,7 +416,7 @@ protected async Task ExecuteWorkloadAsync(string commandArguments, string workin if (!this.WarmUp) { - this.CaptureMetrics(process, commandArguments, relatedContext, cancellationToken); + await this.CaptureMetricsAsync(process, commandArguments, relatedContext, cancellationToken).ConfigureAwait(false); } } } @@ -445,7 +445,7 @@ protected async Task ExecuteWorkloadAsync(string commandArguments, string workin /// Provides context information that will be captured with telemetry events. /// A token that can be used to cancel the operation. /// Bombardier Version - protected string GetBombardierVersion(EventContext telemetryContext, CancellationToken cancellationToken) + protected async Task GetBombardierVersionAsync(EventContext telemetryContext, CancellationToken cancellationToken) { string bombardierPath = this.Combine(this.PackageDirectory, this.Platform == PlatformID.Unix ? "bombardier" : "bombardier.exe"); string bombardierVersion = null; @@ -458,11 +458,11 @@ protected string GetBombardierVersion(EventContext telemetryContext, Cancellatio string versionPattern = @"bombardier\s+(?:version\s+)?v?(\d+\.\d+\.\d+)"; Regex versionRegex = new Regex(versionPattern, RegexOptions.Compiled | RegexOptions.IgnoreCase); - using (IProcessProxy process = this.ExecuteCommandAsync(bombardierPath, commandArguments, workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).Result) + using (IProcessProxy process = await this.ExecuteCommandAsync(bombardierPath, commandArguments, workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false)) { if (!cancellationToken.IsCancellationRequested) { - this.LogProcessDetailsAsync(process, telemetryContext, "BombardierVersion", logToFile: true).Wait(); + await this.LogProcessDetailsAsync(process, telemetryContext, "BombardierVersion", logToFile: true).ConfigureAwait(false); string output = process.StandardOutput.ToString(); Match match = versionRegex.Match(output); @@ -489,7 +489,7 @@ protected string GetBombardierVersion(EventContext telemetryContext, Cancellatio return bombardierVersion; } - private void CaptureMetrics(IProcessProxy workloadProcess, string commandArguments, EventContext context, CancellationToken cancellationToken) + private async Task CaptureMetricsAsync(IProcessProxy workloadProcess, string commandArguments, EventContext context, CancellationToken cancellationToken) { if (!cancellationToken.IsCancellationRequested) { @@ -511,7 +511,7 @@ private void CaptureMetrics(IProcessProxy workloadProcess, string commandArgumen } } - string bombardierVersion = this.GetBombardierVersion(telemetryContext, cancellationToken); + string bombardierVersion = await this.GetBombardierVersionAsync(telemetryContext, cancellationToken).ConfigureAwait(false); this.MetadataContract.AddForScenario( toolName: this.PackageName, diff --git a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs index b3b2daeb78..3e1b110de6 100644 --- a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs @@ -247,10 +247,17 @@ protected override void Dispose(bool disposing) { if (disposing) { - Task.Run((Func)(async () => + try { - await this.ResetNginxAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); - })).Wait(); + Task.Run(async () => + { + await this.ResetNginxAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + }).Wait(TimeSpan.FromSeconds(30)); + } + catch (AggregateException) + { + // Best-effort cleanup during dispose; exceptions are intentionally swallowed. + } } } diff --git a/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs index 9821657e6b..b1c546d1e8 100644 --- a/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs @@ -463,7 +463,7 @@ protected async Task ExecuteWorkloadAsync(string commandArguments, string workin if (!this.WarmUp) { - this.CaptureMetrics(process, commandArguments, this.EmitLatencySpectrum, relatedContext, cancellationToken); + await this.CaptureMetricsAsync(process, commandArguments, this.EmitLatencySpectrum, relatedContext, cancellationToken).ConfigureAwait(false); } } } @@ -492,7 +492,7 @@ protected async Task ExecuteWorkloadAsync(string commandArguments, string workin /// Provides context information that will be captured with telemetry events. /// A token that can be used to cancel the operation. /// Wrk Version - protected string GetWrkVersion(EventContext telemetryContext, CancellationToken cancellationToken) + protected async Task GetWrkVersionAsync(EventContext telemetryContext, CancellationToken cancellationToken) { string wrkVersion = null; @@ -506,11 +506,11 @@ protected string GetWrkVersion(EventContext telemetryContext, CancellationToken string versionPattern = @"wrk\s(\d+\.\d+\.\d+)"; Regex versionRegex = new Regex(versionPattern, RegexOptions.Compiled); - using (IProcessProxy process = this.ExecuteCommandAsync(command, commandArguments, workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).Result) + using (IProcessProxy process = await this.ExecuteCommandAsync(command, commandArguments, workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false)) { if (!cancellationToken.IsCancellationRequested) { - this.LogProcessDetailsAsync(process, telemetryContext, "WrkVersion", logToFile: true).Wait(); + await this.LogProcessDetailsAsync(process, telemetryContext, "WrkVersion", logToFile: true).ConfigureAwait(false); string output = process.StandardOutput.ToString(); Match match = versionRegex.Match(output); @@ -537,7 +537,7 @@ protected string GetWrkVersion(EventContext telemetryContext, CancellationToken return wrkVersion; } - private void CaptureMetrics(IProcessProxy workloadProcess, string commandArguments, bool emitLatencySpectrum, EventContext context, CancellationToken cancellationToken) + private async Task CaptureMetricsAsync(IProcessProxy workloadProcess, string commandArguments, bool emitLatencySpectrum, EventContext context, CancellationToken cancellationToken) { if (!cancellationToken.IsCancellationRequested) { @@ -559,7 +559,7 @@ private void CaptureMetrics(IProcessProxy workloadProcess, string commandArgumen } } - string wrkVersion = this.GetWrkVersion(telemetryContext, cancellationToken); + string wrkVersion = await this.GetWrkVersionAsync(telemetryContext, cancellationToken).ConfigureAwait(false); this.MetadataContract.AddForScenario( toolName: this.PackageName, diff --git a/src/VirtualClient/VirtualClient.Actions/Wrk/WrkMetricParser.cs b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkMetricParser.cs index 65367622a6..75d63fd63f 100644 --- a/src/VirtualClient/VirtualClient.Actions/Wrk/WrkMetricParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkMetricParser.cs @@ -43,7 +43,7 @@ public WrkMetricParser(string resultsText) /// public override IList Parse() { - return this.Parse(); + return this.Parse(emitLatencySpectrum: true); } /// diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK.json index 2f8850c7e9..8a65c54e1f 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK.json @@ -36,7 +36,7 @@ "PackageName": "aspnetbenchmarks", "DotNetSdkPackageName": "dotnetsdk", "TargetFramework": "$.Parameters.TargetFramework", - "Port": "$.Parameters.Port" + "ServerPort": "$.Parameters.Port" } }, { From ddfe093cc2c29e08e59aae3c5c99d23e99b4ff2c Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Fri, 10 Apr 2026 09:05:10 -0700 Subject: [PATCH 05/12] Fix LLM review findings: infinite recursion, nginx stop, fire-and-forget, async deadlock, port mismatch, dispose safety --- .../AspNetBench/BombardierExecutorTests.cs | 13 +++++++------ .../Nginx/NginxServerExecutorTest.cs | 8 ++++---- .../ASPNET/AspNetOrchardServerExecutor.cs | 12 ++++++++++-- .../ASPNET/AspNetServerExecutor.cs | 12 ++++++++++-- .../VirtualClient.Actions/Nginx/NginxExtensions.cs | 2 +- 5 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs index c419243d3e..4236e28bb6 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs @@ -6,6 +6,7 @@ namespace VirtualClient.Actions using System; using System.Collections.Generic; using System.Threading; + using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Moq; using NUnit.Framework; @@ -46,40 +47,40 @@ public void SetupDefaults() } [Test] - public void BombardierExecutorGetBombardierVersionParsesVersionWithVPrefix() + public async Task BombardierExecutorGetBombardierVersionParsesVersionWithVPrefix() { this.mockFixture.SetupProcessOutput(".*--version.*", "bombardier version v1.2.5 linux/arm64"); using (TestBombardierExecutor executor = new TestBombardierExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) { executor.PackageDirectory = this.mockPackage.Path; - string version = executor.GetBombardierVersion(EventContext.None, CancellationToken.None); + string version = await executor.GetBombardierVersionAsync(EventContext.None, CancellationToken.None); Assert.AreEqual("1.2.5", version); } } [Test] - public void BombardierExecutorGetBombardierVersionParsesVersionWithoutVPrefix() + public async Task BombardierExecutorGetBombardierVersionParsesVersionWithoutVPrefix() { this.mockFixture.SetupProcessOutput(".*--version.*", "bombardier version 1.2.5"); using (TestBombardierExecutor executor = new TestBombardierExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) { executor.PackageDirectory = this.mockPackage.Path; - string version = executor.GetBombardierVersion(EventContext.None, CancellationToken.None); + string version = await executor.GetBombardierVersionAsync(EventContext.None, CancellationToken.None); Assert.AreEqual("1.2.5", version); } } [Test] - public void BombardierExecutorGetBombardierVersionReturnsNullOnUnparsableOutput() + public async Task BombardierExecutorGetBombardierVersionReturnsNullOnUnparsableOutput() { this.mockFixture.SetupProcessOutput(".*--version.*", "unrecognized output"); using (TestBombardierExecutor executor = new TestBombardierExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) { executor.PackageDirectory = this.mockPackage.Path; - string version = executor.GetBombardierVersion(EventContext.None, CancellationToken.None); + string version = await executor.GetBombardierVersionAsync(EventContext.None, CancellationToken.None); Assert.IsNull(version); } } diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs index 33df870ace..b8fb8ebd14 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs @@ -274,7 +274,7 @@ public async Task NginxServerExecutorResetsServerAsExpected(PlatformID platform, } else { - Assert.AreEqual(arguments, "systemctl disable nginx", NginxCommand.Stop.ConvertToCommandArgs()); + Assert.AreEqual(arguments, "systemctl stop nginx", NginxCommand.Stop.ConvertToCommandArgs()); } Assert.AreEqual(command, "sudo"); @@ -328,7 +328,7 @@ public void NginxServerExecutorWillResetServerDuringDispose(PlatformID platform, } else { - Assert.AreEqual(arguments, "systemctl disable nginx", NginxCommand.Stop.ConvertToCommandArgs()); + Assert.AreEqual(arguments, "systemctl stop nginx", NginxCommand.Stop.ConvertToCommandArgs()); } Assert.AreEqual(command, "sudo"); @@ -379,7 +379,7 @@ public async Task NginxServerExecutorRunsAsExpected(PlatformID platform, Archite shellScriptCalls++; Assert.AreEqual(workingDir, packagePath); } - else if (new[] { "systemctl restart nginx", "systemctl disable nginx"}.Contains(arguments, StringComparer.OrdinalIgnoreCase)) + else if (new[] { "systemctl restart nginx", "systemctl stop nginx"}.Contains(arguments, StringComparer.OrdinalIgnoreCase)) { nginxServiceCalls++; Assert.IsNull(workingDir); @@ -445,7 +445,7 @@ public void NginxServerExecutorWillResetServerIfFailure(PlatformID platform, Arc // This will ensure nginx version is empty therefore ArgumentException will be thrown. this.memoryProcess.StandardError = new ConcurrentBuffer(new StringBuilder(string.Empty)); } - else if (arguments == "systemctl disable nginx" || arguments == "bash reset.sh") + else if (arguments == "systemctl stop nginx" || arguments == "bash reset.sh") { } else diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs index 7003731451..afd7bf268b 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs @@ -144,8 +144,16 @@ protected override void Dispose(bool disposing) { if (!this.disposed) { - this.KillServerInstancesAsync(null, CancellationToken.None) - .GetAwaiter().GetResult(); + try + { + this.KillServerInstancesAsync(EventContext.None, CancellationToken.None) + .GetAwaiter().GetResult(); + } + catch + { + // Best-effort cleanup during dispose. + } + this.disposed = true; } } diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs index 067a690cc6..a7f4661b0b 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs @@ -155,8 +155,16 @@ protected override void Dispose(bool disposing) { if (!this.disposed) { - this.KillServerInstancesAsync(null, CancellationToken.None) - .GetAwaiter().GetResult(); + try + { + this.KillServerInstancesAsync(EventContext.None, CancellationToken.None) + .GetAwaiter().GetResult(); + } + catch + { + // Best-effort cleanup during dispose. + } + this.disposed = true; } } diff --git a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs index 5981ce65f4..78e51700f4 100644 --- a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs +++ b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs @@ -50,7 +50,7 @@ public static string ConvertToCommandArgs(this NginxCommand command) return "systemctl restart nginx"; case NginxCommand.Stop: - return "systemctl disable nginx"; + return "systemctl stop nginx"; case NginxCommand.GetVersion: return "nginx -V"; From 526b24900cdd81f003d15a601fed3f8d384e20cd Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Fri, 10 Apr 2026 09:55:08 -0700 Subject: [PATCH 06/12] typo fix --- .../VirtualClient.Actions/Bombardier/BombardierExecutor.cs | 1 - src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs | 3 --- 2 files changed, 4 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs index 71d13fe134..34f92f5569 100644 --- a/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs @@ -74,7 +74,6 @@ public string TargetService case "reverse-proxy": case "rp": return "rp"; - case "apiwg": case "apigw": case "api-gateway": return "apigw"; diff --git a/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs index b1c546d1e8..246130ca7d 100644 --- a/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs @@ -78,7 +78,6 @@ public string TargetService case "reverse-proxy": case "rp": return "rp"; - case "apiwg": case "apigw": case "api-gateway": return "apigw"; @@ -436,8 +435,6 @@ protected async Task ExecuteWorkloadAsync(string commandArguments, string workin "/bin/bash", $"-c {wrappedCommand}", workingDir); - - process.RedirectStandardInput = true; } else { From 547585338815724cec6b29fdf9feb6436b75f054 Mon Sep 17 00:00:00 2001 From: Rakesh <153008248+RakeshwarK@users.noreply.github.com> Date: Fri, 10 Apr 2026 09:55:54 -0700 Subject: [PATCH 07/12] Bump version from 3.0.14 to 3.0.15 Signed-off-by: Rakesh <153008248+RakeshwarK@users.noreply.github.com> --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 513cc0b6cf..e265a8cb03 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.14 \ No newline at end of file +3.0.15 From 7d4d06b0dcb1011501f5076395ab240994acbca0 Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Fri, 10 Apr 2026 12:32:23 -0700 Subject: [PATCH 08/12] fix tests --- .../AspNetBenchProfileTests.cs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs index 20287363df..f6f3fbcff2 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs @@ -50,20 +50,6 @@ public void AspNetBenchWorkloadProfileParametersAreInlinedCorrectly(string profi } } - [Test] - [TestCase("PERF-ASPNETBENCH-AFFINITY.json")] - public void AspNetBenchAffinityWorkloadProfileParametersAreInlinedCorrectly(string profile) - { - this.mockFixture.Setup(PlatformID.Unix, agentId: this.clientAgentId).SetupLayout( - new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), - new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); - - using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) - { - WorkloadAssert.ParameterReferencesInlined(executor.Profile); - } - } - [Test] [TestCase("PERF-ASPNETBENCH.json")] public async Task AspNetBenchWorkloadProfileExecutesTheExpectedWorkloadsOnWindowsPlatform(string profile) From 2e2231959f0c45aa73405a9657ed8df01b114f93 Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Fri, 10 Apr 2026 12:40:12 -0700 Subject: [PATCH 09/12] addressing cmnts --- .../Wrk/WrkMetricsParserTest.cs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs index 648808b058..796f1494b9 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs @@ -1,16 +1,14 @@ -using System.Collections.Generic; -using System.IO; -using System.Reflection; -using VirtualClient.Contracts; -using NUnit.Framework; -using VirtualClient; -using VirtualClient.Actions; - // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. namespace VirtualClient.Actions { + using System.Collections.Generic; + using System.IO; + using System.Reflection; + using NUnit.Framework; + using VirtualClient.Contracts; + [TestFixture] [Category("Unit")] public class WrkMetricsParserTest From 95e4b7f2595ea4672c28c90a46286f8fb786cb27 Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Fri, 10 Apr 2026 18:19:58 -0700 Subject: [PATCH 10/12] rename --- .../AspNetBenchProfileTests.cs | 223 ------------------ .../profiles/PERF-ASPNETBENCH-MULTI.json | 138 ----------- .../profiles/PERF-ASPNETBENCH.json | 87 ------- .../profiles/PERF-WEB-ASPNET-BOMBARDIER.json | 87 +++++++ ...json => PERF-WEB-ASPNET-WRK-AFFINITY.json} | 0 ...JSON-WRK.json => PERF-WEB-ASPNET-WRK.json} | 0 .../aspnet-benchmarks-profiles.md | 0 .../dependencies/0040-install-git-repo.md | 2 +- .../dependencies/0051-install-dotnet-sdk.md | 2 +- .../aspnet-benchmarks-profiles.md | 55 ++++- .../aspnet-benchmarks/aspnet-benchmarks.md | 13 +- .../docs/workloads/bombardier/bombardier.md | 2 +- website/docs/workloads/wrk/wrk.md | 4 +- 13 files changed, 146 insertions(+), 467 deletions(-) delete mode 100644 src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs delete mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json delete mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json rename src/VirtualClient/VirtualClient.Main/profiles/{PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json => PERF-WEB-ASPNET-WRK-AFFINITY.json} (100%) rename src/VirtualClient/VirtualClient.Main/profiles/{PERF-WEB-ASPNET-TEJSON-WRK.json => PERF-WEB-ASPNET-WRK.json} (100%) create mode 100644 src/VirtualClient/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs deleted file mode 100644 index f6f3fbcff2..0000000000 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.Linq; - using System.Net; - using System.Threading; - using System.Threading.Tasks; - using NUnit.Framework; - using VirtualClient.Common; - using VirtualClient.Contracts; - - [TestFixture] - [Category("Functional")] - public class AspNetBenchProfileTests - { - private DependencyFixture mockFixture; - private string clientAgentId; - private string serverAgentId; - - [OneTimeSetUp] - public void SetupFixture() - { - this.clientAgentId = $"{Environment.MachineName}-Client"; - this.serverAgentId = $"{Environment.MachineName}-Server"; - - ComponentTypeCache.Instance.LoadComponentTypes(TestDependencies.TestDirectory); - } - - [SetUp] - public void Setup() - { - this.mockFixture = new DependencyFixture(); - } - - [Test] - [TestCase("PERF-ASPNETBENCH.json")] - public void AspNetBenchWorkloadProfileParametersAreInlinedCorrectly(string profile) - { - this.mockFixture.Setup(PlatformID.Unix, agentId: this.clientAgentId).SetupLayout( - new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), - new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); - - using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) - { - WorkloadAssert.ParameterReferencesInlined(executor.Profile); - } - } - - [Test] - [TestCase("PERF-ASPNETBENCH.json")] - public async Task AspNetBenchWorkloadProfileExecutesTheExpectedWorkloadsOnWindowsPlatform(string profile) - { - IEnumerable expectedCommands = this.GetProfileExpectedCommands(PlatformID.Win32NT); - this.SetupDefaultMockBehaviors(PlatformID.Win32NT); - - this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => - { - IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); - - // Add bombardier results for any bombardier execution (with or without affinity) - if (command.Contains("bombardier", StringComparison.OrdinalIgnoreCase) || - arguments.Contains("bombardier", StringComparison.OrdinalIgnoreCase)) - { - if (arguments.Contains("--version")) - { - process.StandardOutput.Append("bombardier version 1.2.5"); - } - else - { - process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_AspNetBench.txt")); - } - } - - return process; - }; - - // Setup API client for client-server communication - this.SetupApiClient(this.serverAgentId, "1.2.3.4"); - - // Execute server actions - this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.serverAgentId); - using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) - { - executor.ExecuteDependencies = false; - await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); - } - - // Execute client actions - this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.clientAgentId); - using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) - { - executor.ExecuteDependencies = false; - await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); - - WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); - } - } - - [Test] - [TestCase("PERF-ASPNETBENCH.json")] - public async Task AspNetBenchWorkloadProfileExecutesTheExpectedWorkloadsOnUnixPlatform(string profile) - { - IEnumerable expectedCommands = this.GetProfileExpectedCommands(PlatformID.Unix); - this.SetupDefaultMockBehaviors(PlatformID.Unix); - - this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => - { - IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); - - // Add bombardier results for any bombardier execution (with or without affinity) - if (command.Contains("bombardier", StringComparison.OrdinalIgnoreCase) || - arguments.Contains("bombardier", StringComparison.OrdinalIgnoreCase)) - { - if (arguments.Contains("--version")) - { - process.StandardOutput.Append("bombardier version 1.2.5"); - } - else - { - process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_AspNetBench.txt")); - } - } - - return process; - }; - - // Setup API client for client-server communication - this.SetupApiClient(this.serverAgentId, "1.2.3.4"); - - // Execute server actions - this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.serverAgentId); - using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) - { - executor.ExecuteDependencies = false; - await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); - } - - // Execute client actions - this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.clientAgentId); - using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) - { - executor.ExecuteDependencies = false; - await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); - - WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); - } - } - - private IEnumerable GetProfileExpectedCommands(PlatformID platform) - { - List commands = null; - switch (platform) - { - case PlatformID.Win32NT: - commands = new List - { - @"pkill dotnet", - @"fuser -n tcp -k 9876", - @"dotnet\.exe build -c Release -p:BenchmarksTargetFramework=net8.0", - @"dotnet\.exe .+Benchmarks\.dll --nonInteractive true --scenarios json --urls http://\*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept:.+ keep-alive", - @"bombardier\.exe --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://1\.2\.3\.4:9876/json --print r --format json" - }; - break; - - case PlatformID.Unix: - commands = new List - { - @"chmod \+x .+bombardier", - @"pkill dotnet", - @"fuser -n tcp -k 9876", - @"dotnet build -c Release -p:BenchmarksTargetFramework=net8.0", - @"dotnet .+Benchmarks\.dll --nonInteractive true --scenarios json --urls http://\*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept:.+ keep-alive", - @"bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://1\.2\.3\.4:9876/json --print r --format json" - }; - break; - } - - return commands; - } - - private void SetupDefaultMockBehaviors(PlatformID platform) - { - if (platform == PlatformID.Win32NT) - { - this.mockFixture.Setup(PlatformID.Win32NT, agentId: this.clientAgentId).SetupLayout( - new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), - new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); - - this.mockFixture.SetupPackage("aspnetbenchmarks", expectedFiles: @"aspnetbench"); - this.mockFixture.SetupPackage("bombardier", expectedFiles: @"bombardier.exe"); - this.mockFixture.SetupPackage("dotnetsdk", expectedFiles: @"dotnet.exe"); - } - else - { - this.mockFixture.Setup(PlatformID.Unix, agentId: this.clientAgentId).SetupLayout( - new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), - new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); - - this.mockFixture.SetupPackage("aspnetbenchmarks", expectedFiles: @"aspnetbench"); - this.mockFixture.SetupPackage("bombardier", expectedFiles: @"bombardier"); - this.mockFixture.SetupPackage("dotnetsdk", expectedFiles: @"dotnet"); - } - - this.mockFixture.SetupDisks(withRemoteDisks: false); - } - - private void SetupApiClient(string serverName, string serverIPAddress) - { - IPAddress.TryParse(serverIPAddress, out IPAddress ipAddress); - IApiClient apiClient = this.mockFixture.ApiClientManager.GetOrCreateApiClient(serverName, ipAddress); - - State state = new State(); - state.Online(true); - - apiClient.CreateStateAsync(nameof(State), state, CancellationToken.None) - .GetAwaiter().GetResult(); - } - } -} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json deleted file mode 100644 index ab5d00b09e..0000000000 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "Description": ".NET benchmarking Workload", - "Metadata": { - "RecommendedMinimumExecutionTime": "00:05:00", - "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", - "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" - }, - "Parameters": { - "DotNetVersion": "8.0.204", - "TargetFramework": "net8.0", - "EnvironmentVariables": "ASPNETCORE_threadCount=1;DOTNET_SYSTEM_NET_SOCKETS_THREAD_COUNT=1", - "AspNetCoreThreadCount": 1, - "DotNetSystemNetSocketsThreadCount": 1 - }, - "Actions": [ - { - "Type": "AspNetServerExecutor", - "Parameters": { - "Role": "Server", - "Scenario": "ExecuteJsonSerializationBenchmark", - "PackageName": "aspnetbenchmarks", - "DotNetSdkPackageName": "dotnetsdk", - "TargetFramework": "$.Parameters.TargetFramework", - "ServerPort": "9876", - "AspNetCoreThreadCount": "$.Parameters.AspNetCoreThreadCount", - "DotNetSystemNetSocketsThreadCount": "$.Parameters.DotNetSystemNetSocketsThreadCount" - } - }, - { - "Type": "WrkExecutor", - "Parameters": { - "Role": "Client", - "Scenario": "ExecuteJsonSerializationBenchmarkWarmUp", - "PackageName": "wrk", - "CommandArguments": "-t 256 -c 256 -d 45s --timeout 10s http://{ServerIp}:9876/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\"", - "WarmUp": true, - "Timeout": "00:05:00" - } - }, - { - "Type": "WrkExecutor", - "Parameters": { - "Role": "Client", - "Scenario": "ExecuteJsonSerializationBenchmark", - "PackageName": "wrk", - "CommandArguments": "-t 256 -c 256 -d 15s --timeout 10s http://{ServerIp}:9876/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\"", - "WarmUp": false, - "Timeout": "00:05:00" - } - } - - ], - "Dependencies": [ - { - "Type": "ChocolateyInstallation", - "Parameters": { - "Scenario": "InstallChocolatey", - "PackageName": "chocolatey" - } - }, - { - "Type": "ChocolateyPackageInstallation", - "Parameters": { - "Scenario": "InstallGit", - "PackageName": "chocolatey", - "Packages": "git" - } - }, - { - "Type": "GitRepoClone", - "Parameters": { - "Scenario": "CloneAspNetBenchmarksRepo", - "RepoUri": "https://github.com/aspnet/Benchmarks.git", - "Commit": "cf5b6ee", - "PackageName": "aspnetbenchmarks" - } - }, - { - "Type": "LinuxPackageInstallation", - "Parameters": { - "Scenario": "InstallLinuxPackages", - "Packages-Apt": "build-essential,unzip", - "Role": "Client" - } - }, - { - "Type": "DependencyPackageInstallation", - "Parameters": { - "Scenario": "InstallWrkConfiguration", - "BlobContainer": "packages", - "BlobName": "wrkconfiguration.1.0.0.zip", - "PackageName": "wrkconfiguration", - "Extract": true, - "Role": "Client" - } - }, - { - "Type": "GitRepoClone", - "Parameters": { - "Scenario": "CloneWrkRepo", - "RepoUri": "https://github.com/wg/wrk.git", - "PackageName": "wrk", - "Role": "Client" - } - }, - { - "Type": "ExecuteCommand", - "Parameters": { - "Scenario": "CompileWrk", - "Command": "make", - "WorkingDirectory": "{PackagePath:wrk}", - "Role": "Client" - } - }, - { - "Type": "DotNetInstallation", - "Parameters": { - "Scenario": "InstallDotNetSdk", - "DotNetVersion": "$.Parameters.DotNetVersion", - "PackageName": "dotnetsdk" - } - }, - { - "Type": "ApiServer", - "Parameters": { - "Scenario": "StartAPIServer" - } - }, - { - "Type": "SetEnvironmentVariable", - "Parameters": { - "Scenario": "SetEnvironmentVariableForAspNet", - "EnvironmentVariables": "$.Parameters.EnvironmentVariables", - "Role": "Server" - } - } - ] -} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json deleted file mode 100644 index d8e1606d0b..0000000000 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "Description": ".NET benchmarking Workload", - "Metadata": { - "RecommendedMinimumExecutionTime": "00:05:00", - "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", - "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" - }, - "Parameters": { - "DotNetVersion": "8.0.204", - "TargetFramework": "net8.0", - "ServerPort": 9876 - }, - "Actions": [ - { - "Type": "AspNetServerExecutor", - "Parameters": { - "Role": "Server", - "Scenario": "ExecuteJsonSerializationBenchmark", - "PackageName": "aspnetbenchmarks", - "DotNetSdkPackageName": "dotnetsdk", - "TargetFramework": "$.Parameters.TargetFramework", - "ServerPort": "$.Parameters.ServerPort" - } - }, - { - "Type": "BombardierExecutor", - "Parameters": { - "Role": "Client", - "Scenario": "ExecuteJsonSerializationBenchmark", - "PackageName": "bombardier", - "CommandArguments": "--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ServerIp}:9876/json --print r --format json", - "WarmUp": false, - "Timeout": "00:05:00" - } - } - ], - "Dependencies": [ - { - "Type": "ChocolateyInstallation", - "Parameters": { - "Scenario": "InstallChocolatey", - "PackageName": "chocolatey" - } - }, - { - "Type": "ChocolateyPackageInstallation", - "Parameters": { - "Scenario": "InstallGit", - "PackageName": "chocolatey", - "Packages": "git" - } - }, - { - "Type": "GitRepoClone", - "Parameters": { - "Scenario": "CloneAspNetBenchmarksRepo", - "RepoUri": "https://github.com/aspnet/Benchmarks.git", - "Commit": "cf5b6ee", - "PackageName": "aspnetbenchmarks" - } - }, - { - "Type": "DotNetInstallation", - "Parameters": { - "Scenario": "InstallDotNetSdk", - "DotNetVersion": "$.Parameters.DotNetVersion", - "PackageName": "dotnetsdk" - } - }, - { - "Type": "DependencyPackageInstallation", - "Parameters": { - "Scenario": "InstallBombardierPackage", - "BlobContainer": "packages", - "BlobName": "bombardier.1.2.5.zip", - "PackageName": "bombardier", - "Extract": true - } - }, - { - "Type": "ApiServer", - "Parameters": { - "Scenario": "StartAPIServer" - } - } - ] -} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json new file mode 100644 index 0000000000..63a09d94ad --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json @@ -0,0 +1,87 @@ +{ + "Description": ".NET benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:05:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" + }, + "Parameters": { + "DotNetVersion": "8.0.204", + "TargetFramework": "net8.0", + "ServerPort": 9876 + }, + "Actions": [ + { + "Type": "AspNetServerExecutor", + "Parameters": { + "Role": "Server", + "Scenario": "ExecuteJsonSerializationBenchmark", + "PackageName": "aspnetbenchmarks", + "DotNetSdkPackageName": "dotnetsdk", + "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "$.Parameters.ServerPort" + } + }, + { + "Type": "BombardierExecutor", + "Parameters": { + "Role": "Client", + "Scenario": "ExecuteJsonSerializationBenchmark", + "PackageName": "bombardier", + "CommandArguments": "--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ServerIp}:9876/json --print r --format json", + "WarmUp": false, + "Timeout": "00:05:00" + } + } + ], + "Dependencies": [ + { + "Type": "ChocolateyInstallation", + "Parameters": { + "Scenario": "InstallChocolatey", + "PackageName": "chocolatey" + } + }, + { + "Type": "ChocolateyPackageInstallation", + "Parameters": { + "Scenario": "InstallGit", + "PackageName": "chocolatey", + "Packages": "git" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneAspNetBenchmarksRepo", + "RepoUri": "https://github.com/aspnet/Benchmarks.git", + "Commit": "cf5b6ee", + "PackageName": "aspnetbenchmarks" + } + }, + { + "Type": "DotNetInstallation", + "Parameters": { + "Scenario": "InstallDotNetSdk", + "DotNetVersion": "$.Parameters.DotNetVersion", + "PackageName": "dotnetsdk" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallBombardierPackage", + "BlobContainer": "packages", + "BlobName": "bombardier.1.2.5.zip", + "PackageName": "bombardier", + "Extract": true + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-WRK-AFFINITY.json similarity index 100% rename from src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json rename to src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-WRK-AFFINITY.json diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-WRK.json similarity index 100% rename from src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK.json rename to src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-WRK.json diff --git a/src/VirtualClient/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md b/src/VirtualClient/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/website/docs/dependencies/0040-install-git-repo.md b/website/docs/dependencies/0040-install-git-repo.md index 3f2244a37c..eb841193da 100644 --- a/website/docs/dependencies/0040-install-git-repo.md +++ b/website/docs/dependencies/0040-install-git-repo.md @@ -47,7 +47,7 @@ In this example, VC clones https://github.com/eembc/coremark.git into the runtim ## Example In this example, VC installs Chocolatey, installs Git on the system and then clones https://github.com/aspnet/Benchmarks.git into the runtime packages directory -* [Profile Example with Chocolatey](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json) +* [Profile Example with Chocolatey](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json)
diff --git a/website/docs/dependencies/0051-install-dotnet-sdk.md b/website/docs/dependencies/0051-install-dotnet-sdk.md index b4d162a63e..115a386fa1 100644 --- a/website/docs/dependencies/0051-install-dotnet-sdk.md +++ b/website/docs/dependencies/0051-install-dotnet-sdk.md @@ -22,7 +22,7 @@ The following section describes the parameters used by the individual component ## Example The following section describes the parameters used by the individual component in the profile. -* [Profile Example](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json) +* [Profile Example](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json) ```json { diff --git a/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md b/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md index 0c395df72a..669fbff13c 100644 --- a/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md +++ b/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md @@ -3,11 +3,11 @@ The following profiles run customer-representative or benchmarking scenarios usi * [Workload Details](./aspnet-benchmarks.md) -## PERF-WEB-ASPNET-TEJSON-WRK.json +## PERF-WEB-ASPNET-WRK.json Runs the ASP.NET TechEmpower JSON serialization benchmark using Wrk. Includes a warm-up pass before the benchmark measurement. Supports .NET 9 and .NET 10 via the `ParametersOn` conditional parameter system. -* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK.json) +* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-WRK.json) * **Supported Platform/Architectures** * linux-x64 @@ -41,17 +41,17 @@ Supports .NET 9 and .NET 10 via the `ParametersOn` conditional parameter system. ```bash # Run with .NET 9 (default) - ./VirtualClient --profile=PERF-WEB-ASPNET-TEJSON-WRK.json --system=Demo --timeout=1440 + ./VirtualClient --profile=PERF-WEB-ASPNET-WRK.json --system=Demo --timeout=1440 # Run with .NET 10 - ./VirtualClient --profile=PERF-WEB-ASPNET-TEJSON-WRK.json --system=Demo --timeout=1440 --parameters="DotNetVersion=10" + ./VirtualClient --profile=PERF-WEB-ASPNET-WRK.json --system=Demo --timeout=1440 --parameters="DotNetVersion=10" ``` -## PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json +## PERF-WEB-ASPNET-WRK-AFFINITY.json Runs the ASP.NET TechEmpower JSON serialization benchmark using Wrk with CPU core affinity for both server and client. Includes a warm-up pass before the benchmark measurement. Supports .NET 9 and .NET 10. -* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json) +* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-WRK-AFFINITY.json) * **Supported Platform/Architectures** * linux-x64 @@ -87,12 +87,51 @@ Includes a warm-up pass before the benchmark measurement. Supports .NET 9 and .N ```bash # Run with default affinity and .NET 9 - ./VirtualClient --profile=PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json --system=Demo --timeout=1440 + ./VirtualClient --profile=PERF-WEB-ASPNET-WRK-AFFINITY.json --system=Demo --timeout=1440 # Run with .NET 10 and custom core affinity - ./VirtualClient --profile=PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json --system=Demo --timeout=1440 --parameters="DotNetVersion=10,,,ServerCoreAffinity=0-3,,,ClientCoreAffinity=4-7" + ./VirtualClient --profile=PERF-WEB-ASPNET-WRK-AFFINITY.json --system=Demo --timeout=1440 --parameters="DotNetVersion=10,,,ServerCoreAffinity=0-3,,,ClientCoreAffinity=4-7" ``` +## PERF-WEB-ASPNET-BOMBARDIER.json +Runs the ASP.NET TechEmpower JSON serialization benchmark using Bombardier as the HTTP client +with 256 concurrent connections. Uses .NET 8 SDK. + +* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json) + +* **Supported Platform/Architectures** + * linux-x64 + * linux-arm64 + * win-x64 + * win-arm64 + +* **Supports Disconnected Scenarios** + * No. Internet connection required. + +* **Dependencies** + The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. + * Internet connection. + + Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: + * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) + +* **Profile Parameters** + + | Parameter | Purpose | Default Value | + |---------------------------|-------------------------------------------------------------------|---------------| + | DotNetVersion | The version of the .NET SDK to download and install. | 8.0.204 | + | TargetFramework | The .NET target framework to run. | net8.0 | + | ServerPort | The port the Kestrel server listens on. | 9876 | + +* **Profile Runtimes** + See the 'Metadata' section of the profile for estimated runtimes. + +* **Usage Examples** + + ```bash + # Execute the workload profile + ./VirtualClient --profile=PERF-WEB-ASPNET-BOMBARDIER.json --system=Demo --timeout=1440 + ``` ## PERF-WEB-ASPNET-ORCHARD-WRK.json Runs the ASP.NET OrchardCore CMS benchmark using Wrk. The server runs the OrchardCore.Cms.Web application and Wrk benchmarks the `/about` endpoint. Includes a warm-up pass before the benchmark measurement. diff --git a/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks.md b/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks.md index 8337681595..089425f7c4 100644 --- a/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks.md +++ b/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks.md @@ -1,7 +1,7 @@ # ASP.NET Benchmarks The ASP.NET benchmarks measure the throughput and latency of ASP.NET Kestrel web applications under sustained HTTP load. The workloads use a client-server architecture where the server runs an ASP.NET application and the client generates -HTTP requests using either [Bombardier](../bombardier/bombardier.md) or [Wrk](../wrk/wrk.md). +HTTP requests using [Bombardier](../bombardier/bombardier.md) or [Wrk](../wrk/wrk.md). Two server workloads are supported: @@ -30,7 +30,7 @@ The ASP.NET benchmark workloads support two deployment modes: The client connects to the server via the loopback address (`127.0.0.1`). This mode is useful for development, validation, and quick smoke testing. -CPU core affinity profiles (e.g., `PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json`) work in +CPU core affinity profiles (e.g., `PERF-WEB-ASPNET-WRK-AFFINITY.json`) work in both modes. In single-VM mode, core affinity is especially useful to prevent the server and client from contending for the same CPU cores. @@ -50,7 +50,7 @@ pipeline including routing, middleware, and content rendering. The following metrics are examples of those captured by the Virtual Client when running the ASP.NET benchmark workloads. ### Bombardier Metrics -When Bombardier is used as the client: +When Bombardier is used as the client (e.g., `PERF-WEB-ASPNET-BOMBARDIER.json`): | Name | Example Value | Unit | Description | |------------------------|--------------------|-------------|-----------------------------------------------------| @@ -70,7 +70,7 @@ When Bombardier is used as the client: | RequestPerSecond P99 | 48600.556224 | Reqs/sec | ASP.NET Web Requests per second (P99) | ### Wrk Metrics -When Wrk is used as the client (e.g., `PERF-WEB-ASPNET-TEJSON-WRK.json`, `PERF-WEB-ASPNET-ORCHARD-WRK.json`): +When Wrk is used as the client (e.g., `PERF-WEB-ASPNET-WRK.json`, `PERF-WEB-ASPNET-ORCHARD-WRK.json`): | Name | Example Value | Unit | Description | |--------------------|---------------|---------------|-----------------------------------------------| @@ -91,8 +91,9 @@ page for per-profile parameters, dependencies, and usage examples. | Profile Name | Description | Client Tool | Server | Platforms | |------------------------------------------|------------------------------------------------------------------------------------|--------------------|-------------------------------|--------------------------------------------| -| PERF-WEB-ASPNET-TEJSON-WRK.json | TechEmpower JSON serialization benchmark using Wrk with warm-up pass. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | -| PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json | TechEmpower JSON serialization benchmark using Wrk with CPU core affinity. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-WRK.json | TechEmpower JSON serialization benchmark using Wrk with warm-up pass. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-WRK-AFFINITY.json | TechEmpower JSON serialization benchmark using Wrk with CPU core affinity. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-BOMBARDIER.json | TechEmpower JSON serialization benchmark using Bombardier with 256 connections. | BombardierExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | | PERF-WEB-ASPNET-ORCHARD-WRK.json | OrchardCore CMS benchmark using Wrk with warm-up pass. | WrkExecutor | AspNetOrchardServerExecutor | linux-x64, linux-arm64 | ## Server Parameters diff --git a/website/docs/workloads/bombardier/bombardier.md b/website/docs/workloads/bombardier/bombardier.md index 0b96422478..9c219cb721 100644 --- a/website/docs/workloads/bombardier/bombardier.md +++ b/website/docs/workloads/bombardier/bombardier.md @@ -61,7 +61,7 @@ The following profiles use Bombardier as the client load generator. | Profile Name | Description | Server | Platforms | |------------------------------|-------------------------------------------------------------------------------------------------|------------------------|----------------------------------------------| -| PERF-ASPNETBENCH.json | ASP.NET JSON serialization benchmark using Bombardier with 256 connections. | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-BOMBARDIER.json | ASP.NET JSON serialization benchmark using Bombardier with 256 connections. | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | ## Parameters The following table describes the key parameters supported by the Bombardier executor. diff --git a/website/docs/workloads/wrk/wrk.md b/website/docs/workloads/wrk/wrk.md index a3bb91f56f..e065bad146 100644 --- a/website/docs/workloads/wrk/wrk.md +++ b/website/docs/workloads/wrk/wrk.md @@ -102,8 +102,8 @@ The following profiles are available for the Wrk/Wrk2 workloads. | PERF-WEB-NGINX-WRK2.json | Nginx web server benchmark using wrk2 with constant request rate and corrected latency. | Wrk2Executor | NginxServerExecutor | linux-x64 | | PERF-WEB-NGINX-WRK-RP.json | Nginx reverse-proxy benchmark using wrk. Uses a three-node layout (Client → Reverse Proxy → Server). | WrkExecutor | NginxServerExecutor (×2) | linux-x64, linux-arm64 | | PERF-WEB-NGINX-WRK2-RP.json | Nginx reverse-proxy benchmark using wrk2. Uses a three-node layout (Client → Reverse Proxy → Server). | Wrk2Executor | NginxServerExecutor (×2) | linux-x64 | -| PERF-WEB-ASPNET-TEJSON-WRK.json | ASP.NET TechEmpower JSON serialization benchmark using wrk. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | -| PERF-WEB-ASPNET-TEJSON-WRK-AFFINITY.json | ASP.NET TechEmpower JSON serialization benchmark using wrk with CPU core affinity. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-WRK.json | ASP.NET TechEmpower JSON serialization benchmark using wrk. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-WRK-AFFINITY.json | ASP.NET TechEmpower JSON serialization benchmark using wrk with CPU core affinity. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | | PERF-WEB-ASPNET-ORCHARD-WRK.json | ASP.NET OrchardCore CMS benchmark using wrk. | WrkExecutor | AspNetOrchardServerExecutor | linux-x64, linux-arm64 | ## Parameters From 7afee9ec80253cce7797f25f764544101cf8713b Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Fri, 10 Apr 2026 18:59:56 -0700 Subject: [PATCH 11/12] up version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e265a8cb03..ec187c4425 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.15 +3.0.16 From 16ec8981f6cfc770f927842e883240c20b9b776e Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Fri, 10 Apr 2026 21:42:02 -0700 Subject: [PATCH 12/12] fix wrk2 setup --- .../NginxWrkProfileTests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs index 99900730c8..273ae3b344 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs @@ -303,6 +303,9 @@ private void SetupDefaultMockBehaviors() // Setup wrk package this.mockFixture.SetupPackage("wrk", expectedFiles: @"wrk"); + // Setup wrk2 package + this.mockFixture.SetupPackage("wrk2", expectedFiles: @"wrk2"); + this.mockFixture.SetupDisks(withRemoteDisks: false); }