diff --git a/VERSION b/VERSION index 312883d27f..ec187c4425 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.15 \ No newline at end of file +3.0.16 diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs deleted file mode 100644 index 046a97deb8..0000000000 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.Linq; - using System.Threading; - using System.Threading.Tasks; - using NUnit.Framework; - using VirtualClient.Common; - using VirtualClient.Contracts; - - [TestFixture] - [Category("Functional")] - public class AspNetBenchProfileTests - { - private DependencyFixture mockFixture; - - [OneTimeSetUp] - public void SetupFixture() - { - this.mockFixture = new DependencyFixture(); - ComponentTypeCache.Instance.LoadComponentTypes(TestDependencies.TestDirectory); - } - - [Test] - [TestCase("PERF-ASPNETBENCH.json")] - public void AspNetBenchWorkloadProfileParametersAreInlinedCorrectly(string profile) - { - this.mockFixture.Setup(PlatformID.Unix); - using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) - { - WorkloadAssert.ParameterReferencesInlined(executor.Profile); - } - } - - [Test] - [TestCase("PERF-ASPNETBENCH.json")] - public async Task AspNetBenchWorkloadProfileExecutesTheExpectedWorkloadsOnWindowsPlatform(string profile) - { - IEnumerable expectedCommands = this.GetProfileExpectedCommands(PlatformID.Win32NT); - this.SetupDefaultMockBehaviors(PlatformID.Win32NT); - // Setup the expectations for the workload - // - Workload package is installed and exists. - // - Workload binaries/executables exist on the file system. - // - The workload generates valid results. - - this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => - { - IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); - if (arguments.Contains("bombardier", StringComparison.OrdinalIgnoreCase)) - { - process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_AspNetBench.txt")); - } - - return process; - }; - - using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) - { - executor.ExecuteDependencies = false; - await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); - - WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); - } - } - - [Test] - [TestCase("PERF-ASPNETBENCH.json")] - public async Task AspNetBenchWorkloadProfileExecutesTheExpectedWorkloadsOnUnixPlatform(string profile) - { - IEnumerable expectedCommands = this.GetProfileExpectedCommands(PlatformID.Unix); - this.SetupDefaultMockBehaviors(PlatformID.Unix); - // Setup the expectations for the workload - // - Workload package is installed and exists. - // - Workload binaries/executables exist on the file system. - // - The workload generates valid results. - - this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => - { - IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); - if (arguments.Contains("bombardier", StringComparison.OrdinalIgnoreCase)) - { - process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_AspNetBench.txt")); - } - - return process; - }; - - using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) - { - executor.ExecuteDependencies = false; - await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); - - WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); - } - } - - private IEnumerable GetProfileExpectedCommands(PlatformID platform) - { - List commands = null; - switch (platform) - { - case PlatformID.Win32NT: - commands = new List - { - @"dotnet\.exe build -c Release -p:BenchmarksTargetFramework=net8.0", - @"dotnet\.exe .+Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept:.+ keep-alive", - @"bombardier\.exe --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:9876/json --print r --format json" - }; - break; - - case PlatformID.Unix: - commands = new List - { - @"chmod \+x .+bombardier", - @"dotnet build -c Release -p:BenchmarksTargetFramework=net8.0", - @"dotnet .+Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept:.+ keep-alive", - @"bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:9876/json --print r --format json" - }; - break; - } - - return commands; - } - - private void SetupDefaultMockBehaviors(PlatformID platform) - { - if (platform == PlatformID.Win32NT) - { - this.mockFixture.Setup(PlatformID.Win32NT); - this.mockFixture.SetupPackage("aspnetbenchmarks", expectedFiles: @"aspnetbench"); - this.mockFixture.SetupPackage("bombardier", expectedFiles: @"win-x64\bombardier.exe"); - this.mockFixture.SetupPackage("dotnetsdk", expectedFiles: @"packages\dotnet\dotnet.exe"); - } - else - { - this.mockFixture.Setup(PlatformID.Unix); - - this.mockFixture.SetupPackage("aspnetbenchmarks", expectedFiles: @"aspnetbench"); - this.mockFixture.SetupPackage("bombardier", expectedFiles: @"linux-x64\bombardier"); - this.mockFixture.SetupPackage("dotnetsdk", expectedFiles: @"packages\dotnet\dotnet"); - } - - this.mockFixture.SetupDisks(withRemoteDisks: false); - } - } -} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs new file mode 100644 index 0000000000..273ae3b344 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs @@ -0,0 +1,324 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Net; + using System.Threading; + using System.Threading.Tasks; + using NUnit.Framework; + using VirtualClient.Common; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Functional")] + public class NginxWrkProfileTests + { + private DependencyFixture mockFixture; + private string clientAgentId; + private string serverAgentId; + + [OneTimeSetUp] + public void SetupFixture() + { + this.clientAgentId = $"{Environment.MachineName}-Client"; + this.serverAgentId = $"{Environment.MachineName}-Server"; + + ComponentTypeCache.Instance.LoadComponentTypes(TestDependencies.TestDirectory); + } + + [SetUp] + public void Setup() + { + this.mockFixture = new DependencyFixture(); + } + + [Test] + [TestCase("PERF-WEB-NGINX-WRK.json")] + [TestCase("PERF-WEB-NGINX-WRK2.json")] + public void NginxWrkProfileParametersAreInlinedCorrectly(string profile) + { + this.mockFixture.Setup(PlatformID.Unix, agentId: this.clientAgentId).SetupLayout( + new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), + new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); + + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + WorkloadAssert.ParameterReferencesInlined(executor.Profile); + } + } + + [Test] + [TestCase("PERF-WEB-NGINX-WRK.json")] + [TestCase("PERF-WEB-NGINX-WRK2.json")] + public void NginxWrkProfileParametersAreAvailable(string profile) + { + this.mockFixture.Setup(PlatformID.Unix, agentId: this.clientAgentId).SetupLayout( + new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), + new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); + + var serverPrams = new List { "PackageName", "Role", "Timeout" }; + + var reverseProxyPrams = new List { "PackageName", "Role", "Timeout" }; + + var clientPrams = new List { "PackageName", "Role", "Timeout", "TestDuration", "FileSizeInKB", "Connection", "ThreadCount", "CommandArguments", "MetricScenario", "Scenario" }; + + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + foreach (var actionBlock in executor.Profile.Actions) + { + string role = actionBlock.Parameters["Role"].ToString(); + + if (role.Equals("server", StringComparison.OrdinalIgnoreCase)) + { + foreach (var pram in serverPrams) + { + if (!actionBlock.Parameters.ContainsKey(pram)) + { + Assert.False(true, $"{actionBlock.Type} does not have {pram} parameter."); + } + } + } + else if (role.Equals("reverseproxy", StringComparison.OrdinalIgnoreCase)) + { + foreach (var pram in reverseProxyPrams) + { + if (!actionBlock.Parameters.ContainsKey(pram)) + { + Assert.False(true, $"{actionBlock.Type} does not have {pram} parameter."); + } + } + } + else + { + foreach (var pram in clientPrams) + { + if (!actionBlock.Parameters.ContainsKey(pram)) + { + Assert.False(true, $"{actionBlock.Type} does not have {pram} parameter."); + } + } + } + } + } + } + + [Test] + [TestCase("PERF-WEB-NGINX-WRK.json")] + public async Task NginxWrkProfileExecutesTheExpectedWorkloadsOnUnixPlatform(string profile) + { + IEnumerable expectedCommands = this.GetProfileExpectedCommands(); + this.SetupDefaultMockBehaviors(); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); + + // Handle nginx version check (uses stderr) + if (arguments.Contains("nginx -V", StringComparison.OrdinalIgnoreCase)) + { + process.StandardError.Append("nginx version: nginx/1.18.0\nbuilt with OpenSSL 1.1.1f\nTLS SNI support enabled"); + } + + // Handle nginx setup scripts + if (arguments.Contains("setup-reset.sh") || arguments.Contains("setup-content.sh") || arguments.Contains("setup-config.sh") || arguments.Contains("reset.sh")) + { + process.StandardOutput.Append("Script executed successfully"); + } + + // Handle nginx start/stop commands + if (arguments.Contains("systemctl")) + { + process.StandardOutput.Append("Service command executed"); + } + + // Add wrk results for any wrk execution + if (command.Contains("wrk", StringComparison.OrdinalIgnoreCase) || + arguments.Contains("wrk", StringComparison.OrdinalIgnoreCase)) + { + if (arguments.Contains("--version")) + { + process.StandardOutput.Append("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer"); + } + else + { + process.StandardOutput.Append(TestDependencies.GetResourceFileContents("wrkStandardExample1.txt")); + } + } + + return process; + }; + + // Setup API client for client-server communication + this.SetupApiClient(this.serverAgentId, "1.2.3.4"); + + // Execute server actions + this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.serverAgentId); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + executor.ExecuteDependencies = false; + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + } + + // Execute client actions + this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.clientAgentId); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + executor.ExecuteDependencies = false; + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + + WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); + } + } + + [Test] + [TestCase("PERF-WEB-NGINX-WRK2.json")] + public async Task NginxWrk2ProfileExecutesTheExpectedWorkloadsOnUnixPlatform(string profile) + { + IEnumerable expectedCommands = this.GetProfileExpectedCommandsForWrk2(); + this.SetupDefaultMockBehaviors(); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); + + // Handle nginx version check (uses stderr) + if (arguments.Contains("nginx -V", StringComparison.OrdinalIgnoreCase)) + { + process.StandardError.Append("nginx version: nginx/1.18.0\nbuilt with OpenSSL 1.1.1f\nTLS SNI support enabled"); + } + + // Handle nginx setup scripts + if (arguments.Contains("setup-reset.sh") || arguments.Contains("setup-content.sh") || arguments.Contains("setup-config.sh") || arguments.Contains("reset.sh")) + { + process.StandardOutput.Append("Script executed successfully"); + } + + // Handle nginx start/stop commands + if (arguments.Contains("systemctl")) + { + process.StandardOutput.Append("Service command executed"); + } + + // Add wrk2 results for any wrk execution + if (command.Contains("wrk", StringComparison.OrdinalIgnoreCase) || + arguments.Contains("wrk", StringComparison.OrdinalIgnoreCase)) + { + if (arguments.Contains("--version")) + { + process.StandardOutput.Append("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer"); + } + else + { + process.StandardOutput.Append(TestDependencies.GetResourceFileContents("wrkStandardExample1.txt")); + } + } + + return process; + }; + + // Setup API client for client-server communication + this.SetupApiClient(this.serverAgentId, "1.2.3.4"); + + // Execute server actions + this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.serverAgentId); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + executor.ExecuteDependencies = false; + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + } + + // Execute client actions + this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(this.clientAgentId); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + executor.ExecuteDependencies = false; + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + + WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); + } + } + + private IEnumerable GetProfileExpectedCommands() + { + // Expected commands for PERF-WEB-NGINX-WRK.json + return new List + { + @"chmod \+x .+wrk", + @"sudo systemctl stop nginx", + @"sudo systemctl start nginx", + @"bash .+runwrk\.sh --version", + // Sample commands for various connection scenarios + @"bash .+runwrk\.sh ""--latency --threads \d+ --connections 100 --duration 150s --timeout 10s https://1\.2\.3\.4/api_new/1kb""", + @"bash .+runwrk\.sh ""--latency --threads \d+ --connections 1000 --duration 150s --timeout 10s https://1\.2\.3\.4/api_new/1kb""", + @"bash .+runwrk\.sh ""--latency --threads \d+ --connections 5000 --duration 150s --timeout 10s https://1\.2\.3\.4/api_new/1kb""", + @"bash .+runwrk\.sh ""--latency --threads \d+ --connections 10000 --duration 150s --timeout 10s https://1\.2\.3\.4/api_new/1kb""" + }; + } + + private IEnumerable GetProfileExpectedCommandsForWrk2() + { + // Expected commands for PERF-WEB-NGINX-WRK2.json + return new List + { + @"chmod \+x .+wrk", + @"sudo systemctl stop nginx", + @"sudo systemctl start nginx", + @"bash .+runwrk\.sh --version", + // Sample commands for various rate and connection scenarios + @"bash .+runwrk\.sh ""--rate 1000 --latency --threads \d+ --connections 100 --duration 150s --timeout 10s https://1\.2\.3\.4/api_new/1kb""", + @"bash .+runwrk\.sh ""--rate 1000 --latency --threads \d+ --connections 1000 --duration 150s --timeout 10s https://1\.2\.3\.4/api_new/1kb""" + }; + } + + private void SetupDefaultMockBehaviors() + { + this.mockFixture.Setup(PlatformID.Unix, agentId: this.clientAgentId).SetupLayout( + new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), + new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); + + // Setup nginx configuration package with expected files + string nginxPackagePath = this.mockFixture.GetPackagePath("nginxconfiguration"); + this.mockFixture.SetupPackage("nginxconfiguration", expectedFiles: new string[] + { + "setup-reset.sh", + "setup-config.sh", + "setup-content.sh", + "reset.sh", + "nginx.conf" + }); + + // Mock the required files exist in the filesystem + this.mockFixture.SetupFile($"{nginxPackagePath}/linux-x64/setup-reset.sh"); + this.mockFixture.SetupFile($"{nginxPackagePath}/linux-x64/setup-config.sh"); + this.mockFixture.SetupFile($"{nginxPackagePath}/linux-x64/setup-content.sh"); + this.mockFixture.SetupFile($"{nginxPackagePath}/linux-x64/reset.sh"); + + // Setup wrk configuration package + this.mockFixture.SetupPackage("wrkconfiguration", expectedFiles: @"runwrk.sh"); + + // Setup wrk package + this.mockFixture.SetupPackage("wrk", expectedFiles: @"wrk"); + + // Setup wrk2 package + this.mockFixture.SetupPackage("wrk2", expectedFiles: @"wrk2"); + + this.mockFixture.SetupDisks(withRemoteDisks: false); + } + + private void SetupApiClient(string serverName, string serverIPAddress) + { + IPAddress.TryParse(serverIPAddress, out IPAddress ipAddress); + IApiClient apiClient = this.mockFixture.ApiClientManager.GetOrCreateApiClient(serverName, ipAddress); + + State state = new State(); + state.Online(true); + + apiClient.CreateStateAsync(nameof(State), state, CancellationToken.None) + .GetAwaiter().GetResult(); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetBenchExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetBenchExecutorTests.cs deleted file mode 100644 index b95aa5a1bd..0000000000 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetBenchExecutorTests.cs +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.Diagnostics; - using System.IO; - using System.Linq; - using System.Reflection; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Extensions.DependencyInjection; - using Moq; - using NUnit.Framework; - using VirtualClient.Common; - using VirtualClient.Common.Telemetry; - using VirtualClient.Contracts; - - [TestFixture] - [Category("Unit")] - public class AspNetBenchExecutorTests : MockFixture - { - public void SetupTest(PlatformID platform) - { - if (platform == PlatformID.Win32NT) - { - this.Setup(PlatformID.Win32NT); - - DependencyPath mockAspNetBenchPackage = new DependencyPath("aspnetbenchmarks", this.PlatformSpecifics.GetPackagePath("aspnetbenchmarks")); - DependencyPath mockDotNetPackage = new DependencyPath("dotnetsdk", this.PlatformSpecifics.GetPackagePath("dotnet")); - DependencyPath mockBombardierPackage = new DependencyPath("bombardier", this.PlatformSpecifics.GetPackagePath("bombardier")); - this.PackageManager.OnGetPackage(mockAspNetBenchPackage.Name).ReturnsAsync(mockAspNetBenchPackage); - this.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); - this.PackageManager.OnGetPackage(mockBombardierPackage.Name).ReturnsAsync(mockBombardierPackage); - } - else - { - this.Setup(PlatformID.Unix); - - DependencyPath mockAspNetBenchPackage = new DependencyPath("aspnetbenchmarks", this.PlatformSpecifics.GetPackagePath("aspnetbenchmarks")); - DependencyPath mockDotNetPackage = new DependencyPath("dotnetsdk", this.PlatformSpecifics.GetPackagePath("dotnet")); - DependencyPath mockBombardierPackage = new DependencyPath("bombardier", this.PlatformSpecifics.GetPackagePath("bombardier")); - this.PackageManager.OnGetPackage(mockAspNetBenchPackage.Name).ReturnsAsync(mockAspNetBenchPackage); - this.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); - this.PackageManager.OnGetPackage(mockBombardierPackage.Name).ReturnsAsync(mockBombardierPackage); - } - - this.File.Reset(); - this.File.Setup(f => f.Exists(It.IsAny())) - .Returns(true); - this.Directory.Setup(f => f.Exists(It.IsAny())) - .Returns(true); - this.FileSystem.SetupGet(fs => fs.File).Returns(this.File.Object); - - this.Parameters = new Dictionary() - { - { nameof(AspNetBenchExecutor.PackageName), "aspnetbenchmarks" }, - { nameof(AspNetBenchExecutor.DotNetSdkPackageName), "dotnetsdk" }, - { nameof(AspNetBenchExecutor.BombardierPackageName), "bombardier" }, - { nameof(AspNetBenchExecutor.TargetFramework), "net123.321" }, - { nameof(AspNetBenchExecutor.Port), "12321" } - }; - } - - [Test] - public void AspNetBenchExecutorThrowsIfCannotFindAspNetBenchPackage() - { - this.SetupTest(PlatformID.Win32NT); - this.PackageManager.OnGetPackage("aspnetbenchmarks").ReturnsAsync(value: null); - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); - } - } - - [Test] - public void AspNetBenchExecutorThrowsIfCannotFindBombardierPackage() - { - this.SetupTest(PlatformID.Unix); - this.PackageManager.OnGetPackage("bombardier").ReturnsAsync(value: null); - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); - } - } - - [Test] - public void AspNetBenchExecutorThrowsIfCannotFindDotNetSDKPackage() - { - this.SetupTest(PlatformID.Unix); - this.PackageManager.OnGetPackage("dotnetsdk").ReturnsAsync(value: null); - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); - } - } - - [Test] - public async Task AspNetBenchExecutorRunsTheExpectedWorkloadCommandInLinux() - { - this.SetupTest(PlatformID.Unix); - - string packageDirectory = this.GetPackagePath(); - ProcessStartInfo expectedInfo = new ProcessStartInfo(); - List expectedCommands = new List() - { - $@"sudo chmod +x ""{packageDirectory}/bombardier/linux-x64/bombardier""", - $@"sudo {packageDirectory}/dotnet/dotnet build -c Release -p:BenchmarksTargetFramework=net123.321", - $@"sudo {packageDirectory}/dotnet/dotnet {packageDirectory}/aspnetbenchmarks/src/Benchmarks/bin/Release/net123.321/Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:12321 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive""", - $@"sudo {packageDirectory}/bombardier/linux-x64/bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:12321/json --print r --format json" - }; - - int commandExecuted = 0; - this.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => - { - if (expectedCommands.Any(c => c == $"{exe} {arguments}")) - { - commandExecuted++; - } - - IProcessProxy process = new InMemoryProcess() - { - ExitCode = 0, - OnStart = () => true, - OnHasExited = () => true - }; - string exampleResults = MockFixture.ReadFile(MockFixture.ExamplesDirectory, "Bombardier", "BombardierExample.txt"); - process.StandardOutput.Append(exampleResults); - return process; - }; - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); - } - - Assert.AreEqual(4, commandExecuted); - } - - [Test] - public async Task AspNetBenchExecutorRunsTheExpectedWorkloadCommandInWindows() - { - this.SetupTest(PlatformID.Win32NT); - - string packageDirectory = this.GetPackagePath(); - ProcessStartInfo expectedInfo = new ProcessStartInfo(); - - List expectedCommands = new List() - { - $@"{packageDirectory}\dotnet\dotnet.exe build -c Release -p:BenchmarksTargetFramework=net123.321", - $@"{packageDirectory}\dotnet\dotnet.exe {packageDirectory}\aspnetbenchmarks\src\Benchmarks\bin\Release\net123.321\Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:12321 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive""", - $@"{packageDirectory}\bombardier\win-x64\bombardier.exe --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:12321/json --print r --format json" - }; - - int commandExecuted = 0; - this.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => - { - if (expectedCommands.Any(c => c == $"{exe} {arguments}")) - { - commandExecuted++; - } - - IProcessProxy process = new InMemoryProcess() - { - ExitCode = 0, - OnStart = () => true, - OnHasExited = () => true - }; - string exampleResults = MockFixture.ReadFile(MockFixture.ExamplesDirectory, "Bombardier", "BombardierExample.txt"); - process.StandardOutput.Append(exampleResults); - return process; - }; - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); - } - - Assert.AreEqual(3, commandExecuted); - } - - private class TestAspNetBenchExecutor : AspNetBenchExecutor - { - public TestAspNetBenchExecutor(IServiceCollection dependencies, IDictionary parameters) - : base(dependencies, parameters) - { - } - - public new Task ExecuteAsync(EventContext context, CancellationToken cancellationToken) - { - return base.ExecuteAsync(context, cancellationToken); - } - } - } -} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs new file mode 100644 index 0000000000..0387e76c1e --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs @@ -0,0 +1,254 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + using System.Net; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class AspNetOrchardServerExecutorTests + { + private MockFixture mockFixture; + private DependencyPath mockOrchardCorePackage; + private DependencyPath mockDotNetPackage; + + [Test] + public void AspNetOrchardServerOrchardExecutorThrowsIfCannotFindAspNetOrchardPackage() + { + this.SetupDefaultMockBehaviors(PlatformID.Win32NT); + this.mockFixture.PackageManager.OnGetPackage("orchardcore").ReturnsAsync(value: null); + + using (TestAspNetOrchardServerExecutor executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void AspNetOrchardServerOrchardExecutorThrowsIfCannotFindDotNetSDKPackage() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.PackageManager.OnGetPackage("dotnetsdk").ReturnsAsync(value: null); + + using (TestAspNetOrchardServerExecutor executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public async Task AspNetOrchardServerExecutorRunsTheExpectedWorkloadCommandInLinux() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + + this.mockFixture.TrackProcesses(); + + using (var executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None); + + this.mockFixture.Tracking.AssertCommandsExecuted(true, + "pkill OrchardCore", + "fuser -n tcp -k 5014", + $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet publish -c Release --sc -f net9\\.0 {Regex.Escape(this.mockOrchardCorePackage.Path)}/src/OrchardCore\\.Cms\\.Web/OrchardCore\\.Cms\\.Web\\.csproj", + $"nohup {Regex.Escape(this.mockOrchardCorePackage.Path)}/src/OrchardCore\\.Cms\\.Web/bin/Release/net9\\.0/linux-x64/publish/OrchardCore\\.Cms\\.Web --urls http://\\*:5014" + ); + + this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 1); + } + } + + [Test] + public async Task AspNetOrchardServerExecutorRunsTheExpectedWorkloadCommandInWindows() + { + this.SetupDefaultMockBehaviors(PlatformID.Win32NT); + + this.mockFixture.TrackProcesses(); + + using (var executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None); + + this.mockFixture.Tracking.AssertCommandsExecuted(true, + "pkill OrchardCore", + "fuser -n tcp -k 5014", + $"{Regex.Escape(this.mockDotNetPackage.Path)}\\\\dotnet\\.exe publish -c Release --sc -f net9\\.0 {Regex.Escape(this.mockOrchardCorePackage.Path)}\\\\src\\\\OrchardCore\\.Cms\\.Web\\\\OrchardCore\\.Cms\\.Web\\.csproj", + $"nohup {Regex.Escape(this.mockOrchardCorePackage.Path)}\\\\src\\\\OrchardCore\\.Cms\\.Web\\\\bin\\\\Release\\\\net9\\.0\\\\win-x64\\\\publish\\\\OrchardCore\\.Cms\\.Web --urls http://\\*:5014" + ); + + this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 1); + } + } + + [Test] + public async Task AspNetServerExecutorInitializeAsyncSetsCorrectPaths() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.Layout = new EnvironmentLayout(new List + { + new ClientInstance("Server", "1.2.3.4", ClientRole.Server), + new ClientInstance("Client", "5.6.7.8", ClientRole.Client) + }); + + using (TestAspNetOrchardServerExecutor executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.InitializeAsync(EventContext.None, CancellationToken.None); + + // Verify correct paths are set + string expectedAspNetDir = this.mockFixture.Combine( + this.mockOrchardCorePackage.Path, + "src", + "OrchardCore.Cms.Web"); + + Assert.AreEqual(expectedAspNetDir, executor.AspnetOrchardDirectory); + + string expectedDotNetPath = this.mockFixture.Combine( + this.mockDotNetPackage.Path, + "dotnet"); + + Assert.AreEqual(expectedDotNetPath, executor.DotNetExePath); + + // Verify API client is initialized + Assert.IsNotNull(executor.ServerApi); + } + } + + [Test] + public void AspNetOrchardServerExecutorThrowsWhenBindToCoresIsTrueButCoreAffinityIsNotProvided() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.Parameters[nameof(AspNetOrchardServerExecutor.BindToCores)] = true; + this.mockFixture.Parameters.Remove(nameof(AspNetOrchardServerExecutor.CoreAffinity)); + + using (TestAspNetOrchardServerExecutor executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.Throws(() => executor.Validate()); + } + } + + [Test] + public async Task AspNetOrchardServerExecutorExecutesWithCoreAffinityOnLinux() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.Parameters[nameof(AspNetOrchardServerExecutor.BindToCores)] = true; + this.mockFixture.Parameters[nameof(AspNetOrchardServerExecutor.CoreAffinity)] = "0-3"; + + this.mockFixture.TrackProcesses(); + + using (var executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None); + + // Verify numactl was used with correct core affinity + // Actual format: /bin/bash -c "numactl -C 0-3 nohup ..." + this.mockFixture.Tracking.AssertCommandsExecuted(true, + "pkill OrchardCore", + "fuser -n tcp -k 5014", + $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet publish -c Release --sc -f net9\\.0 {Regex.Escape(this.mockOrchardCorePackage.Path)}/src/OrchardCore\\.Cms\\.Web/OrchardCore\\.Cms\\.Web\\.csproj", + "/bin/bash -c \\\"numactl -C 0-3 .*" + ); + } + } + + private class TestAspNetOrchardServerExecutor : AspNetOrchardServerExecutor + { + public TestAspNetOrchardServerExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + this.ServerRetryPolicy = Policy.NoOpAsync(); + } + + public string AspnetOrchardDirectory + { + get + { + var field = typeof(AspNetOrchardServerExecutor).GetField("aspnetOrchardDirectory", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public string DotNetExePath + { + get + { + var field = typeof(AspNetOrchardServerExecutor).GetField("dotnetExePath", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public new Task InitializeAsync(EventContext context, CancellationToken cancellationToken) + { + return base.InitializeAsync(context, cancellationToken); + } + + public new void Validate() + { + base.Validate(); + } + + protected override Task WaitForPortReadyAsync(EventContext context, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } + + private void SetupDefaultMockBehaviors(PlatformID platform) + { + if (platform == PlatformID.Win32NT) + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Win32NT); + this.mockOrchardCorePackage = new DependencyPath("orchardcore", this.mockFixture.PlatformSpecifics.GetPackagePath("orchardcore")); + this.mockDotNetPackage = new DependencyPath("dotnetsdk", this.mockFixture.PlatformSpecifics.GetPackagePath("dotnet")); + this.mockFixture.PackageManager.OnGetPackage(mockOrchardCorePackage.Name).ReturnsAsync(mockOrchardCorePackage); + this.mockFixture.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); + } + else + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Unix); + this.mockOrchardCorePackage = new DependencyPath("orchardcore", this.mockFixture.PlatformSpecifics.GetPackagePath("orchardcore")); + this.mockDotNetPackage = new DependencyPath("dotnetsdk", this.mockFixture.PlatformSpecifics.GetPackagePath("dotnet")); + this.mockFixture.PackageManager.OnGetPackage(mockOrchardCorePackage.Name).ReturnsAsync(mockOrchardCorePackage); + this.mockFixture.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); + } + + this.mockFixture.File.Reset(); + this.mockFixture.File.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.mockFixture.Directory.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.mockFixture.FileSystem.SetupGet(fs => fs.File).Returns(this.mockFixture.File.Object); + + this.mockFixture.Parameters = new Dictionary() + { + { nameof(AspNetOrchardServerExecutor.PackageName), "orchardcore" }, + { nameof(AspNetOrchardServerExecutor.DotNetSdkPackageName), "dotnetsdk" }, + { nameof(AspNetOrchardServerExecutor.TargetFramework), "net9.0" }, + { nameof(AspNetOrchardServerExecutor.ServerPort), "5014" } + }; + + this.mockFixture.ApiClient.OnUpdateState(nameof(State)) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK)); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs new file mode 100644 index 0000000000..2969a77e4b --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs @@ -0,0 +1,292 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Linq; + using System.Net; + using System.Reflection; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + using VirtualClient.Actions.Memtier; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class AspNetServerExecutorTests + { + private MockFixture mockFixture; + private DependencyPath mockAspNetBenchPackage; + private DependencyPath mockDotNetPackage; + + [Test] + public void AspNetServerExecutorThrowsIfCannotFindAspNetBenchPackage() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.PackageManager.OnGetPackage("aspnetbenchmarks").ReturnsAsync(value: null); + + using (TestAspNetServerExecutor executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void AspNetServerExecutorThrowsIfCannotFindDotNetSDKPackage() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.PackageManager.OnGetPackage("dotnetsdk").ReturnsAsync(value: null); + + using (TestAspNetServerExecutor executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public async Task AspNetServerExecutorRunsTheExpectedWorkloadCommandInLinux() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + + this.mockFixture + .TrackProcesses() + .SetupProcessOutput( + ".*", + File.ReadAllText(Path.Combine( + Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), + "Examples", "Bombardier", "BombardierExample.txt"))); + + using (var executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None); + + this.mockFixture.Tracking.AssertCommandsExecuted(true, + "pkill dotnet", + "fuser -n tcp -k 9876", + $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet build -c Release -p:BenchmarksTargetFramework=net8\\.0", + $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet {Regex.Escape(this.mockAspNetBenchPackage.Path)}/src/Benchmarks/bin/Release/net8\\.0/Benchmarks\\.dll --nonInteractive true --scenarios json --urls http://\\*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header \\\"Accept: application/json,text/html;q=0\\.9,application/xhtml\\+xml;q=0\\.9,application/xml;q=0\\.8,\\*/\\*;q=0\\.7\\\" --header \\\"Connection: keep-alive\\\"" + ); + + this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 1); + } + } + + [Test] + public async Task AspNetServerExecutorRunsTheExpectedWorkloadCommandInWindows() + { + this.SetupDefaultMockBehaviors(PlatformID.Win32NT); + + this.mockFixture + .TrackProcesses() + .SetupProcessOutput( + ".*", + File.ReadAllText(Path.Combine( + Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), + "Examples", "Bombardier", "BombardierExample.txt"))); + + using (var executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None); + + this.mockFixture.Tracking.AssertCommandsExecuted(true, + "pkill dotnet", + "fuser -n tcp -k 9876", + $"{Regex.Escape(this.mockDotNetPackage.Path)}\\\\dotnet\\.exe build -c Release -p:BenchmarksTargetFramework=net8\\.0", + $"{Regex.Escape(this.mockDotNetPackage.Path)}\\\\dotnet\\.exe {Regex.Escape(this.mockAspNetBenchPackage.Path)}\\\\src\\\\Benchmarks\\\\bin\\\\Release\\\\net8\\.0\\\\Benchmarks\\.dll --nonInteractive true --scenarios json --urls http://\\*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header \\\"Accept: application/json,text/html;q=0\\.9,application/xhtml\\+xml;q=0\\.9,application/xml;q=0\\.8,\\*/\\*;q=0\\.7\\\" --header \\\"Connection: keep-alive\\\"" + ); + + this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 1); + } + } + + [Test] + public async Task AspNetServerExecutorInitializeAsyncSetsCorrectPaths() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.Layout = new EnvironmentLayout(new List + { + new ClientInstance("Server", "1.2.3.4", ClientRole.Server), + new ClientInstance("Client", "5.6.7.8", ClientRole.Client) + }); + + using (TestAspNetServerExecutor executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.InitializeAsync(EventContext.None, CancellationToken.None); + + // Verify correct paths are set + string expectedAspNetDir = this.mockFixture.Combine( + this.mockAspNetBenchPackage.Path, + "src", + "Benchmarks"); + + Assert.AreEqual(expectedAspNetDir, executor.AspNetBenchDirectory); + + string expectedDotNetPath = this.mockFixture.Combine( + this.mockDotNetPackage.Path, + "dotnet"); + + Assert.AreEqual(expectedDotNetPath, executor.DotNetExePath); + + // Verify API client is initialized + Assert.IsNotNull(executor.ServerApi); + } + } + + [Test] + public void AspNetServerExecutorThrowsWhenBindToCoresIsTrueButCoreAffinityIsNotProvided() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.Parameters[nameof(AspNetServerExecutor.BindToCores)] = true; + this.mockFixture.Parameters.Remove(nameof(AspNetServerExecutor.CoreAffinity)); + + using (TestAspNetServerExecutor executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.Throws(() => executor.Validate()); + } + } + + [Test] + public async Task AspNetServerExecutorExecutesWithCoreAffinityOnLinux() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.Parameters[nameof(AspNetServerExecutor.BindToCores)] = true; + this.mockFixture.Parameters[nameof(AspNetServerExecutor.CoreAffinity)] = "0-7"; + + this.mockFixture + .TrackProcesses() + .SetupProcessOutput( + ".*", + File.ReadAllText(Path.Combine( + Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), + "Examples", "Bombardier", "BombardierExample.txt"))); + + using (var executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None); + + // Verify numactl was used with correct core affinity + // The actual command format is: /bin/bash -c "numactl -C 0-7 /path/to/dotnet /path/to/dll ..." + this.mockFixture.Tracking.AssertCommandsExecuted(true, + "pkill dotnet", + "fuser -n tcp -k 9876", + $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet build -c Release -p:BenchmarksTargetFramework=net8\\.0", + "/bin/bash -c \\\"numactl -C 0-7 .*" + ); + } + } + + private class TestAspNetServerExecutor : AspNetServerExecutor + { + public TestAspNetServerExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + this.ServerRetryPolicy = Policy.NoOpAsync(); + } + + public string AspNetBenchDirectory + { + get + { + var field = typeof(AspNetServerExecutor).GetField("aspnetBenchDirectory", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public string AspNetBenchDllPath + { + get + { + var field = typeof(AspNetServerExecutor).GetField("aspnetBenchDllPath", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public string DotNetExePath + { + get + { + var field = typeof(AspNetServerExecutor).GetField("dotnetExePath", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public new Task InitializeAsync(EventContext context, CancellationToken cancellationToken) + { + return base.InitializeAsync(context, cancellationToken); + } + + public new void Validate() + { + base.Validate(); + } + + protected override Task WaitForPortReadyAsync(EventContext context, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } + + private void SetupDefaultMockBehaviors(PlatformID platform) + { + if (platform == PlatformID.Win32NT) + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Win32NT); + this.mockAspNetBenchPackage = new DependencyPath("aspnetbenchmarks", this.mockFixture.PlatformSpecifics.GetPackagePath("aspnetbenchmarks")); + this.mockDotNetPackage = new DependencyPath("dotnetsdk", this.mockFixture.PlatformSpecifics.GetPackagePath("dotnet")); + + this.mockFixture.PackageManager.OnGetPackage(mockAspNetBenchPackage.Name).ReturnsAsync(mockAspNetBenchPackage); + this.mockFixture.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); + + this.mockFixture.ApiClient.OnUpdateState(nameof(ServerState)) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK)); + } + else + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Unix); + + this.mockAspNetBenchPackage = new DependencyPath("aspnetbenchmarks", this.mockFixture.PlatformSpecifics.GetPackagePath("aspnetbenchmarks")); + this.mockDotNetPackage = new DependencyPath("dotnetsdk", this.mockFixture.PlatformSpecifics.GetPackagePath("dotnet")); + this.mockFixture.PackageManager.OnGetPackage(mockAspNetBenchPackage.Name).ReturnsAsync(mockAspNetBenchPackage); + this.mockFixture.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); + } + + this.mockFixture.File.Reset(); + this.mockFixture.File.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.mockFixture.Directory.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.mockFixture.FileSystem.SetupGet(fs => fs.File).Returns(this.mockFixture.File.Object); + + this.mockFixture.Parameters = new Dictionary() + { + { nameof(AspNetServerExecutor.PackageName), "aspnetbenchmarks" }, + { nameof(AspNetServerExecutor.DotNetSdkPackageName), "dotnetsdk" }, + { nameof(AspNetServerExecutor.TargetFramework), "net8.0" }, + { nameof(AspNetServerExecutor.ServerPort), "9876" }, + { nameof(AspNetServerExecutor.AspNetCoreThreadCount), "1" }, + { nameof(AspNetServerExecutor.DotNetSystemNetSocketsThreadCount), "1" } + }; + + this.mockFixture.ApiClient.OnUpdateState(nameof(State)) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK)); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs new file mode 100644 index 0000000000..4236e28bb6 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierExecutorTests.cs @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class BombardierExecutorTests + { + private MockFixture mockFixture; + private DependencyPath mockPackage; + + [SetUp] + public void SetupDefaults() + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Unix); + + this.mockPackage = new DependencyPath("bombardier", this.mockFixture.GetPackagePath("bombardier")); + + this.mockFixture.File.Reset(); + this.mockFixture.File.Setup(f => f.Exists(It.IsAny())).Returns(true); + this.mockFixture.FileSystem.SetupGet(fs => fs.File).Returns(this.mockFixture.File.Object); + + this.mockFixture.Layout = new EnvironmentLayout(new List + { + new ClientInstance($"{Environment.MachineName}", "1.2.3.4", ClientRole.Client), + new ClientInstance($"{Environment.MachineName}-Server", "1.2.3.5", ClientRole.Server) + }); + + this.mockFixture.Parameters = new Dictionary + { + { nameof(BombardierExecutor.PackageName), "bombardier" }, + { nameof(BombardierExecutor.Scenario), "Bombardier_Benchmark" }, + { nameof(BombardierExecutor.CommandArguments), "--connections 200 --duration 15s http://{ServerIp}:9090/json" } + }; + } + + [Test] + public async Task BombardierExecutorGetBombardierVersionParsesVersionWithVPrefix() + { + this.mockFixture.SetupProcessOutput(".*--version.*", "bombardier version v1.2.5 linux/arm64"); + + using (TestBombardierExecutor executor = new TestBombardierExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + executor.PackageDirectory = this.mockPackage.Path; + string version = await executor.GetBombardierVersionAsync(EventContext.None, CancellationToken.None); + Assert.AreEqual("1.2.5", version); + } + } + + [Test] + public async Task BombardierExecutorGetBombardierVersionParsesVersionWithoutVPrefix() + { + this.mockFixture.SetupProcessOutput(".*--version.*", "bombardier version 1.2.5"); + + using (TestBombardierExecutor executor = new TestBombardierExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + executor.PackageDirectory = this.mockPackage.Path; + string version = await executor.GetBombardierVersionAsync(EventContext.None, CancellationToken.None); + Assert.AreEqual("1.2.5", version); + } + } + + [Test] + public async Task BombardierExecutorGetBombardierVersionReturnsNullOnUnparsableOutput() + { + this.mockFixture.SetupProcessOutput(".*--version.*", "unrecognized output"); + + using (TestBombardierExecutor executor = new TestBombardierExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + executor.PackageDirectory = this.mockPackage.Path; + string version = await executor.GetBombardierVersionAsync(EventContext.None, CancellationToken.None); + Assert.IsNull(version); + } + } + + private class TestBombardierExecutor : BombardierExecutor + { + public TestBombardierExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + } + + public new string PackageDirectory + { + get => base.PackageDirectory; + set => base.PackageDirectory = value; + } + + public new async Task GetBombardierVersionAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return await base.GetBombardierVersionAsync(telemetryContext, cancellationToken); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Nginx/NginxVersionExample.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Nginx/NginxVersionExample.txt new file mode 100644 index 0000000000..adb96f528d --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Nginx/NginxVersionExample.txt @@ -0,0 +1,4 @@ +nginx version: nginx/1.18.0 (Ubuntu) +built with OpenSSL 1.1.1f 31 Mar 2020 +TLS SNI support enabled +configure arguments: --with-cc-opt='-g -O2 -fdebug-prefix-map=/build/nginx-lUTckl/nginx-1.18.0=. -fstack-protector-strong -Wformat -Werror=format-security -fPIC -Wdate-time -D_FORTIFY_SOURCE=2' --with-ld-opt='-Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now -fPIC' --prefix=/usr/share/nginx --conf-path=/etc/nginx/nginx.conf --http-log-path=/var/log/nginx/access.log --error-log-path=/var/log/nginx/error.log --lock-path=/var/lock/nginx.lock --pid-path=/run/nginx.pid --modules-path=/usr/lib/nginx/modules --http-client-body-temp-path=/var/lib/nginx/body --http-fastcgi-temp-path=/var/lib/nginx/fastcgi --http-proxy-temp-path=/var/lib/nginx/proxy --http-scgi-temp-path=/var/lib/nginx/scgi --http-uwsgi-temp-path=/var/lib/nginx/uwsgi --with-debug --with-compat --with-pcre-jit --with-http_ssl_module --with-http_stub_status_module --with-http_realip_module --with-http_auth_request_module --with-http_v2_module --with-http_dav_module --with-http_slice_module --with-threads --with-http_addition_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_image_filter_module=dynamic --with-http_sub_module --with-http_xslt_module=dynamic --with-stream=dynamic --with-stream_ssl_module --with-mail=dynamic --with-mail_ssl_module \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample1.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample1.txt new file mode 100644 index 0000000000..c21ca9cf78 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample1.txt @@ -0,0 +1,10 @@ +Running 2m test @ http://10.1.0.15/api_new/10kb + 1 threads and 10000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 102.45ms 17.10ms 134.53ms 76.19% + Req/Sec 0.67 1.15 2.00 66.67% + 21 requests in 2.50m, 6.69KB read + Socket errors: connect 0, read 1645610, write 16, timeout 0 + Non-2xx or 3xx responses: 21 +Requests/sec: 0.14 +Transfer/sec: 45.63B \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample2.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample2.txt new file mode 100644 index 0000000000..f46572ad5c --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample2.txt @@ -0,0 +1,14 @@ +Running 30s test @ http://10.1.0.13/index.html + 5 threads and 100 connections + Thread calibration: mean lat.: 1.019ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.009ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.988ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.963ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.958ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.98ms 468.49us 8.97ms 70.07% + Req/Sec 420.73 49.56 0.89k 84.45% + 59956 requests in 30.00s, 18.64MB read + Non-2xx or 3xx responses: 59956 +Requests/sec: 1998.42 +Transfer/sec: 636.13KB \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample1.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample1.txt new file mode 100644 index 0000000000..c3c069347b --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample1.txt @@ -0,0 +1,96 @@ +Running 1m test @ https://10.1.0.26/api_new/1kb + 2 threads and 20 connections + Thread calibration: mean lat.: 4197.437ms, rate sampling interval: 15204ms + Thread calibration: mean lat.: 4197.977ms, rate sampling interval: 15147ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 29.27s 12.10s 50.20s 57.59% + Req/Sec 8.17k 163.44 8.32k 66.67% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 29.56s + 75.000% 39.75s + 90.000% 45.97s + 99.000% 49.74s + 99.900% 50.17s + 99.990% 50.23s + 99.999% 50.23s +100.000% 50.23s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 8372.223 0.000000 2 1.00 + 12525.567 0.100000 81614 1.11 + 16605.183 0.200000 163195 1.25 + 20660.223 0.300000 244674 1.43 + 24805.375 0.400000 326214 1.67 + 29556.735 0.500000 407688 2.00 + 31604.735 0.550000 448517 2.22 + 33619.967 0.600000 489638 2.50 + 35618.815 0.650000 530556 2.86 + 37683.199 0.700000 570904 3.33 + 39747.583 0.750000 611770 4.00 + 40796.159 0.775000 632420 4.44 + 41811.967 0.800000 652450 5.00 + 42827.775 0.825000 672721 5.71 + 43876.351 0.850000 693461 6.67 + 44892.159 0.875000 713577 8.00 + 45449.215 0.887500 723987 8.89 + 45973.503 0.900000 734157 10.00 + 46497.791 0.912500 744472 11.43 + 47054.847 0.925000 754722 13.33 + 47579.135 0.937500 764883 16.00 + 47841.279 0.943750 769668 17.78 + 48103.423 0.950000 774858 20.00 + 48365.567 0.956250 780158 22.86 + 48627.711 0.962500 785303 26.67 + 48857.087 0.968750 789786 32.00 + 48988.159 0.971875 792512 35.56 + 49119.231 0.975000 795257 40.00 + 49250.303 0.978125 797991 45.71 + 49381.375 0.981250 800387 53.33 + 49512.447 0.984375 802822 64.00 + 49577.983 0.985938 804076 71.11 + 49643.519 0.987500 805311 80.00 + 49709.055 0.989062 806567 91.43 + 49774.591 0.990625 807793 106.67 + 49840.127 0.992188 809006 128.00 + 49872.895 0.992969 809631 142.22 + 49905.663 0.993750 810256 160.00 + 49938.431 0.994531 810831 182.86 + 50003.967 0.995313 811970 213.33 + 50036.735 0.996094 812575 256.00 + 50036.735 0.996484 812575 284.44 + 50069.503 0.996875 813155 320.00 + 50069.503 0.997266 813155 365.71 + 50102.271 0.997656 813642 426.67 + 50135.039 0.998047 814120 512.00 + 50135.039 0.998242 814120 568.89 + 50135.039 0.998437 814120 640.00 + 50167.807 0.998633 814706 731.43 + 50167.807 0.998828 814706 853.33 + 50167.807 0.999023 814706 1024.00 + 50167.807 0.999121 814706 1137.78 + 50167.807 0.999219 814706 1280.00 + 50167.807 0.999316 814706 1462.86 + 50200.575 0.999414 815155 1706.67 + 50200.575 0.999512 815155 2048.00 + 50200.575 0.999561 815155 2275.56 + 50200.575 0.999609 815155 2560.00 + 50200.575 0.999658 815155 2925.71 + 50200.575 0.999707 815155 3413.33 + 50200.575 0.999756 815155 4096.00 + 50200.575 0.999780 815155 4551.11 + 50200.575 0.999805 815155 5120.00 + 50200.575 0.999829 815155 5851.43 + 50200.575 0.999854 815155 6826.67 + 50200.575 0.999878 815155 8192.00 + 50200.575 0.999890 815155 9102.22 + 50233.343 0.999902 815242 10240.00 + 50233.343 1.000000 815242 inf +#[Mean = 29266.261, StdDeviation = 12102.812] +#[Max = 50200.576, Total count = 815242] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 978315 requests in 1.00m, 1.17GB read +Requests/sec: 16305.17 +Transfer/sec: 20.01MB \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample2.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample2.txt new file mode 100644 index 0000000000..d6a5c1349d --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample2.txt @@ -0,0 +1,204 @@ +Running 30s test @ http://10.1.0.13/index.html + 2 threads and 100 connections + Thread calibration: mean lat.: 1.137ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.015ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.54ms 0.86ms 9.76ms 70.18% + Req/Sec 1.05k 1.39k 5.55k 89.70% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.43ms + 75.000% 1.98ms + 90.000% 2.68ms + 99.000% 3.96ms + 99.900% 6.93ms + 99.990% 8.99ms + 99.999% 9.77ms +100.000% 9.77ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.175 0.000000 1 1.00 + 0.566 0.100000 3954 1.11 + 0.776 0.200000 7913 1.25 + 1.003 0.300000 11866 1.43 + 1.226 0.400000 15800 1.67 + 1.427 0.500000 19773 2.00 + 1.517 0.550000 21738 2.22 + 1.610 0.600000 23711 2.50 + 1.715 0.650000 25678 2.86 + 1.837 0.700000 27663 3.33 + 1.982 0.750000 29634 4.00 + 2.067 0.775000 30635 4.44 + 2.163 0.800000 31614 5.00 + 2.261 0.825000 32598 5.71 + 2.381 0.850000 33586 6.67 + 2.519 0.875000 34567 8.00 + 2.601 0.887500 35060 8.89 + 2.683 0.900000 35553 10.00 + 2.769 0.912500 36051 11.43 + 2.867 0.925000 36538 13.33 + 2.981 0.937500 37038 16.00 + 3.047 0.943750 37281 17.78 + 3.105 0.950000 37534 20.00 + 3.163 0.956250 37773 22.86 + 3.235 0.962500 38022 26.67 + 3.327 0.968750 38271 32.00 + 3.375 0.971875 38392 35.56 + 3.435 0.975000 38515 40.00 + 3.507 0.978125 38636 45.71 + 3.589 0.981250 38765 53.33 + 3.689 0.984375 38886 64.00 + 3.749 0.985938 38946 71.11 + 3.825 0.987500 39008 80.00 + 3.911 0.989062 39068 91.43 + 4.011 0.990625 39130 106.67 + 4.167 0.992188 39193 128.00 + 4.271 0.992969 39223 142.22 + 4.383 0.993750 39255 160.00 + 4.535 0.994531 39284 182.86 + 4.719 0.995313 39315 213.33 + 4.967 0.996094 39346 256.00 + 5.087 0.996484 39364 284.44 + 5.383 0.996875 39377 320.00 + 5.623 0.997266 39392 365.71 + 5.851 0.997656 39408 426.67 + 6.087 0.998047 39423 512.00 + 6.195 0.998242 39431 568.89 + 6.331 0.998437 39439 640.00 + 6.543 0.998633 39446 731.43 + 6.663 0.998828 39454 853.33 + 7.011 0.999023 39462 1024.00 + 7.351 0.999121 39466 1137.78 + 7.511 0.999219 39470 1280.00 + 7.603 0.999316 39473 1462.86 + 7.687 0.999414 39477 1706.67 + 7.951 0.999512 39481 2048.00 + 8.099 0.999561 39484 2275.56 + 8.115 0.999609 39485 2560.00 + 8.207 0.999658 39487 2925.71 + 8.335 0.999707 39489 3413.33 + 8.599 0.999756 39491 4096.00 + 8.631 0.999780 39492 4551.11 + 8.783 0.999805 39493 5120.00 + 8.927 0.999829 39494 5851.43 + 8.983 0.999854 39495 6826.67 + 8.991 0.999878 39496 8192.00 + 8.991 0.999890 39496 9102.22 + 9.231 0.999902 39497 10240.00 + 9.231 0.999915 39497 11702.86 + 9.271 0.999927 39498 13653.33 + 9.271 0.999939 39498 16384.00 + 9.271 0.999945 39498 18204.44 + 9.503 0.999951 39499 20480.00 + 9.503 0.999957 39499 23405.71 + 9.503 0.999963 39499 27306.67 + 9.503 0.999969 39499 32768.00 + 9.503 0.999973 39499 36408.89 + 9.767 0.999976 39500 40960.00 + 9.767 1.000000 39500 inf +#[Mean = 1.537, StdDeviation = 0.861] +#[Max = 9.760, Total count = 39500] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + + Latency Distribution (HdrHistogram - Uncorrected Latency (measured without taking delayed starts into account)) + 50.000% 483.00us + 75.000% 1.12ms + 90.000% 1.71ms + 99.000% 2.87ms + 99.900% 5.76ms + 99.990% 8.02ms + 99.999% 8.41ms +100.000% 8.41ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.135 0.000000 1 1.00 + 0.242 0.100000 4008 1.11 + 0.287 0.200000 7981 1.25 + 0.335 0.300000 11912 1.43 + 0.392 0.400000 15829 1.67 + 0.483 0.500000 19763 2.00 + 0.552 0.550000 21732 2.22 + 0.652 0.600000 23706 2.50 + 0.802 0.650000 25676 2.86 + 0.959 0.700000 27650 3.33 + 1.115 0.750000 29633 4.00 + 1.200 0.775000 30626 4.44 + 1.285 0.800000 31612 5.00 + 1.375 0.825000 32597 5.71 + 1.475 0.850000 33586 6.67 + 1.586 0.875000 34567 8.00 + 1.645 0.887500 35064 8.89 + 1.713 0.900000 35550 10.00 + 1.788 0.912500 36050 11.43 + 1.872 0.925000 36541 13.33 + 1.964 0.937500 37032 16.00 + 2.018 0.943750 37282 17.78 + 2.081 0.950000 37532 20.00 + 2.147 0.956250 37779 22.86 + 2.219 0.962500 38023 26.67 + 2.301 0.968750 38268 32.00 + 2.353 0.971875 38395 35.56 + 2.397 0.975000 38514 40.00 + 2.459 0.978125 38638 45.71 + 2.531 0.981250 38761 53.33 + 2.625 0.984375 38888 64.00 + 2.679 0.985938 38945 71.11 + 2.741 0.987500 39007 80.00 + 2.821 0.989062 39068 91.43 + 2.907 0.990625 39130 106.67 + 3.029 0.992188 39193 128.00 + 3.119 0.992969 39223 142.22 + 3.209 0.993750 39254 160.00 + 3.371 0.994531 39284 182.86 + 3.583 0.995313 39315 213.33 + 3.793 0.996094 39346 256.00 + 3.943 0.996484 39363 284.44 + 4.175 0.996875 39377 320.00 + 4.423 0.997266 39392 365.71 + 4.703 0.997656 39408 426.67 + 5.007 0.998047 39423 512.00 + 5.163 0.998242 39431 568.89 + 5.223 0.998437 39439 640.00 + 5.355 0.998633 39446 731.43 + 5.651 0.998828 39454 853.33 + 5.783 0.999023 39462 1024.00 + 5.887 0.999121 39466 1137.78 + 5.971 0.999219 39470 1280.00 + 6.075 0.999316 39473 1462.86 + 6.147 0.999414 39477 1706.67 + 6.715 0.999512 39481 2048.00 + 6.963 0.999561 39483 2275.56 + 7.059 0.999609 39485 2560.00 + 7.195 0.999658 39487 2925.71 + 7.295 0.999707 39489 3413.33 + 7.423 0.999756 39491 4096.00 + 7.571 0.999780 39492 4551.11 + 7.687 0.999805 39493 5120.00 + 7.739 0.999829 39494 5851.43 + 7.823 0.999854 39495 6826.67 + 8.015 0.999878 39496 8192.00 + 8.015 0.999890 39496 9102.22 + 8.163 0.999902 39497 10240.00 + 8.163 0.999915 39497 11702.86 + 8.223 0.999927 39498 13653.33 + 8.223 0.999939 39498 16384.00 + 8.223 0.999945 39498 18204.44 + 8.399 0.999951 39499 20480.00 + 8.399 0.999957 39499 23405.71 + 8.399 0.999963 39499 27306.67 + 8.399 0.999969 39499 32768.00 + 8.399 0.999973 39499 36408.89 + 8.407 0.999976 39500 40960.00 + 8.407 1.000000 39500 inf +#[Mean = 0.782, StdDeviation = 0.678] +#[Max = 8.400, Total count = 39500] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 58902 requests in 30.00s, 18.31MB read + Non-2xx or 3xx responses: 58902 +Requests/sec: 1963.39 +Transfer/sec: 624.98KB \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample3.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample3.txt new file mode 100644 index 0000000000..28ff69c259 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample3.txt @@ -0,0 +1,14 @@ +Running 2m test @ http://10.9.0.7/api_new/10kb + 32 threads and 5000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 4.12ms 2.38ms 219.68ms 73.08% + Req/Sec 38.62k 3.74k 82.02k 72.05% + Latency Distribution + 50% 3.85ms + 75% 5.05ms + 90% 7.27ms + 99% 11.16ms + 184596465 requests in 2.50m, 56.05GB read + Non-2xx or 3xx responses: 184596465 +Requests/sec: 1229838.61 +Transfer/sec: 382.35MB \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs new file mode 100644 index 0000000000..b8fb8ebd14 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs @@ -0,0 +1,542 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net.Http; + using System.Reflection; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Common; + using VirtualClient.Contracts; + using Microsoft.CodeAnalysis; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Moq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Telemetry; + using Architecture = System.Runtime.InteropServices.Architecture; + + [TestFixture] + [NUnit.Framework.Category("Unit")] + public class NginxServerExecutorTest + { + private MockFixture mockFixture; + private InMemoryProcess memoryProcess; + private Item serverState; + private TimeSpan timeout = TimeSpan.FromMinutes(10); + private string packageName = "nginxconfiguration"; + + [SetUp] + public void SetupTests() + { + this.serverState = new Item(nameof(State), new State()); + this.mockFixture = new MockFixture(); + this.memoryProcess = new InMemoryProcess + { + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true, + StandardError = new ConcurrentBuffer(new StringBuilder($"nginx version: v1\n")) + }; + } + + [Test] + [TestCase(PlatformID.Win32NT, Architecture.X64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public void NginxServerExecutorThrowsErrorIfPlatformIsWrong(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters.Add(nameof(this.packageName), this.packageName); + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void NginxServerExecutorThrowsErrorPackageIsMissing(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxServerExecutorInitializesAsExpected(PlatformID platform, Architecture architecture) + { + /* + When Nginx Server Initialize, these are the expectations: + 1.Set up Api Client for client and server. + 2.Verify packages are installed and shell scripts exist inside. + 3.Set up reset(execute: "setup - reset.sh") - only applicable for first time + a.This will ensure, server's sysctl configuration is saved so when virtual client exits, it is able to reset the server back to its original state. + 4. Set up content. (execute: "setup-content.sh FileSizeInKB") + a.This will create a file that can be used during testing. + 5. Set up config (execute: "setup-config.sh) + a.This will make changes to server configuration. + 6. Delete any states that is saved in Server. + */ + + this.mockFixture.Setup(platform, architecture); + this.mockFixture.Parameters.Add(nameof(this.packageName), this.packageName); + this.mockFixture.Parameters.Add("FileSizeInKb", 5); + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + + // pkg setup + DependencyFixture fixture = new DependencyFixture(); + fixture.Setup(platform, architecture); + fixture.SetupPackage(this.packageName); + string packagePath = executor.PlatformSpecifics.ToPlatformSpecificPath(fixture.PackageManager.FirstOrDefault(), platform, architecture).Path; + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(y => y == this.packageName), It.IsAny())) + .ReturnsAsync(fixture.PackageManager.FirstOrDefault()); + + string resetFilePath = executor.PlatformSpecifics.Combine(packagePath, "reset.sh"); + string resetOutput = Guid.NewGuid().ToString(); + int processCount = 0; + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (processCount == 0) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-reset.sh"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(resetOutput)); + } + else if (processCount == 1) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-content.sh 5"); + } + else if (processCount == 2) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-config.sh auto Client 1.2.3.5"); + } + else + { + Assert.Fail("Only 3 process expected."); + } + + processCount++; + return this.memoryProcess; + }; + + string[] expectedFiles = new string[] + { + executor.PlatformSpecifics.Combine(packagePath, "setup-reset.sh"), + executor.PlatformSpecifics.Combine(packagePath, "setup-content.sh"), + executor.PlatformSpecifics.Combine(packagePath,"setup-config.sh") + }; + + this.mockFixture.FileSystem.Setup(x => x.File.Exists(It.Is(x => x == resetFilePath))).Returns(false); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.Is(x => x != resetFilePath))) + .Returns(true) + .Callback((string fileName) => + { + if (!expectedFiles.Any(y => y == fileName)) + { + Assert.Fail($"Unexpected File Name: {fileName}. \n{string.Join("\n", expectedFiles)}"); + } + }); + + // Create file for reset + Mock mockFileStream = new Mock(); + this.mockFixture.FileStream.Setup(f => f.New(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(mockFileStream.Object) + .Callback((string path, FileMode mode, FileAccess access, FileShare share) => + { + Assert.AreEqual(resetFilePath, path); + Assert.IsTrue(mode == FileMode.Create); + Assert.IsTrue(access == FileAccess.ReadWrite); + Assert.IsTrue(share == FileShare.ReadWrite); + }); + + mockFileStream + .Setup(x => x.Write(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((byte[] data, int offset, int count) => + { + byte[] byteData = Encoding.Default.GetBytes(resetOutput); + Assert.AreEqual(offset, 0); + Assert.AreEqual(count, byteData.Length); + Assert.AreEqual(data, byteData); + }); + + + + await executor.InitializeAsync().ConfigureAwait(false); + Assert.AreEqual(processCount, 3); + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(4)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxServerExecutorInitializesAsExpectedForSecondTime(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters.Add(nameof(this.packageName), this.packageName); + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + DependencyFixture fixture = new DependencyFixture(); + fixture.Setup(platform, architecture); + fixture.SetupPackage(this.packageName); + string packagePath = executor.PlatformSpecifics.ToPlatformSpecificPath(fixture.PackageManager.FirstOrDefault(), platform, architecture).Path; + + int processCount = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (processCount == 0) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-content.sh 1"); + } + else if (processCount == 1) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-config.sh auto Client 1.2.3.5"); + } + else + { + Assert.Fail("Only 2 process expected."); + } + + processCount++; + return this.memoryProcess; + }; + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(y => y == this.packageName), It.IsAny())) + .ReturnsAsync(fixture.PackageManager.FirstOrDefault()); + + var expectedFiles = new string[] + { + executor.PlatformSpecifics.Combine(packagePath, "setup-reset.sh"), + executor.PlatformSpecifics.Combine(packagePath, "setup-content.sh"), + executor.PlatformSpecifics.Combine(packagePath,"setup-config.sh"), + executor.PlatformSpecifics.Combine(packagePath, "reset.sh") + }; + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string fileName) => + { + if(!expectedFiles.Any(y => y == fileName)) + { + Assert.Fail($"Unexpected File Name: {fileName}. \n{string.Join("\n", expectedFiles)}"); + } + }); + + await executor.InitializeAsync().ConfigureAwait(false); + Assert.AreEqual(processCount, 2); + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(4)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxServerExecutorResetsServerAsExpected(PlatformID platform, Architecture architecture) + { + TimeSpan timeout = TimeSpan.FromMinutes(5); + this.mockFixture.Setup(platform, architecture, nameof(State)); + int processCount = 0; + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (processCount == 0) + { + // During every reset, nginx will delete existing content. + // "delete-content.sh" is downloaded from blob. + Assert.AreEqual(arguments, "bash reset.sh"); + } + else + { + Assert.AreEqual(arguments, "systemctl stop nginx", NginxCommand.Stop.ConvertToCommandArgs()); + } + + Assert.AreEqual(command, "sudo"); + Assert.IsNull(workingDir); + processCount++; + return this.memoryProcess; + }; + + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + await executor.ResetNginxAsync().ConfigureAwait(false); + Assert.AreEqual(processCount, 2); + + this.mockFixture.ApiClient.Verify(x => x.UpdateStateAsync( + It.Is(x => x == nameof(State)), + It.Is>(x => x.Definition.Online(null) == false), + It.IsAny(), + It.IsAny>()), + Times.Exactly(1)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void ResetServerWillSwallowExceptions(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => { throw new SchemaException(); }; + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Assert.DoesNotThrowAsync(async () => await executor.ResetNginxAsync().ConfigureAwait(false)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void NginxServerExecutorWillResetServerDuringDispose(PlatformID platform, Architecture architecture) + { + // Dispose will call reset nginx + TimeSpan timeout = TimeSpan.FromMinutes(5); + this.mockFixture.Setup(platform, architecture, nameof(State)); + int processCount = 0; + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (processCount == 0) + { + // During every reset, nginx will delete existing content. + // "delete-content.sh" is downloaded from blob. + Assert.AreEqual(arguments, "bash reset.sh"); + } + else + { + Assert.AreEqual(arguments, "systemctl stop nginx", NginxCommand.Stop.ConvertToCommandArgs()); + } + + Assert.AreEqual(command, "sudo"); + Assert.IsNull(workingDir); + processCount++; + return this.memoryProcess; + }; + + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + executor.Dispose(true); + Assert.AreEqual(processCount, 2); + this.mockFixture.ApiClient.Verify(x => x.UpdateStateAsync( + It.Is(x => x == nameof(State)), + It.Is>(x => x.Definition.Online(null) == false), + It.IsAny(), + It.IsAny>()), + Times.Exactly(1)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxServerExecutorRunsAsExpected(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters.Add(nameof(this.packageName), this.packageName); + this.mockFixture.Parameters.Add(nameof(this.timeout), this.timeout.ToString()); + this.mockFixture.Parameters.Add("pollingInterval", TimeSpan.FromSeconds(1).ToString()); + + int nginxServiceCalls = 0; + int shellScriptCalls = 0; + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + DependencyFixture fixture = new DependencyFixture(); + fixture.Setup(platform, architecture); + fixture.SetupPackage(packageName); + + string packagePath = executor.PlatformSpecifics.ToPlatformSpecificPath(fixture.PackageManager.FirstOrDefault(), platform, architecture).Path; + this.mockFixture.PackageManager.Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())).ReturnsAsync(fixture.PackageManager.FirstOrDefault()); + this.mockFixture.FileSystem.Setup(x => x.File.Exists(It.IsAny())).Returns(true); + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.AreEqual(command, "sudo"); + + if (new[] {"bash setup-content.sh 1", "bash setup-config.sh", "bash setup-config.sh auto Client 1.2.3.5", "bash reset.sh" }.Contains(arguments, StringComparer.OrdinalIgnoreCase)) + { + shellScriptCalls++; + Assert.AreEqual(workingDir, packagePath); + } + else if (new[] { "systemctl restart nginx", "systemctl stop nginx"}.Contains(arguments, StringComparer.OrdinalIgnoreCase)) + { + nginxServiceCalls++; + Assert.IsNull(workingDir); + } + else if (arguments == "nginx -V") + { + nginxServiceCalls++; + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Nginx"); + string outputPath = Path.Combine(examplesDirectory, @"NginxVersionExample.txt"); + string rawText = File.ReadAllText(outputPath); + this.memoryProcess.StandardError = new ConcurrentBuffer(new StringBuilder(rawText)); + } + else + { + Assert.Fail($"Unexpected Arguments: {arguments}"); + } + + return this.memoryProcess; + }; + + + Item onlineClientState = new Item(nameof(State), new State()); + onlineClientState.Definition.Online(true); + + Item expiredState = new Item(nameof(State), new State()); + expiredState.Definition.Timeout(DateTime.UtcNow.AddDays(-1)); + + // Set up for polling online client state. + this.mockFixture.ApiClient.SetupSequence(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK, onlineClientState)) // Polling for online state + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK, expiredState)); // Get client state + + this.mockFixture.ApiClient.Setup(x => x.CreateStateAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + + this.mockFixture.ApiClient.Setup(x => x.UpdateStateAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + + await executor.InitializeAsync().ConfigureAwait(false); + await executor.ExecuteAsync().ConfigureAwait(false); + + Assert.AreEqual(shellScriptCalls, 3); + Assert.AreEqual(nginxServiceCalls, 3); + + // nginx version and local online state + this.mockFixture.ApiClient.Verify(x => x.CreateStateAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>()), Times.Exactly(2)); + + // first loop and reset + this.mockFixture.ApiClient.Verify(x => x.UpdateStateAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny>()), Times.Exactly(2)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void NginxServerExecutorWillResetServerIfFailure(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + int processCount = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (arguments == "nginx -V") + { + // This will ensure nginx version is empty therefore ArgumentException will be thrown. + this.memoryProcess.StandardError = new ConcurrentBuffer(new StringBuilder(string.Empty)); + } + else if (arguments == "systemctl stop nginx" || arguments == "bash reset.sh") + { + } + else + { + Assert.Fail(); + } + + processCount++; + return this.memoryProcess; + }; + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => await executor.ExecuteAsync().ConfigureAwait(false)); + Assert.AreEqual(processCount, 3); + + // Expected to delete local server state & will create new offline state. + this.mockFixture.ApiClient.Verify(x => x.UpdateStateAsync( + It.Is(x => x == nameof(State)), + It.Is>(x => x.Definition.Online(null) == false), + It.IsAny(), + It.IsAny>()), + Times.Exactly(1)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxExecutorParsesNginxVersion(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Nginx"); + string outputPath = Path.Combine(examplesDirectory, @"NginxVersionExample.txt"); + string rawText = File.ReadAllText(outputPath); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (arguments == "nginx -V") + { + // This will ensure nginx version is empty therefore ArgumentException will be thrown. + this.memoryProcess.StandardError = new ConcurrentBuffer(new StringBuilder(rawText)); + } + else + { + Assert.Fail(); + } + + return this.memoryProcess; + }; + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Dictionary version = await executor.GetNginxVersionAsync().ConfigureAwait(false); + + Assert.AreEqual(version["nginxVersion"], "nginx/1.18.0 (Ubuntu)"); + Assert.IsTrue(version["sslVersion"].Contains("1.1.1f", StringComparison.OrdinalIgnoreCase)); + Assert.AreEqual(version["serverNameIndicationSupport"], "TLS SNI support enabled"); + Assert.IsTrue(version["arguments"].StartsWith("--with-cc-opt='-g -O2")); + } + + private class TestNginxServerExecutor : NginxServerExecutor + { + public TestNginxServerExecutor(MockFixture mockFixture, IDictionary parameters = null) + : base(mockFixture.Dependencies, mockFixture.Parameters) + { + this.ServerApi = mockFixture.ApiClient.Object; + this.ClientApi = mockFixture.ApiClient.Object; + } + + public new void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async Task InitializeAsync() + { + await base.InitializeAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ExecuteAsync() + { + await base.ExecuteAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ResetNginxAsync() + { + await base.ResetNginxAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task> GetNginxVersionAsync() + { + return await base.GetNginxVersionAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/VirtualClient.Actions.UnitTests.csproj b/src/VirtualClient/VirtualClient.Actions.UnitTests/VirtualClient.Actions.UnitTests.csproj index c41956fee4..4edb5fd60e 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/VirtualClient.Actions.UnitTests.csproj +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/VirtualClient.Actions.UnitTests.csproj @@ -101,6 +101,7 @@ + diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/Wrk2ExecutorTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/Wrk2ExecutorTest.cs new file mode 100644 index 0000000000..ce9e42d36a --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/Wrk2ExecutorTest.cs @@ -0,0 +1,532 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Reflection; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Common; + using VirtualClient.Contracts; + using Microsoft.CodeAnalysis; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Moq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Telemetry; + using Architecture = System.Runtime.InteropServices.Architecture; + + [TestFixture] + [Category("Unit")] + public class Wrk2ExecutorTests + { + private string ClientStateId = nameof(ClientStateId); + private string ServerStateId = nameof(ServerStateId); + + private MockFixture mockFixture; + private DependencyFixture dependencyFixture; + private InMemoryProcess memoryProcess; + private Dictionary defaultProperties; + private string packageName = "wrk2"; + private string scriptpackageName = "wrkconfiguration"; + + [SetUp] + public void SetupTests() + { + this.mockFixture = new MockFixture(); + this.memoryProcess = new InMemoryProcess + { + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + + this.defaultProperties = new Dictionary() + { + { "PackageName", this.packageName }, + { "Scenario", "1000r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb" }, + { "CommandArguments", "--latency --threads{ThreadCount} --connections{Connection} --duration{Duration.TotalSeconds}s" }, + { "Connection", 100 }, + { "ThreadCount", 10 }, + { "MaxCoreCount", 10 }, + { "TestDuration", "00:02:30"}, + { "Timeout", "00:20:00"}, + { "FileSizeInKB", 10}, + { "Role", "Client"}, + { "Tags", "Networking,NGINX,WRK2"}, + }; + + dependencyFixture = new DependencyFixture(); + } + + [Test] + [TestCase(PlatformID.Win32NT, Architecture.X64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public void Wrk2ExecutorThrowsErrorIfPlatformIsWrong(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ClientStateId); + this.mockFixture.Parameters = this.defaultProperties; + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + Assert.IsFalse(VirtualClientComponent.IsSupported(executor)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void Wrk2ExecutorThrowsErrorIfPackageIsMissing(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ClientStateId); + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + + this.mockFixture.Parameters = this.defaultProperties; + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.packageName); + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.FirstOrDefault()); + + TestWrk2Executor executor2 = new TestWrk2Executor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor2.InitializeAsync().ConfigureAwait(false); + }); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void Wrk2ExecutorOnlySupportsWrk2(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ClientStateId); + this.mockFixture.Parameters = new Dictionary() + { + { "PackageName", "wrk" }, + }; + + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage("wrk"); + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.FirstOrDefault()); + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + DependencyException exc = Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + + Assert.AreEqual(exc.Message, "TestWrk2Executor did not find correct package in the directory. Supported Package: wrk2. Package Provided: wrk"); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task Wrk2ExecutorInitializesAsExpected(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ServerStateId); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.packageName); + dependencyFixture.SetupPackage(this.scriptpackageName); + TestWrk2Executor dummyExecutor = new TestWrk2Executor(this.mockFixture); + + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.packageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptpackageName).FirstOrDefault(), platform, architecture).Path; + string[] expectedFiles = new string[] + { + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath, "setup-reset.sh"), + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath,"setup-config.sh"), + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath,"reset.sh"), + dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath,"wrk"), + dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath,"runwrk.sh") + }; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string fileName) => + { + if (!expectedFiles.Any(y => y == fileName)) + { + Assert.Fail($"Unexpected File Name: {fileName}. \n{string.Join("\n", expectedFiles)}"); + } + }); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptpackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptpackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.packageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.packageName).FirstOrDefault()); + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(expectedFiles.Count())); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + [Ignore("Unit test is way too complex and needs to be refactored.")] + public async Task Wrk2ExecutorExecutesAsyncAsExpected(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ServerStateId); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.packageName); + dependencyFixture.SetupPackage(this.scriptpackageName); + TestWrk2Executor dummyExecutor = new TestWrk2Executor(this.mockFixture); + + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.packageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptpackageName).FirstOrDefault(), platform, architecture).Path; + string[] expectedFiles = new string[] + { + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath, "setup-reset.sh"), + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath,"setup-config.sh"), + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath,"reset.sh"), + dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath,"wrk"), + dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath,"runwrk.sh") + }; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string fileName) => + { + if (!expectedFiles.Any(y => y == fileName)) + { + Assert.Fail($"Unexpected File Name: {fileName}. \n{string.Join("\n", expectedFiles)}"); + } + }); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptpackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptpackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.packageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.packageName).FirstOrDefault()); + + Item serverState = new Item(this.ServerStateId, new State()); + serverState.Definition.Online(true); + Item clientState = new Item(this.ClientStateId, new State()); + + // create local state + this.mockFixture.ApiClient + .Setup(x => x.CreateStateAsync(It.Is(y => y == this.ClientStateId), It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)) + .Callback((string id, State state, CancellationToken _, IAsyncPolicy __) => + { + Assert.IsTrue(state.Online()); + }); + + this.mockFixture.ApiClient.Setup(s => s.GetStateAsync(It.Is(x => x == this.ServerStateId), It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => + { + Item result = new Item(this.ServerStateId, new State()); + result.Definition.Online(true); + return this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, result); + }); + + this.mockFixture.ApiClient.Setup(s => s.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => + { + // both ServerStateId and serverVersion will be covered with this set up. + Item result = new Item(this.ServerStateId, new State()); + result.Definition.Online(true); + return this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, result); + }); + + // update local state before running working + this.mockFixture.ApiClient + .Setup(x => x.UpdateStateAsync(It.Is(y => y == this.ClientStateId), It.IsAny>(), It.IsAny(), It.IsAny>())) + .Callback((string id, Item state, CancellationToken _, IAsyncPolicy __) => + { + Assert.IsTrue(state.Definition.Online()); + }); + + // delete local state before exiting + this.mockFixture.ApiClient.Setup(x => x.DeleteStateAsync(It.Is(y => y == this.ClientStateId), It.IsAny(), It.IsAny>())); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.IsTrue(command == "sudo" || command.EndsWith("wrk")); + + if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")} --version")); + } + + return this.memoryProcess; + }; + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + Assert.ThrowsAsync(async () => + { + await executor.ExecuteAsync().ConfigureAwait(false); + }, "wrk2 did not write metrics to console."); + + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(8)); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.IsTrue(command == "sudo" || command.EndsWith("wrk")); + + if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")} --version")); + } + else + { + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + + return this.memoryProcess; + }; + TestWrk2Executor executor2 = new TestWrk2Executor(this.mockFixture); + await executor2.InitializeAsync().ConfigureAwait(false); + await executor2.ExecuteAsync().ConfigureAwait(false); + + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(15)); + this.mockFixture.ApiClient.Verify(x => x.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())); + } + + [Test] + [TestCase("-L -R 1000 -t 10 -c 50 -d 100s --timeout 10s https://{serverip}/api_new/1kb")] + [TestCase("{serverip}_{clientip}_{reverseproxyip}_{serverip}")] + public void WrkClientExecutorReturnsCorrectArguments(string commandArg) + { + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); + ClientInstance serverInstance = new ClientInstance(name: nameof(State), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(State), ipAddress: "5.6.7.8", role: ClientRole.Client); + ClientInstance reverseProxyInstance = new ClientInstance(name: nameof(State), ipAddress: "9.0.1.2", role: ClientRole.ReverseProxy); + + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance, reverseProxyInstance }); + + this.mockFixture.Parameters = new Dictionary() + { + { "CommandArguments", commandArg } + }; + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + string expected = executor.GetCommandLineArguments(); + + string result = commandArg + .Replace("{serverip}", "1.2.3.4") + .Replace("{clientip}", "5.6.7.8") + .Replace("{reverseproxyip}", "9.0.1.2"); + + Assert.AreEqual(expected, result); + } + + [Test] + public void Wrk2ExecutorThrowsWhenBindToCoresIsTrueButCoreAffinityIsNotProvided() + { + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); + ClientInstance serverInstance = new ClientInstance(name: nameof(State), ipAddress: "1.2.3.4", role: ClientRole.Server); + + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance }); + this.mockFixture.Parameters = new Dictionary() + { + { "PackageName", "wrk2" }, + { "CommandArguments", "-t 10 -c 100 -d 15s http://{ServerIp}:9876/json" }, + { "BindToCores", true } + }; + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + Assert.Throws(() => executor.Validate()); + } + + [Test] + [Ignore("Unit test requires additional state management setup and needs to be simplified.")] + public async Task Wrk2ExecutorExecutesWithCoreAffinityOnLinux() + { + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64, nameof(State)); + ClientInstance serverInstance = new ClientInstance(name: nameof(State), ipAddress: "1.2.3.4", role: ClientRole.Server); + + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance }); + this.mockFixture.Parameters = new Dictionary() + { + { "PackageName", "wrk2" }, + { "CommandArguments", "-t 10 -c 100 -d 15s http://{ServerIp}:9876/json" }, + { "BindToCores", true }, + { "CoreAffinity", "0-7" }, + { "TargetService", "server" } + }; + + DependencyFixture dependencyFixture = new DependencyFixture(); + dependencyFixture.Setup(PlatformID.Unix, Architecture.X64); + dependencyFixture.SetupPackage("wrk2"); + dependencyFixture.SetupPackage("wrkconfiguration"); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((string name, CancellationToken token) => dependencyFixture.PackageManager.FirstOrDefault(p => p.Name == name)); + + this.mockFixture.FileSystem.Setup(x => x.File.Exists(It.IsAny())).Returns(true); + + // Setup API client for state management + this.mockFixture.ApiClient + .Setup(x => x.CreateStateAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK)); + + this.mockFixture.ApiClient + .Setup(x => x.UpdateStateAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK)); + + this.mockFixture.ApiClient + .Setup(x => x.DeleteStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.NoContent)); + + this.mockFixture.ApiClient.Setup(s => s.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => + { + Item result = new Item(nameof(State), new State()); + result.Definition.Online(true); + return this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, result); + }); + + this.mockFixture + .TrackProcesses() + .SetupProcessOutput( + "--version", + "wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer") + .SetupProcessOutput( + ".*wrk.*-t 10 -c 100.*", + File.ReadAllText(Path.Combine( + Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), + "Examples", "Wrk", "wrkStandardExample1.txt"))); + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + await executor.InitializeAsync(); + await executor.ExecuteAsync(); + + // Verify numactl was used with correct core affinity + this.mockFixture.Tracking.AssertCommandsExecuted(true, + "sudo bash -c \\\"numactl -C 0-7 .*wrk.*" + ); + + this.mockFixture.Tracking.AssertCommandExecutedTimes("numactl", 1); + } + + [Test] + public async Task WrkClientExecutorReturnsCorrectArguments() + { + string commandArgumentInput = @"--rate 1000 --latency --threads 10 --connections 100 --duration 100s --timeout 10s https://{serverip}/api_new/5kb"; + ClientInstance serverInstance = new ClientInstance(name: nameof(State), ipAddress: "1.2.3.4", role: ClientRole.Server); + + string directory = @"some/random\dir/name/"; + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance }); + this.mockFixture.Parameters = new Dictionary() + { + {"CommandArguments", commandArgumentInput }, + { "Scenario", "bar" }, + { "ToolName", "wrk" }, + { "PackageName", "wrk" }, + { "FileSizeInKB", 5}, + { "TestDuration", TimeSpan.FromSeconds(60).ToString() } + }; + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + executor.PackageDirectory = directory; + string result = executor.GetCommandLineArguments(); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string file) => + { + string result = executor.Combine(directory, "runwrk.sh"); + Assert.AreEqual(file, result); + }); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + string results = commandArgumentInput.Replace("{serverip}", "1.2.3.4"); + Assert.AreEqual(command, "sudo"); + if (arguments.Contains("--version")) + { + Assert.AreEqual(arguments, $"bash {executor.Combine(directory, "runwrk.sh")} --version"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + } + else + { + Assert.AreEqual(arguments, $"bash {executor.Combine(directory, "runwrk.sh")} \"{results}\""); + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + Assert.AreEqual(workingDir, directory); + return this.memoryProcess; + }; + + await executor.ExecuteWorkloadAsync(result, workingDir: directory).ConfigureAwait(false); + } + + public void SetUpWorkloadOutput() + { + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample2.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + + private class TestWrk2Executor : Wrk2Executor + { + public TestWrk2Executor(MockFixture mockFixture) + : base(mockFixture.Dependencies, mockFixture.Parameters) + { + this.ServerApi = mockFixture.ApiClient.Object; + this.ClientFlowRetryPolicy = Policy.NoOpAsync(); + this.ClientRetryPolicy = Policy.NoOpAsync(); + } + + public new void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public string GetCommandLineArguments() + { + return base.GetCommandLineArguments(CancellationToken.None); + } + + public async Task InitializeAsync() + { + await base.InitializeAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ExecuteAsync() + { + await base.ExecuteAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ExecuteWorkloadAsync(string commandArguments, string workingDir) + { + await base.ExecuteWorkloadAsync(commandArguments, workingDir, EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public new void Validate() + { + base.Validate(); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs new file mode 100644 index 0000000000..7d272e474a --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs @@ -0,0 +1,683 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Reflection; + using System.Text; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Common; + using VirtualClient.Actions.Memtier; + using VirtualClient.Contracts; + using Microsoft.CodeAnalysis; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Moq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Telemetry; + using Architecture = System.Runtime.InteropServices.Architecture; + + [TestFixture] + [Category("Unit")] + public class WrkExecutorTests + { + private MockFixture mockFixture; + private DependencyFixture dependencyFixture; + private InMemoryProcess memoryProcess; + private Dictionary defaultProperties; + private string wrkPackageName = "wrk"; + private string scriptPackageName = "wrkconfiguration"; + + [SetUp] + public void SetupTests() + { + this.mockFixture = new MockFixture(); + this.memoryProcess = new InMemoryProcess + { + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + this.defaultProperties = new Dictionary() + { + { "PackageName", this.wrkPackageName }, + { "Scenario", "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb" }, + { "CommandArguments", "--latency --threads{ThreadCount} --connections{Connection} --duration{Duration.TotalSeconds}s" }, + { "Connection", 100 }, + { "ThreadCount", 10 }, + { "MaxCoreCount", 10 }, + { "TestDuration", "00:02:30"}, + { "Timeout", "00:20:00"}, + { "FileSizeInKB", 10}, + { "Role", "Client"}, + { "Tags", "Networking,NGINX,WRK"}, + }; + + dependencyFixture = new DependencyFixture(); + } + + public void SetSingleServerInstance() + { + // Setup: + // One server instance running on port 9876 with affinity to 4 logical processors + this.mockFixture.ApiClient.OnGetState(nameof(ServerState)) + .ReturnsAsync(this.mockFixture.CreateHttpResponse( + HttpStatusCode.OK, + new Item(nameof(ServerState), new ServerState(new List + { + new PortDescription + { + CpuAffinity = "0,1,2,3", + Port = 9876 + } + })))); + + this.mockFixture.ApiClientManager.Setup(mgr => mgr.GetOrCreateApiClient(It.IsAny(), It.IsAny())) + .Returns((id, instance) => this.mockFixture.ApiClient.Object); + + this.mockFixture.ApiClient.OnGetHeartbeat() + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + + this.mockFixture.ApiClient.OnGetServerOnline() + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + } + + [Test] + [TestCase("-L -t 10 -c 50 -d 100s --timeout 10s https://{serverip}/api_new/1kb")] + [TestCase("{serverip}_{clientip}_{reverseproxyip}")] + public void WrkClientExecutorReturnsCorrectArguments(string commandArg) + { + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + ClientInstance reverseProxyInstance = new ClientInstance(name: nameof(ClientRole.ReverseProxy), ipAddress: "9.0.1.2", role: ClientRole.ReverseProxy); + + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance, reverseProxyInstance }); + + this.mockFixture.Parameters = new Dictionary() + { + { "CommandArguments", commandArg } + }; + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + string results = commandArg + .Replace("{serverip}", "1.2.3.4") + .Replace("{clientip}", "5.6.7.8") + .Replace("{reverseproxyip}", "9.0.1.2"); + + Assert.AreEqual(executor.GetCommandLineArguments(), results); + } + + [Test] + public async Task WrkClientExecutorRunsWorkloadWithCorrectArguments() + { + string commandArgumentInput = @"--latency --threads 5 --connections 100 --duration 60s --timeout 10s https://{serverip}/api_new/5kb"; + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + ClientInstance reverseProxyInstance = new ClientInstance(name: nameof(ClientRole.ReverseProxy), ipAddress: "9.0.1.2", role: ClientRole.ReverseProxy); + + string directory = @"some/random\dir/name/"; + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance, reverseProxyInstance }); + this.mockFixture.Parameters = new Dictionary() + { + {"CommandArguments", commandArgumentInput }, + { "Scenario", "bar" }, + { "ToolName", "wrk" }, + { "PackageName", "wrk" }, + { "FileSizeInKB", 5}, + { "TestDuration", TimeSpan.FromSeconds(60).ToString() }, + { "TargetService", "server"} + }; + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + executor.PackageDirectory = directory; + this.SetUpWorkloadOutput(); + string result = executor.GetCommandLineArguments(); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string file) => + { + string result = executor.Combine(directory, "runwrk.sh"); + Assert.AreEqual(file, result); + }); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + string results = commandArgumentInput.Replace("{serverip}", "1.2.3.4"); + Assert.AreEqual(command, "sudo"); + if (arguments.Contains("--version")) + { + Assert.AreEqual(arguments, $"bash {executor.Combine(directory, "runwrk.sh")} --version"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + } + else + { + Assert.AreEqual(arguments, $"bash {executor.Combine(directory, "runwrk.sh")} \"{results}\""); + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + Assert.AreEqual(workingDir, directory); + return this.memoryProcess; + }; + + await executor.ExecuteWorkloadAsync(result, workingDir: directory).ConfigureAwait(false); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task WrkClientExecutorSetsServerWarmedUpFlagAfterWarmupExecution(PlatformID platform, Architecture architecture) + { + // Arrange + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance }); + + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + + // Setup parameters with WarmUp=true + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.Parameters.Add("WarmUp", true); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((string packageName, CancellationToken cancellationToken) => + dependencyFixture.PackageManager.FirstOrDefault(pkg => pkg.Name == packageName)); + + // Setup API client for server heartbeat and state + this.mockFixture.ApiClient.Setup(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => + { + Item result = new Item(nameof(State), new State()); + result.Definition.Online(true); + return this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, result); + }); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + } + else + { + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + + return this.memoryProcess; + }; + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + + // Act + await executor.InitializeAsync().ConfigureAwait(false); + await executor.ExecuteAsync().ConfigureAwait(false); + + // Assert + Assert.IsTrue(executor.GetIsServerWarmedUp(), "IsServerWarmedUp flag should be set to true after warm-up execution"); + } + + [Test] + [TestCase(PlatformID.Win32NT, Architecture.X64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public void WrkClientExecutorThrowsErrorIfPlatformIsWrong(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters = this.defaultProperties; + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + Assert.IsFalse(VirtualClientComponent.IsSupported(executor)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void WrkExecutorOnlySupportsWrkandWrk2(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters = new Dictionary() + { + { "PackageName", "wrk2" }, + }; + + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage("wrk2"); + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.FirstOrDefault()); + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + DependencyException exc = Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + + Assert.AreEqual(exc.Message, "TestWrkExecutor did not find correct package in the directory. Supported Package: wrk. Package Provided: wrk2"); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void WrkClientExecutorThrowsErrorIfPackageIsMissing(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.SetSingleServerInstance(); + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + + this.mockFixture.Parameters = this.defaultProperties; + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.FirstOrDefault()); + + TestWrkExecutor executor2 = new TestWrkExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor2.InitializeAsync().ConfigureAwait(false); + }); + + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task WrkClientExecutorSkipsExecutionWhenWarmupAndServerIsWarmedUp(PlatformID platform, Architecture architecture) + { + // Arrange + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance }); + + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.Parameters.Add("WarmUp", true); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((string packageName, CancellationToken cancellationToken) => + dependencyFixture.PackageManager.FirstOrDefault(pkg => pkg.Name == packageName)); + + // Used to track if ExecuteWorkloadAsync is called + bool workloadExecuted = false; + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + executor.SetIsServerWarmedUp(true); // Simulate server already warmed up + + // Override the ExecuteWorkloadAsync to track if it's called + executor.ExecuteWorkloadAsyncCallback = () => workloadExecuted = true; + + // Act + await executor.InitializeAsync().ConfigureAwait(false); + await executor.ExecuteAsync().ConfigureAwait(false); + + // Assert + Assert.IsFalse(workloadExecuted, "WorkloadAsync should not be executed when WarmUp=true and server is already warmed up"); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task WrkClientExecutorSetsUpWrkClientForFirstTime(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + TestWrkExecutor dummyExecutor = new TestWrkExecutor(this.mockFixture); + + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault(), platform, architecture).Path; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.Is(x => x == "reset.sh"))) + .Returns(false); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.Is(x => x != "reset.sh"))) + .Returns(true); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.wrkPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault()); + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task WrkClientExecutorInitializesAsExpected(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + TestWrkExecutor dummyExecutor = new TestWrkExecutor(this.mockFixture); + + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault(), platform, architecture).Path; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string file) => + { + if (file.EndsWith("wrk")) + { + string result = dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "wrk"); + Assert.AreEqual(file, result); + } + else + { + string result = dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath, Path.GetFileName(file)); + Assert.AreEqual(file, result); + } + }); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.wrkPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault()); + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64, "server")] + [TestCase(PlatformID.Unix, Architecture.Arm64, "server")] + [TestCase(PlatformID.Unix, Architecture.X64, "rp")] + [TestCase(PlatformID.Unix, Architecture.Arm64, "rp")] + public async Task WrkClientExecutorExecutesAsyncAsExpected(PlatformID platform, Architecture architecture, string targetService) + { + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + ClientInstance reverseProxyInstance = new ClientInstance(name: nameof(ClientRole.ReverseProxy), ipAddress: "9.0.1.2", role: ClientRole.ReverseProxy); + + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance, reverseProxyInstance }); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + TestWrkExecutor dummyExecutor = new TestWrkExecutor(this.mockFixture); + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault(), platform, architecture).Path; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.Parameters.Add("TargetService", targetService); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string file) => + { + if (file.EndsWith("wrk")) + { + string result = dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "wrk"); + Assert.AreEqual(file, result); + } + else if (file.EndsWith("runwrk.sh")) + { + string result = dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh"); + Assert.AreEqual(file, result); + } + else + { + string result = dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath, Path.GetFileName(file)); + Assert.AreEqual(file, result); + } + }); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.wrkPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault()); + + // create local state + this.mockFixture.ApiClient + .Setup(x => x.CreateStateAsync(It.Is(y => y == nameof(State)), It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)) + .Callback((string id, State state, CancellationToken _, IAsyncPolicy __) => + { + Assert.IsTrue(state.Online()); + }); + + this.mockFixture.ApiClient.Setup(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => + { + Item result = new Item(nameof(State), new State()); + result.Definition.Online(true); + return this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, result); + }); + + // delete local state before exiting + this.mockFixture.ApiClient.Setup(x => x.DeleteStateAsync(It.Is(y => y == nameof(State)), It.IsAny(), It.IsAny>())); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.AreEqual(command, "sudo"); + + if (arguments.Contains("chmod")) + { + Assert.AreEqual(arguments, $"chmod +x \"{dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "wrk")}\""); + } + else if (arguments.Contains("setup-config")) + { + Assert.AreEqual(arguments, $"bash setup-config.sh"); + } + else if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")} --version")); + } + else + { + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")}")); + } + + return this.memoryProcess; + }; + //this.SetUpWorkloadOutput(); + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + Assert.ThrowsAsync(async () => + { + await executor.ExecuteAsync().ConfigureAwait(false); + }); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.AreEqual(command, "sudo"); + + if (arguments.Contains("chmod")) + { + Assert.AreEqual(arguments, $"chmod +x \"{dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "wrk")}\""); + } + else if (arguments.Contains("setup-config")) + { + Assert.AreEqual(arguments, $"bash setup-config.sh"); + } + else if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")} --version")); + } + else + { + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")}")); + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + + return this.memoryProcess; + }; + TestWrkExecutor executor2 = new TestWrkExecutor(this.mockFixture); + await executor2.InitializeAsync().ConfigureAwait(false); + await executor2.ExecuteAsync().ConfigureAwait(false); + + this.mockFixture.ApiClient.Verify(x => x.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())); + } + + [Test] + public async Task GetWrkVersionReturnsCorrectVersion() + { + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + string expectedVersion = "4.2.0"; + string wrkOutput = $"wrk {expectedVersion} [epoll] Copyright (C) 2012 Will Glozer"; + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture + .TrackProcesses() + .SetupProcessOutput("--version", wrkOutput); + + string actualVersion = await executor.GetWrkVersionAsync(); + + Assert.AreEqual(expectedVersion, actualVersion); + this.mockFixture.Tracking.AssertCommandsExecuted(true, + $"sudo bash {Regex.Escape(executor.Combine(executor.PackageDirectory, WrkExecutor.WrkRunShell))} --version" + ); + } + + [Test] + public async Task GetWrkVersion_ReturnsNull_WhenVersionCannotBeParsed() + { + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture + .TrackProcesses() + .SetupProcessOutput("--version", "Invalid output without version"); + + string version = await executor.GetWrkVersionAsync(); + Assert.IsNull(version); + + this.mockFixture.Tracking.AssertCommandsExecuted(true, "sudo bash .* --version"); + } + + + public void SetUpWorkloadOutput() + { + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("")); + } + + private class TestWrkExecutor : WrkExecutor + { + public Action ExecuteWorkloadAsyncCallback { get; set; } + + public TestWrkExecutor(MockFixture mockFixture) + : base(mockFixture.Dependencies, mockFixture.Parameters) + { + this.ServerApi = mockFixture.ApiClient.Object; + this.ReverseProxyApi = mockFixture.ApiClient.Object; + this.ClientFlowRetryPolicy = Policy.NoOpAsync(); + this.ClientRetryPolicy = Policy.NoOpAsync(); + } + + public new void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public string GetCommandLineArguments() + { + return base.GetCommandLineArguments(CancellationToken.None); + } + + public bool GetIsServerWarmedUp() + { + return base.IsServerWarmedUp; + } + + public async Task GetWrkVersionAsync() + { + return await base.GetWrkVersionAsync(EventContext.None, CancellationToken.None); + } + + public void SetIsServerWarmedUp(bool value) + { + base.IsServerWarmedUp = value; + } + + public async Task InitializeAsync() + { + await base.InitializeAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task SetupWrkClient() + { + await base.SetupWrkClient(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ExecuteWorkloadAsync(string commandArguments, string workingDir) + { + await base.ExecuteWorkloadAsync(commandArguments, workingDir, EventContext.None, CancellationToken.None).ConfigureAwait(false); + ExecuteWorkloadAsyncCallback?.Invoke(); + } + + public async Task ExecuteAsync() + { + await base.ExecuteAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs new file mode 100644 index 0000000000..796f1494b9 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System.Collections.Generic; + using System.IO; + using System.Reflection; + using NUnit.Framework; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class WrkMetricsParserTest + { + private static string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + + [Test] + [TestCase(true)] + [TestCase(false)] + [TestCase(null)] + public void WRKParsesResultsCorrectly01(bool emitSpectrum) + { + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + IList actualMetrics = parser.Parse(emitSpectrum); + + Assert.AreEqual(parser.GetTestConfig(), "Running 1m test @ https://10.1.0.26/api_new/1kb with 2 threads and 20 connections"); + MetricAssert.Exists(actualMetrics, "latency_p50", 29.56 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p75", 39.75 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p90", 45.97 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99", 49.74 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_9", 50.17 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_99", 50.23 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_999", 50.23 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p100", 50.23 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "requests/sec", 16305.17); + MetricAssert.Exists(actualMetrics, "transfers/sec", 20.01, MetricUnit.Megabytes); + + if (emitSpectrum == true) + { + + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_000000", 8372.223); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_887500", 45449.215); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_997266", 50069.503); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_999805", 50200.575); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_999902", 50233.343); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p1_000000", 50233.343); + } + + } + + [Test] + public void WRKParsesResultsCorrectly02() + { + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample2.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + IList actualMetrics = parser.Parse(true); + + // Error + MetricAssert.Exists(actualMetrics, "Non-2xx or 3xx responses", 58902); + + // Raw data + MetricAssert.Exists(actualMetrics, "requests/sec", 1963.39); + MetricAssert.Exists(actualMetrics, "transfers/sec", 0.61033203125, MetricUnit.Megabytes); + + //Latency Distribution + MetricAssert.Exists(actualMetrics, "latency_p50", 1.43 , MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p75", 1.98 , MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p90", 2.68 , MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99", 3.96, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_9", 6.93, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_99", 8.99, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_999", 9.77, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p100", 9.77, MetricUnit.Milliseconds); + + //Uncorrected Latency Distribution + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p50", 483 * 0.001, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p75", 1.12, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p90", 1.71, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p99", 2.87, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p99_9", 5.76, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p99_99", 8.02, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p99_999", 8.41, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p100", 8.41, MetricUnit.Milliseconds); + + // latency spectrum + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_000000", 0.175); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_775000", 2.067); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_937500", 2.981); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_998437", 6.331); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_999219", 7.511); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_999969", 9.503); + + // Uncorrected latency spectrum + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_000000", 0.135); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_100000", 0.242); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_200000", 0.287); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_981250", 2.531); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_996094", 3.793); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_999939", 8.223); + + Assert.AreEqual(parser.GetTestConfig(), "Running 30s test @ http://10.1.0.13/index.html with 2 threads and 100 connections"); + } + + [Test] + public void WRKParsesResultsCorrectly03() + { + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample3.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + IList actualMetrics = parser.Parse(true); + + MetricAssert.Exists(actualMetrics, "Non-2xx or 3xx responses", 184596465); + Assert.AreEqual(parser.GetTestConfig(), "Running 2m test @ http://10.9.0.7/api_new/10kb with 32 threads and 5000 connections"); + // Raw data + MetricAssert.Exists(actualMetrics, "requests/sec", 1229838.61); + MetricAssert.Exists(actualMetrics, "transfers/sec", 382.35, MetricUnit.Megabytes); + + //Latency Distribution + MetricAssert.Exists(actualMetrics, "latency_p50", 3.85, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p75", 5.05, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p90", 7.27, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99", 11.16, MetricUnit.Milliseconds); + } + + [Test] + public void WRKParsesErrorCorrectly01() + { + string outputPath = Path.Combine(examplesDirectory, @"wrkErrorExample1.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + Assert.AreEqual(parser.GetTestConfig(), "Running 2m test @ http://10.1.0.15/api_new/10kb with 1 threads and 10000 connections"); + + WorkloadException exc = Assert.Throws(() => + { + parser.Parse(); + }); + + Assert.AreEqual(exc.Message, "Socket errors: connect 0, read 1645610, write 16, timeout 0"); + } + + [Test] + public void WRKParsesErrorCorrectly02() + { + string outputPath = Path.Combine(examplesDirectory, @"wrkErrorExample2.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + Assert.AreEqual(parser.GetTestConfig(), "Running 30s test @ http://10.1.0.13/index.html with 5 threads and 100 connections"); + + Assert.DoesNotThrow(() => { parser.Parse(); }); + + IList actualMetrics = parser.Parse(); + MetricAssert.Exists(actualMetrics, "Non-2xx or 3xx responses", 59956); + + MetricAssert.Exists(actualMetrics, "requests/sec", 1998.42); + MetricAssert.Exists(actualMetrics, "transfers/sec", 636.13/1024.0, MetricUnit.Megabytes); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchBaseExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchBaseExecutor.cs deleted file mode 100644 index 1628f0192e..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchBaseExecutor.cs +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.IO.Abstractions; - using System.Runtime.InteropServices; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.CodeAnalysis; - using Microsoft.Extensions.DependencyInjection; - using VirtualClient.Common; - using VirtualClient.Common.Extensions; - using VirtualClient.Common.Platform; - using VirtualClient.Common.Telemetry; - using VirtualClient.Contracts; - using VirtualClient.Contracts.Metadata; - using static System.Net.Mime.MediaTypeNames; - - /// - /// The AspNetBench workload executor. - /// - [SupportedPlatforms("linux-arm64,linux-x64,win-arm64,win-x64")] - public abstract class AspNetBenchBaseExecutor : VirtualClientMultiRoleComponent - { - private IFileSystem fileSystem; - private IPackageManager packageManager; - private IStateManager stateManager; - private ISystemManagement systemManagement; - - private string dotnetExePath; - private string aspnetBenchDirectory; - private string aspnetBenchDllPath; - private string bombardierFilePath; - private string wrkFilePath; - private string serverArgument; - private string clientArgument; - - /// - /// Constructor for - /// - /// Provides required dependencies to the component. - /// Parameters defined in the profile or supplied on the command line. - public AspNetBenchBaseExecutor(IServiceCollection dependencies, IDictionary parameters) - : base(dependencies, parameters) - { - this.systemManagement = this.Dependencies.GetService(); - this.packageManager = this.systemManagement.PackageManager; - this.stateManager = this.systemManagement.StateManager; - this.fileSystem = this.systemManagement.FileSystem; - } - - /// - /// The name of the package where the AspNetBench package is downloaded. - /// - public string TargetFramework - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.TargetFramework)).ToLower(); - } - } - - /// - /// The port for ASPNET to run. - /// - public string Port - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.Port), "9876"); - } - } - - /// - /// The name of the package where the bombardier package is downloaded. - /// - public string BombardierPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.BombardierPackageName), "bombardier"); - } - } - - /// - /// The name of the package where the wrk package is downloaded. - /// - public string WrkPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.WrkPackageName), "wrk"); - } - } - - /// - /// The name of the package where the DotNetSDK package is downloaded. - /// - public string DotNetSdkPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.DotNetSdkPackageName), "dotnetsdk"); - } - } - - /// - /// ASPNETCORE_threadCount - /// - public string AspNetCoreThreadCount - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.AspNetCoreThreadCount), 1); - } - } - - /// - /// DOTNET_SYSTEM_NET_SOCKETS_THREAD_COUNT - /// - public string DotNetSystemNetSocketsThreadCount - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.DotNetSystemNetSocketsThreadCount), 1); - } - } - - /// - /// wrk commandline - /// - public string WrkCommandLine - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.WrkCommandLine), string.Empty); - } - } - - /// - /// Initializes the environment for execution of the AspNetBench workload. - /// - protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - // This workload needs three packages: aspnetbenchmarks, dotnetsdk, bombardier - DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, CancellationToken.None) - .ConfigureAwait(false); - - if (workloadPackage != null) - { - // the directory we are looking for is at the src/Benchmarks - this.aspnetBenchDirectory = this.Combine(workloadPackage.Path, "src", "Benchmarks"); - } - - try - { - // Check for Bombardier Package, if not available try wrk package - DependencyPath bombardierPackage = await this.GetPlatformSpecificPackageAsync(this.BombardierPackageName, cancellationToken); - - if (bombardierPackage != null) - { - this.bombardierFilePath = this.Combine(bombardierPackage.Path, this.Platform == PlatformID.Unix ? "bombardier" : "bombardier.exe"); - await this.systemManagement.MakeFileExecutableAsync(this.bombardierFilePath, this.Platform, cancellationToken); - } - } - catch (DependencyException) - { - DependencyPath wrkPackage = await this.packageManager.GetPackageAsync(this.WrkPackageName, cancellationToken); - - if (wrkPackage != null) - { - this.wrkFilePath = this.Combine(wrkPackage.Path, "wrk"); - await this.systemManagement.MakeFileExecutableAsync(this.wrkFilePath, this.Platform, cancellationToken); - } - } - } - - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - /// - protected async Task BuildAspNetBenchAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - DependencyPath dotnetSdkPackage = await this.packageManager.GetPackageAsync(this.DotNetSdkPackageName, CancellationToken.None) - .ConfigureAwait(false); - - if (dotnetSdkPackage == null) - { - throw new DependencyException( - $"The expected DotNet SDK package does not exist on the system or is not registered.", - ErrorReason.WorkloadDependencyMissing); - } - - this.dotnetExePath = this.Combine(dotnetSdkPackage.Path, this.Platform == PlatformID.Unix ? "dotnet" : "dotnet.exe"); - // ~/vc/packages/dotnet/dotnet build -c Release -p:BenchmarksTargetFramework=net9.0 - // Build the aspnetbenchmark project - string buildArgument = $"build -c Release -p:BenchmarksTargetFramework={this.TargetFramework}"; - await this.ExecuteCommandAsync(this.dotnetExePath, buildArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken) - .ConfigureAwait(false); - - // "C:\Users\vcvmadmin\Benchmarks\src\Benchmarks\bin\Release\net9.0\Benchmarks.dll" - this.aspnetBenchDllPath = this.Combine( - this.aspnetBenchDirectory, - "bin", - "Release", - this.TargetFramework, - "Benchmarks.dll"); - } - - /// - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// - protected void CaptureMetrics(IProcessProxy process, EventContext telemetryContext) - { - try - { - this.MetadataContract.AddForScenario( - "AspNetBench", - $"{this.clientArgument},{this.serverArgument}", - toolVersion: null); - - this.MetadataContract.Apply(telemetryContext); - - WrkMetricParser parser = new WrkMetricParser(process.StandardOutput.ToString()); - - this.Logger.LogMetrics( - toolName: "AspNetBench", - scenarioName: $"ASP.NET_{this.TargetFramework}_Performance", - process.StartTime, - process.ExitTime, - parser.Parse(), - metricCategorization: "json", - scenarioArguments: $"Client: {this.clientArgument} | Server: {this.serverArgument}", - this.Tags, - telemetryContext); - } - catch (Exception exc) - { - throw new WorkloadResultsException($"Failed to parse bombardier output.", exc, ErrorReason.InvalidResults); - } - } - - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected Task StartAspNetServerAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - // Example: - // dotnet \Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:5000 --server Kestrel --kestrelTransport Sockets --protocol http - // --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" --header "Connection: keep-alive" - - string options = $"--nonInteractive true --scenarios json --urls http://*:{this.Port} --server Kestrel --kestrelTransport Sockets --protocol http"; - string headers = @"--header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive"""; - this.serverArgument = $"{this.aspnetBenchDllPath} {options} {headers}"; - - return this.ExecuteCommandAsync(this.dotnetExePath, this.serverArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken); - } - - /// - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected async Task RunBombardierAsync(string ipAddress, EventContext telemetryContext, CancellationToken cancellationToken) - { - using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) - { - // https://pkg.go.dev/github.com/codesenberg/bombardier - // ./bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:5000/json --print r --format json - this.clientArgument = $"--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ipAddress}:{this.Port}/json --print r --format json"; - - using (IProcessProxy process = await this.ExecuteCommandAsync(this.bombardierFilePath, this.clientArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken, runElevated: true) - .ConfigureAwait(false)) - { - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "AspNetBench", logToFile: true); - - process.ThrowIfWorkloadFailed(); - this.CaptureMetrics(process, telemetryContext); - } - } - } - } - - /// - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected async Task RunWrkAsync(string ipAddress, EventContext telemetryContext, CancellationToken cancellationToken) - { - using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) - { - // https://pkg.go.dev/github.com/codesenberg/bombardier - // ./wrk -t 256 -c 256 -d 15s --timeout 10s http://10.1.0.23:9876/json --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" - this.clientArgument = this.WrkCommandLine; - this.clientArgument = this.clientArgument.Replace("{ipAddress}", ipAddress); - this.clientArgument = this.clientArgument.Replace("{port}", this.Port); - - using (IProcessProxy process = await this.ExecuteCommandAsync(this.wrkFilePath, this.clientArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken, runElevated: true) - .ConfigureAwait(false)) - { - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "wrk", logToFile: true); - - process.ThrowIfWorkloadFailed(); - this.CaptureMetrics(process, telemetryContext); - } - } - } - } - } -} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchClientExecutor.cs deleted file mode 100644 index f8af30830a..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchClientExecutor.cs +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.Linq; - using System.Net; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Logging; - using Polly; - using VirtualClient; - using VirtualClient.Actions.Memtier; - using VirtualClient.Actions.NetworkPerformance; - using VirtualClient.Common; - using VirtualClient.Common.Contracts; - using VirtualClient.Common.Extensions; - using VirtualClient.Common.Telemetry; - using VirtualClient.Contracts; - using VirtualClient.Contracts.Metadata; - - /// - /// AspNetBench Benchmark Client Executor. - /// - public class AspNetBenchClientExecutor : AspNetBenchBaseExecutor - { - /// - /// Initializes a new instance of the class. - /// - /// Provides all of the required dependencies to the Virtual Client component. - /// An enumeration of key-value pairs that can control the execution of the component./param> - public AspNetBenchClientExecutor(IServiceCollection dependencies, IDictionary parameters = null) - : base(dependencies, parameters) - { - this.PollingTimeout = TimeSpan.FromMinutes(40); - } - - /// - /// Executes client side. - /// - protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - await this.WaitForRoleAsync(ClientRole.Server, telemetryContext, cancellationToken).ConfigureAwait(false); - string serverIPAddress = this.GetLayoutClientInstances(ClientRole.Server).First().IPAddress; - await this.RunWrkAsync(serverIPAddress, telemetryContext, cancellationToken).ConfigureAwait(false); - } - - /// - /// Initializes the environment and dependencies for client of AspNetBench Benchmark workload. - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - await base.InitializeAsync(telemetryContext, cancellationToken).ConfigureAwait(false); - - this.RegisterToTerminateRole(ClientRole.Server); - } - } -} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchExecutor.cs deleted file mode 100644 index b67cc23cfc..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchExecutor.cs +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.IO.Abstractions; - using System.Runtime.InteropServices; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.CodeAnalysis; - using Microsoft.Extensions.DependencyInjection; - using VirtualClient.Common; - using VirtualClient.Common.Extensions; - using VirtualClient.Common.Telemetry; - using VirtualClient.Contracts; - using VirtualClient.Contracts.Metadata; - - /// - /// The AspNetBench workload executor. - /// - public class AspNetBenchExecutor : VirtualClientComponent - { - private IFileSystem fileSystem; - private IPackageManager packageManager; - private IStateManager stateManager; - private ISystemManagement systemManagement; - - private string dotnetExePath; - private string aspnetBenchDirectory; - private string aspnetBenchDllPath; - private string bombardierFilePath; - private string serverArgument; - private string clientArgument; - - private Action killServer; - - /// - /// Constructor for - /// - /// Provides required dependencies to the component. - /// Parameters defined in the profile or supplied on the command line. - public AspNetBenchExecutor(IServiceCollection dependencies, IDictionary parameters) - : base(dependencies, parameters) - { - this.systemManagement = this.Dependencies.GetService(); - this.packageManager = this.systemManagement.PackageManager; - this.stateManager = this.systemManagement.StateManager; - this.fileSystem = this.systemManagement.FileSystem; - } - - /// - /// The name of the package where the AspNetBench package is downloaded. - /// - public string TargetFramework - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchExecutor.TargetFramework)).ToLower(); - } - } - - /// - /// The port for ASPNET to run. - /// - public string Port - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchExecutor.Port), "9876"); - } - } - - /// - /// The name of the package where the bombardier package is downloaded. - /// - public string BombardierPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchExecutor.BombardierPackageName), "bombardier"); - } - } - - /// - /// The name of the package where the DotNetSDK package is downloaded. - /// - public string DotNetSdkPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchExecutor.DotNetSdkPackageName), "dotnetsdk"); - } - } - - /// - /// Executes the AspNetBench workload. - /// - protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - Task serverTask = this.StartAspNetServerAsync(cancellationToken); - await this.RunBombardierAsync(telemetryContext, cancellationToken); - - this.killServer.Invoke(); - } - - /// - /// Initializes the environment for execution of the AspNetBench workload. - /// - protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - // This workload needs three packages: aspnetbenchmarks, dotnetsdk, bombardier - DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, CancellationToken.None, throwIfNotfound: true); - - // the directory we are looking for is at the src/Benchmarks - this.aspnetBenchDirectory = this.Combine(workloadPackage.Path, "src", "Benchmarks"); - - DependencyPath bombardierPackage = await this.GetPlatformSpecificPackageAsync(this.BombardierPackageName, cancellationToken); - this.bombardierFilePath = this.Combine(bombardierPackage.Path, this.Platform == PlatformID.Unix ? "bombardier" : "bombardier.exe"); - - await this.systemManagement.MakeFileExecutableAsync(this.bombardierFilePath, this.Platform, cancellationToken); - - DependencyPath dotnetSdkPackage = await this.packageManager.GetPackageAsync(this.DotNetSdkPackageName, CancellationToken.None); - - if (dotnetSdkPackage == null) - { - throw new DependencyException( - $"The expected DotNet SDK package does not exist on the system or is not registered.", - ErrorReason.WorkloadDependencyMissing); - } - - this.dotnetExePath = this.Combine(dotnetSdkPackage.Path, this.Platform == PlatformID.Unix ? "dotnet" : "dotnet.exe"); - - // ~/vc/packages/dotnet/dotnet build -c Release -p:BenchmarksTargetFramework=net9.0 - // Build the aspnetbenchmark project - string buildArgument = $"build -c Release -p:BenchmarksTargetFramework={this.TargetFramework}"; - await this.ExecuteCommandAsync(this.dotnetExePath, buildArgument, this.aspnetBenchDirectory, cancellationToken); - - // "C:\Users\vcvmadmin\Benchmarks\src\Benchmarks\bin\Release\net9.0\Benchmarks.dll" - this.aspnetBenchDllPath = this.Combine( - this.aspnetBenchDirectory, - "bin", - "Release", - this.TargetFramework, - "Benchmarks.dll"); - } - - private void CaptureMetrics(IProcessProxy process, EventContext telemetryContext) - { - try - { - this.MetadataContract.AddForScenario( - "AspNetBench", - $"{this.clientArgument},{this.serverArgument}", - toolVersion: null); - - this.MetadataContract.Apply(telemetryContext); - - BombardierMetricsParser parser = new BombardierMetricsParser(process.StandardOutput.ToString()); - - this.Logger.LogMetrics( - toolName: "AspNetBench", - scenarioName: $"ASP.NET_{this.TargetFramework}_Performance", - process.StartTime, - process.ExitTime, - parser.Parse(), - metricCategorization: "json", - scenarioArguments: $"Client: {this.clientArgument} | Server: {this.serverArgument}", - this.Tags, - telemetryContext); - } - catch (Exception exc) - { - throw new WorkloadResultsException($"Failed to parse bombardier output.", exc, ErrorReason.InvalidResults); - } - } - - private Task StartAspNetServerAsync(CancellationToken cancellationToken) - { - // Example: - // dotnet \Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:5000 --server Kestrel --kestrelTransport Sockets --protocol http - // --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" --header "Connection: keep-alive" - - string options = $"--nonInteractive true --scenarios json --urls http://localhost:{this.Port} --server Kestrel --kestrelTransport Sockets --protocol http"; - string headers = @"--header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive"""; - this.serverArgument = $"{this.aspnetBenchDllPath} {options} {headers}"; - - return this.ExecuteCommandAsync(this.dotnetExePath, this.serverArgument, this.aspnetBenchDirectory, cancellationToken, isServer: true); - } - - private async Task RunBombardierAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) - { - // https://pkg.go.dev/github.com/codesenberg/bombardier - // ./bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:5000/json --print r --format json - this.clientArgument = $"--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:{this.Port}/json --print r --format json"; - - using (IProcessProxy process = await this.ExecuteCommandAsync(this.bombardierFilePath, this.clientArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken, runElevated: true) - .ConfigureAwait(false)) - { - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "AspNetBench", logToFile: true); - - process.ThrowIfWorkloadFailed(); - this.CaptureMetrics(process, telemetryContext); - } - } - } - } - - private async Task ExecuteCommandAsync(string pathToExe, string commandLineArguments, string workingDirectory, CancellationToken cancellationToken, bool isServer = false) - { - if (!cancellationToken.IsCancellationRequested) - { - this.Logger.LogTraceMessage($"Executing process '{pathToExe}' '{commandLineArguments}' at directory '{workingDirectory}'."); - - EventContext telemetryContext = EventContext.Persisted() - .AddContext("command", pathToExe) - .AddContext("commandArguments", commandLineArguments); - - using (IProcessProxy process = this.systemManagement.ProcessManager.CreateElevatedProcess(this.Platform, pathToExe, commandLineArguments, workingDirectory)) - { - if (isServer) - { - this.killServer = () => process.SafeKill(this.Logger); - } - - this.CleanupTasks.Add(() => process.SafeKill(this.Logger)); - await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); - - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext); - - if (!isServer) - { - // We will kill the server at the end, exit code is -1, and we don't want it to log as failure. - process.ThrowIfWorkloadFailed(); - } - } - } - } - } - } -} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchServerExecutor.cs deleted file mode 100644 index db60f3b6b6..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchServerExecutor.cs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Extensions.DependencyInjection; - using VirtualClient; - using VirtualClient.Actions.Memtier; - using VirtualClient.Common.Contracts; - using VirtualClient.Common.Telemetry; - - /// - /// Redis Benchmark Client Executor. - /// - public class AspNetBenchServerExecutor : AspNetBenchBaseExecutor - { - /// - /// Initializes a new instance of the class. - /// - /// Provides all of the required dependencies to the Virtual Client component. - /// An enumeration of key-value pairs that can control the execution of the component./param> - public AspNetBenchServerExecutor(IServiceCollection dependencies, IDictionary parameters = null) - : base(dependencies, parameters) - { - } - - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - await this.BuildAspNetBenchAsync(telemetryContext, cancellationToken).ConfigureAwait(false); - - await this.StartAspNetServerAsync(telemetryContext, cancellationToken).ConfigureAwait(false); - await this.WaitAsync(cancellationToken) - .ConfigureAwait(false); - } - - private async Task GetServerStateAsync(IApiClient serverApiClient, CancellationToken cancellationToken) - { - Item state = await serverApiClient.GetStateAsync( - nameof(ServerState), - cancellationToken); - - if (state == null) - { - throw new WorkloadException( - $"Expected server state information missing. The server did not return state indicating the details for the Memcached server(s) running.", - ErrorReason.WorkloadUnexpectedAnomaly); - } - - return state.Definition; - } - } -} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs new file mode 100644 index 0000000000..afd7bf268b --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs @@ -0,0 +1,486 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO.Abstractions; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Net.Sockets; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// AspNet Orchard Server Executor. + /// + [SupportedPlatforms("linux-arm64,linux-x64,win-arm64,win-x64")] + public class AspNetOrchardServerExecutor : VirtualClientMultiRoleComponent + { + private Task serverProcess; + private bool disposed; + private IFileSystem fileSystem; + private IPackageManager packageManager; + private IStateManager stateManager; + private ISystemManagement systemManagement; + + private string dotnetExePath; + private string aspnetOrchardDirectory; + private string aspnetOrchardPublishPath; + + /// + /// Initializes a new instance of the class. + /// + /// Provides all of the required dependencies to the Virtual Client component. + /// An enumeration of key-value pairs that can control the execution of the component./param> + public AspNetOrchardServerExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + this.systemManagement = this.Dependencies.GetService(); + this.packageManager = this.systemManagement.PackageManager; + this.stateManager = this.systemManagement.StateManager; + this.fileSystem = this.systemManagement.FileSystem; + this.ServerRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(3, (retries) => TimeSpan.FromSeconds(retries)); + } + + /// + /// The name of the package where the AspNetBench package is downloaded. + /// + public string TargetFramework + { + get + { + return this.Parameters.GetValue(nameof(AspNetOrchardServerExecutor.TargetFramework)).ToLower(); + } + } + + /// + /// The port to run for Orchard Server. + /// + public string ServerPort + { + get + { + return this.Parameters.GetValue(nameof(AspNetOrchardServerExecutor.ServerPort), "5014"); + } + } + + /// + /// API Client that is used to communicate with server-hosted instance of the Virtual Client Server. + /// + public IApiClient ServerApi { get; set; } + + /// + /// The name of the package where the wrk package is downloaded. + /// + public string WrkPackageName + { + get + { + return this.Parameters.GetValue(nameof(AspNetOrchardServerExecutor.WrkPackageName), "wrk"); + } + } + + /// + /// The name of the package where the DotNetSDK package is downloaded. + /// + public string DotNetSdkPackageName + { + get + { + return this.Parameters.GetValue(nameof(AspNetOrchardServerExecutor.DotNetSdkPackageName), "dotnetsdk"); + } + } + + /// + /// Gets or sets whether to bind the workload to specific CPU cores. + /// + public bool BindToCores + { + get + { + return this.Parameters.GetValue(nameof(this.BindToCores), defaultValue: false); + } + } + + /// + /// Gets the CPU core affinity specification (e.g., "0-3", "0,2,4,6"). + /// + public string CoreAffinity + { + get + { + this.Parameters.TryGetValue(nameof(this.CoreAffinity), out IConvertible value); + return value?.ToString(); + } + } + + /// + /// A retry policy to apply to the server when starting to handle transient issues that + /// would otherwise prevent it from starting successfully. + /// + protected IAsyncPolicy ServerRetryPolicy { get; set; } + + /// + /// Disposes of resources used by the executor including shutting down any + /// instances of server running. + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + if (!this.disposed) + { + try + { + this.KillServerInstancesAsync(EventContext.None, CancellationToken.None) + .GetAwaiter().GetResult(); + } + catch + { + // Best-effort cleanup during dispose. + } + + this.disposed = true; + } + } + } + + /// + /// Initializes the environment for execution of the AspNetBench workload. + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, cancellationToken) + .ConfigureAwait(false); + if (workloadPackage == null) + { + throw new DependencyException( + $"The expected workload package '{this.PackageName}' does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } + else + { + this.aspnetOrchardDirectory = this.Combine(workloadPackage.Path, "src", "OrchardCore.Cms.Web"); + } + + DependencyPath dotnetSdkPackage = await this.packageManager.GetPackageAsync(this.DotNetSdkPackageName, cancellationToken) + .ConfigureAwait(false); + if (dotnetSdkPackage == null) + { + throw new DependencyException( + $"The expected DotNet SDK package does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } + + this.dotnetExePath = this.Combine(dotnetSdkPackage.Path, this.Platform == PlatformID.Unix ? "dotnet" : "dotnet.exe"); + + this.InitializeApiClients(); + } + + /// + /// Validates the component parameters. + /// + protected override void Validate() + { + base.Validate(); + + if (this.BindToCores) + { + this.ThrowIfParameterNotDefined(nameof(this.CoreAffinity)); + } + } + + /// + /// Initializes API client. + /// + protected void InitializeApiClients() + { + IApiClientManager clientManager = this.Dependencies.GetService(); + + if (!this.IsMultiRoleLayout()) + { + this.ServerApi = clientManager.GetOrCreateApiClient(IPAddress.Loopback.ToString(), IPAddress.Loopback); + } + else + { + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + } + } + + /// + /// + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + /// + protected async Task BuildAspNetOrchardAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + string buildArgument = $"publish -c Release --sc -f {this.TargetFramework} {this.Combine(this.aspnetOrchardDirectory, "OrchardCore.Cms.Web.csproj")}"; + await this.ExecuteCommandAsync(this.dotnetExePath, buildArgument, this.aspnetOrchardDirectory, telemetryContext, cancellationToken) + .ConfigureAwait(false); + + // bin/Release/net9.0/linux-x64/publish + this.aspnetOrchardPublishPath = this.Combine( + this.aspnetOrchardDirectory, + "bin", + "Release", + this.TargetFramework, + this.PlatformArchitectureName, + "publish"); + } + + /// + /// + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + protected override Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return this.Logger.LogMessageAsync($"{this.TypeName}.ExecuteServer", telemetryContext, async () => + { + try + { + this.SetServerOnline(false); + + await this.ServerApi.PollForHeartbeatAsync(TimeSpan.FromMinutes(5), cancellationToken); + + await this.DeleteStateAsync(telemetryContext, cancellationToken); + await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + await this.BuildAspNetOrchardAsync(telemetryContext, cancellationToken); + this.StartServerInstances(telemetryContext, cancellationToken); + await this.WaitForPortReadyAsync(telemetryContext, cancellationToken); + + await this.SaveStateAsync(telemetryContext, cancellationToken); + this.SetServerOnline(true); + + if (!this.IsMultiRoleLayout()) + { + // In single-VM mode, clear cleanup tasks to prevent the base class + // CleanupAsync from killing the server process. The server must stay + // alive for the subsequent client action. It will be killed by + // KillServerInstancesAsync on the next iteration or by Dispose. + this.CleanupTasks.Clear(); + } + else + { + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) + { + await Task.WhenAny(this.serverProcess); + if (cancellationToken.IsCancellationRequested) + { + await Task.WhenAll(this.serverProcess); + } + } + } + } + catch + { + this.SetServerOnline(false); + await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + throw; + } + }); + } + + /// + /// Waits for the configured port to start accepting TCP connections. + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected virtual async Task WaitForPortReadyAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + int port = int.Parse(this.ServerPort); + TimeSpan timeout = TimeSpan.FromMinutes(5); + DateTime deadline = DateTime.UtcNow.Add(timeout); + + this.Logger.LogTraceMessage($"{this.TypeName}: Waiting for server to accept connections on port {port}..."); + + while (DateTime.UtcNow < deadline && !cancellationToken.IsCancellationRequested) + { + try + { + using (TcpClient client = new TcpClient()) + { + await client.ConnectAsync(IPAddress.Loopback, port).ConfigureAwait(false); + this.Logger.LogTraceMessage($"{this.TypeName}: Server is accepting connections on port {port}."); + return; + } + } + catch (SocketException) + { + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false); + } + } + + cancellationToken.ThrowIfCancellationRequested(); + throw new WorkloadException( + $"The server did not start accepting connections on port {port} within {timeout}.", + ErrorReason.WorkloadFailed); + } + + private Task DeleteStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + return this.Logger.LogMessageAsync($"{this.TypeName}.DeleteState", relatedContext, async () => + { + using (HttpResponseMessage response = await this.ServerApi.DeleteStateAsync(nameof(State), cancellationToken)) + { + relatedContext.AddResponseContext(response); + if (response.StatusCode != HttpStatusCode.NoContent) + { + response.ThrowOnError(ErrorReason.HttpNonSuccessResponse); + } + } + }); + } + + private async Task KillServerInstancesAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + this.Logger.LogTraceMessage($"{this.TypeName}.KillServerInstances"); + await this.ExecuteCommandAsync("pkill", "OrchardCore", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); + await this.ExecuteCommandAsync("fuser", $"-n tcp -k {this.ServerPort}", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); + + await this.WaitAsync(TimeSpan.FromSeconds(3), cancellationToken); + } + + private void StartServerInstances(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + + this.Logger.LogMessage($"{this.TypeName}.StartServerInstances", relatedContext, () => + { + try + { + string command = "nohup"; + string workingDirectory = this.aspnetOrchardDirectory; + string commandArguments = $"{this.Combine(this.aspnetOrchardPublishPath, "OrchardCore.Cms.Web")} --urls http://*:{this.ServerPort}"; + + relatedContext.AddContext("command", command); + relatedContext.AddContext("commandArguments", commandArguments); + relatedContext.AddContext("workingDir", workingDirectory); + relatedContext.AddContext("bindToCores", this.BindToCores); + relatedContext.AddContext("coreAffinity", this.CoreAffinity); + + this.serverProcess = this.StartServerInstanceAsync(command, commandArguments, workingDirectory, relatedContext, cancellationToken); + } + catch (OperationCanceledException) + { + // Expected whenever certain operations (e.g. Task.Delay) are cancelled. + } + }); + } + + private Task StartServerInstanceAsync(string command, string commandArguments, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken) + { + return (this.ServerRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => + { + try + { + IProcessProxy process = null; + ProcessAffinityConfiguration affinityConfig = null; + + if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) + { + affinityConfig = ProcessAffinityConfiguration.Create( + this.Platform, + this.CoreAffinity); + + telemetryContext.AddContext("affinityMask", affinityConfig.ToString()); + + if (this.Platform == PlatformID.Win32NT) + { + process = this.systemManagement.ProcessManager.CreateProcess( + command, + commandArguments, + workingDirectory); + } + else + { + string fullCommandLine = $"{command} {commandArguments}"; + + LinuxProcessAffinityConfiguration linuxConfig = (LinuxProcessAffinityConfiguration)affinityConfig; + string wrappedCommand = linuxConfig.GetCommandWithAffinity(null, fullCommandLine); + + process = this.systemManagement.ProcessManager.CreateProcess( + "/bin/bash", + $"-c {wrappedCommand}", + workingDirectory); + } + } + else + { + process = await this.ExecuteCommandAsync(command, commandArguments, workingDirectory, telemetryContext, cancellationToken); + } + + using (process) + { + if (affinityConfig != null) + { + if (this.Platform == PlatformID.Win32NT) + { + process.Start(); + process.ApplyAffinity((WindowsProcessAffinityConfiguration)affinityConfig); + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + } + else + { + await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "Orchard"); + process.ThrowIfWorkloadFailed(successCodes: new int[] { 0 }); + } + } + } + catch (OperationCanceledException) + { + // Expected whenever certain operations (e.g. Task.Delay) are cancelled. + } + catch (Exception exc) + { + this.Logger.LogMessage( + $"{this.TypeName}.StartServerInstanceError", + LogLevel.Error, + telemetryContext.Clone().AddError(exc)); + + throw; + } + }); + } + + private Task SaveStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + return this.Logger.LogMessageAsync($"{this.TypeName}.SaveState", relatedContext, async () => + { + Item serverState = new Item(nameof(State), new State()); + serverState.Definition.Online(true); + using (HttpResponseMessage response = await this.ServerApi.UpdateStateAsync(serverState.Id, serverState, cancellationToken)) + { + relatedContext.AddResponseContext(response); + response.ThrowOnError(ErrorReason.HttpNonSuccessResponse); + } + }); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs new file mode 100644 index 0000000000..a7f4661b0b --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs @@ -0,0 +1,500 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO.Abstractions; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Net.Sockets; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// AspNet Server Executor. + /// + [SupportedPlatforms("linux-arm64,linux-x64,win-arm64,win-x64")] + public class AspNetServerExecutor : VirtualClientMultiRoleComponent + { + private Task serverProcess; + private bool disposed; + private IFileSystem fileSystem; + private IPackageManager packageManager; + private IStateManager stateManager; + private ISystemManagement systemManagement; + + private string dotnetExePath; + private string aspnetBenchDirectory; + private string aspnetBenchDllPath; + + /// + /// Initializes a new instance of the class. + /// + /// Provides all of the required dependencies to the Virtual Client component. + /// An enumeration of key-value pairs that can control the execution of the component./param> + public AspNetServerExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + this.systemManagement = this.Dependencies.GetService(); + this.packageManager = this.systemManagement.PackageManager; + this.stateManager = this.systemManagement.StateManager; + this.fileSystem = this.systemManagement.FileSystem; + this.ServerRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(3, (retries) => TimeSpan.FromSeconds(retries)); + } + + /// + /// The name of the package where the AspNetBench package is downloaded. + /// + public string TargetFramework + { + get + { + return this.Parameters.GetValue(nameof(AspNetServerExecutor.TargetFramework)).ToLower(); + } + } + + /// + /// The port for ASPNET to run. + /// + public string ServerPort + { + get + { + return this.Parameters.GetValue(nameof(AspNetServerExecutor.ServerPort), "9876"); + } + } + + /// + /// API Client that is used to communicate with server-hosted instance of the Virtual Client Server. + /// + public IApiClient ServerApi { get; set; } + + /// + /// The name of the package where the DotNetSDK package is downloaded. + /// + public string DotNetSdkPackageName + { + get + { + return this.Parameters.GetValue(nameof(AspNetServerExecutor.DotNetSdkPackageName), "dotnetsdk"); + } + } + + /// + /// ASPNETCORE_threadCount + /// + public string AspNetCoreThreadCount + { + get + { + return this.Parameters.GetValue(nameof(AspNetServerExecutor.AspNetCoreThreadCount), "1"); + } + } + + /// + /// DOTNET_SYSTEM_NET_SOCKETS_THREAD_COUNT + /// + public string DotNetSystemNetSocketsThreadCount + { + get + { + return this.Parameters.GetValue(nameof(AspNetServerExecutor.DotNetSystemNetSocketsThreadCount), "1"); + } + } + + /// + /// Gets or sets whether to bind the workload to specific CPU cores. + /// + public bool BindToCores + { + get + { + return this.Parameters.GetValue(nameof(this.BindToCores), defaultValue: false); + } + } + + /// + /// Gets the CPU core affinity specification (e.g., "0-3", "0,2,4,6"). + /// + public string CoreAffinity + { + get + { + this.Parameters.TryGetValue(nameof(this.CoreAffinity), out IConvertible value); + return value?.ToString(); + } + } + + /// + /// A retry policy to apply to the server when starting to handle transient issues that + /// would otherwise prevent it from starting successfully. + /// + protected IAsyncPolicy ServerRetryPolicy { get; set; } + + /// + /// Disposes of resources used by the executor including shutting down any + /// instances of ASP.NET server running. + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + if (!this.disposed) + { + try + { + this.KillServerInstancesAsync(EventContext.None, CancellationToken.None) + .GetAwaiter().GetResult(); + } + catch + { + // Best-effort cleanup during dispose. + } + + this.disposed = true; + } + } + } + + /// + /// Initializes the environment for execution of the AspNetBench workload. + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, cancellationToken) + .ConfigureAwait(false); + + if (workloadPackage != null) + { + this.aspnetBenchDirectory = this.Combine(workloadPackage.Path, "src", "Benchmarks"); + } + else + { + throw new DependencyException( + $"The expected workload package '{this.PackageName}' does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } + + DependencyPath dotnetSdkPackage = await this.packageManager.GetPackageAsync(this.DotNetSdkPackageName, cancellationToken) + .ConfigureAwait(false); + + if (dotnetSdkPackage == null) + { + throw new DependencyException( + $"The expected DotNet SDK package does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } + + this.dotnetExePath = this.Combine(dotnetSdkPackage.Path, this.Platform == PlatformID.Unix ? "dotnet" : "dotnet.exe"); + + this.InitializeApiClients(); + } + + /// + /// Validates the component parameters. + /// + protected override void Validate() + { + base.Validate(); + + if (this.BindToCores) + { + this.ThrowIfParameterNotDefined(nameof(this.CoreAffinity)); + } + } + + /// + /// Initializes API client. + /// + protected void InitializeApiClients() + { + IApiClientManager clientManager = this.Dependencies.GetService(); + + if (!this.IsMultiRoleLayout()) + { + this.ServerApi = clientManager.GetOrCreateApiClient(IPAddress.Loopback.ToString(), IPAddress.Loopback); + } + else + { + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + } + } + + /// + /// + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + protected override Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return this.Logger.LogMessageAsync($"{this.TypeName}.ExecuteServer", telemetryContext, async () => + { + try + { + this.SetServerOnline(false); + + await this.ServerApi.PollForHeartbeatAsync(TimeSpan.FromMinutes(5), cancellationToken); + + await this.DeleteStateAsync(telemetryContext, cancellationToken); + await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + await this.BuildAspNetBenchAsync(telemetryContext, cancellationToken); + this.StartServerInstances(telemetryContext, cancellationToken); + await this.WaitForPortReadyAsync(telemetryContext, cancellationToken); + + await this.SaveStateAsync(telemetryContext, cancellationToken); + this.SetServerOnline(true); + + if (!this.IsMultiRoleLayout()) + { + // In single-VM mode, clear cleanup tasks to prevent the base class + // CleanupAsync from killing the server process. The server must stay + // alive for the subsequent client action. It will be killed by + // KillServerInstancesAsync on the next iteration or by Dispose. + this.CleanupTasks.Clear(); + } + else + { + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) + { + await Task.WhenAny(this.serverProcess); + + if (cancellationToken.IsCancellationRequested) + { + await Task.WhenAll(this.serverProcess); + } + } + } + } + catch + { + this.SetServerOnline(false); + await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + throw; + } + }); + } + + /// + /// Builds the ASP.NET Benchmark application + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + protected async Task BuildAspNetBenchAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + string buildArgument = $"build -c Release -p:BenchmarksTargetFramework={this.TargetFramework}"; + await this.ExecuteCommandAsync(this.dotnetExePath, buildArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken) + .ConfigureAwait(false); + + this.aspnetBenchDllPath = this.Combine( + this.aspnetBenchDirectory, + "bin", + "Release", + this.TargetFramework, + "Benchmarks.dll"); + } + + /// + /// Waits for the configured port to start accepting TCP connections. + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected virtual async Task WaitForPortReadyAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + int port = int.Parse(this.ServerPort); + TimeSpan timeout = TimeSpan.FromMinutes(5); + DateTime deadline = DateTime.UtcNow.Add(timeout); + + this.Logger.LogTraceMessage($"{this.TypeName}: Waiting for server to accept connections on port {port}..."); + + while (DateTime.UtcNow < deadline && !cancellationToken.IsCancellationRequested) + { + try + { + using (TcpClient client = new TcpClient()) + { + await client.ConnectAsync(IPAddress.Loopback, port).ConfigureAwait(false); + this.Logger.LogTraceMessage($"{this.TypeName}: Server is accepting connections on port {port}."); + return; + } + } + catch (SocketException) + { + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false); + } + } + + cancellationToken.ThrowIfCancellationRequested(); + throw new WorkloadException( + $"The server did not start accepting connections on port {port} within {timeout}.", + ErrorReason.WorkloadFailed); + } + + private Task DeleteStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + return this.Logger.LogMessageAsync($"{this.TypeName}.DeleteState", relatedContext, async () => + { + using (HttpResponseMessage response = await this.ServerApi.DeleteStateAsync(nameof(State), cancellationToken)) + { + relatedContext.AddResponseContext(response); + if (response.StatusCode != HttpStatusCode.NoContent) + { + response.ThrowOnError(ErrorReason.HttpNonSuccessResponse); + } + } + }); + } + + private async Task KillServerInstancesAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + this.Logger.LogTraceMessage($"{this.TypeName}.KillServerInstances"); + + await this.ExecuteCommandAsync("pkill", "dotnet", this.aspnetBenchDirectory, telemetryContext, cancellationToken); + + await this.ExecuteCommandAsync("fuser", $"-n tcp -k {this.ServerPort}", this.aspnetBenchDirectory, telemetryContext, cancellationToken); + + await this.WaitAsync(TimeSpan.FromSeconds(3), cancellationToken); + } + + private void StartServerInstances(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + + this.Logger.LogMessage($"{this.TypeName}.StartServerInstances", relatedContext, () => + { + try + { + string options = $"--nonInteractive true --scenarios json --urls http://*:{this.ServerPort} --server Kestrel --kestrelTransport Sockets --protocol http"; + string headers = @"--header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive"""; + string commandArguments = $"{this.aspnetBenchDllPath} {options} {headers}"; + string workingDirectory = this.aspnetBenchDirectory; + + relatedContext.AddContext("command", this.dotnetExePath); + relatedContext.AddContext("commandArguments", commandArguments); + relatedContext.AddContext("workingDir", workingDirectory); + relatedContext.AddContext("bindToCores", this.BindToCores); + relatedContext.AddContext("coreAffinity", this.CoreAffinity); + + this.serverProcess = this.StartServerInstanceAsync(this.dotnetExePath, commandArguments, workingDirectory, relatedContext, cancellationToken); + } + catch (OperationCanceledException) + { + // Expected whenever certain operations (e.g. Task.Delay) are cancelled. + } + }); + } + + private Task StartServerInstanceAsync(string command, string commandArguments, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken) + { + return (this.ServerRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => + { + try + { + IProcessProxy process = null; + ProcessAffinityConfiguration affinityConfig = null; + + if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) + { + affinityConfig = ProcessAffinityConfiguration.Create( + this.Platform, + this.CoreAffinity); + + telemetryContext.AddContext("affinityMask", affinityConfig.ToString()); + + if (this.Platform == PlatformID.Win32NT) + { + process = this.systemManagement.ProcessManager.CreateProcess( + command, + commandArguments, + workingDirectory); + } + else + { + string fullCommandLine = $"{command} {commandArguments}"; + + LinuxProcessAffinityConfiguration linuxConfig = (LinuxProcessAffinityConfiguration)affinityConfig; + string wrappedCommand = linuxConfig.GetCommandWithAffinity(null, fullCommandLine); + + process = this.systemManagement.ProcessManager.CreateProcess( + "/bin/bash", + $"-c {wrappedCommand}", + workingDirectory); + } + } + else + { + process = await this.ExecuteCommandAsync(command, commandArguments, workingDirectory, telemetryContext, cancellationToken); + } + + using (process) + { + if (affinityConfig != null) + { + if (this.Platform == PlatformID.Win32NT) + { + process.Start(); + process.ApplyAffinity((WindowsProcessAffinityConfiguration)affinityConfig); + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + } + else + { + await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "AspNetBenchmarks"); + process.ThrowIfWorkloadFailed(successCodes: new int[] { 0 }); + } + } + } + catch (OperationCanceledException) + { + // Expected whenever certain operations (e.g. Task.Delay) are cancelled. + } + catch (Exception exc) + { + this.Logger.LogMessage( + $"{this.TypeName}.StartServerInstanceError", + LogLevel.Error, + telemetryContext.Clone().AddError(exc)); + + throw; + } + }); + } + + private Task SaveStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + return this.Logger.LogMessageAsync($"{this.TypeName}.SaveState", relatedContext, async () => + { + Item serverState = new Item(nameof(State), new State()); + serverState.Definition.Online(true); + using (HttpResponseMessage response = await this.ServerApi.UpdateStateAsync(serverState.Id, serverState, cancellationToken)) + { + relatedContext.AddResponseContext(response); + response.ThrowOnError(ErrorReason.HttpNonSuccessResponse); + } + }); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs new file mode 100644 index 0000000000..34f92f5569 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs @@ -0,0 +1,538 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.IO.Abstractions; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + using VirtualClient.Contracts.Metadata; + + /// + /// The Bombardier Client executor. + /// + [SupportedPlatforms("linux-arm64,linux-x64,win-x64,win-arm64")] + public class BombardierExecutor : VirtualClientMultiRoleComponent + { + /// + /// Constructor + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public BombardierExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + this.ClientFlowRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(2, (retries) => TimeSpan.FromSeconds(retries * 2)); + + this.ClientRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(2, (retries) => TimeSpan.FromSeconds(retries)); + + this.SystemManagement = this.Dependencies.GetService(); + } + + /// + /// API Client that is used to communicate with server-hosted instance of the Virtual Client Server. + /// + public IApiClient ServerApi { get; set; } + + /// + /// API Client that is used to communicate with ReverseProxy instance of the Virtual Client Server. + /// + public IApiClient ReverseProxyApi { get; set; } + + /// + /// Provides components and services necessary for interacting with the local system and environment. + /// + public ISystemManagement SystemManagement { get; } + + /// + /// Option for testing webserver (default), reverse-proxy (rp), api-gateway (apigw) + /// + public string TargetService + { + get + { + switch (this.Parameters.GetValue(nameof(this.TargetService), string.Empty).ToLower()) + { + case "reverse-proxy": + case "rp": + return "rp"; + case "apigw": + case "api-gateway": + return "apigw"; + case "server": + return "server"; + default: + IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); + if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + { + return "server"; + } + else + { + return "rp"; + } + } + } + } + + /// + /// The command line argument defined in the profile. + /// + public string CommandArguments + { + get + { + return this.Parameters.GetValue(nameof(this.CommandArguments)); + } + } + + /// + /// Gets or sets whether to bind the workload to specific CPU cores. + /// + public bool BindToCores + { + get + { + return this.Parameters.GetValue(nameof(this.BindToCores), defaultValue: false); + } + } + + /// + /// Gets the CPU core affinity specification (e.g., "0-3", "0,2,4,6"). + /// + public string CoreAffinity + { + get + { + this.Parameters.TryGetValue(nameof(this.CoreAffinity), out IConvertible value); + return value?.ToString(); + } + } + + /// + /// Polling Timeout + /// + public TimeSpan Timeout + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.Timeout), TimeSpan.FromMinutes(5)); + } + } + + /// + /// Parameter defines true/false whether the action is meant to warm up the server. + /// We do not capture metrics on warm up operations. + /// + public bool WarmUp + { + get + { + return this.Parameters.GetValue(nameof(this.WarmUp), false); + } + } + + /// + /// The path to the Bombardier package. + /// + public string PackageDirectory { get; set; } + + /// + /// True/false whether the server instance has been warmed up. + /// + protected bool IsServerWarmedUp { get; set; } + + /// + /// The retry policy to apply to the client-side execution workflow. + /// + protected IAsyncPolicy ClientFlowRetryPolicy { get; set; } + + /// + /// The retry policy to apply to each Bombardier workload instance when trying to startup + /// against a target server. + /// + protected IAsyncPolicy ClientRetryPolicy { get; set; } + + /// + /// Initializes the executor dependencies, package locations, server api, etc... + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + DependencyPath bombardierPackage = await this.GetPlatformSpecificPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); + + this.PackageDirectory = bombardierPackage.Path; + + // Make bombardier executable on Unix systems + if (this.Platform == PlatformID.Unix) + { + string bombardierPath = this.Combine(this.PackageDirectory, "bombardier"); + await this.SystemManagement.MakeFileExecutableAsync(bombardierPath, this.Platform, cancellationToken).ConfigureAwait(false); + } + + this.InitializeApiClients(); + } + + /// + /// Validates the component parameters. + /// + protected override void Validate() + { + base.Validate(); + + if (this.BindToCores) + { + this.ThrowIfParameterNotDefined(nameof(this.CoreAffinity)); + } + } + + /// + /// Executes component logic + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + if (!this.WarmUp || !this.IsServerWarmedUp) + { + Task clientWorkloadTask; + + clientWorkloadTask = this.ClientFlowRetryPolicy.ExecuteAsync(async () => + { + if (!cancellationToken.IsCancellationRequested) + { + this.Logger.LogTraceMessage("Synchronization: Poll server API for heartbeat..."); + await this.ServerApi.PollForHeartbeatAsync(this.PollingTimeout, cancellationToken); + this.Logger.LogTraceMessage("Synchronization: Poll server for online signal..."); + + await this.ServerApi.PollForExpectedStateAsync(nameof(State), (state => state.Online()), this.Timeout, cancellationToken).ConfigureAwait(false); + + this.Logger.LogTraceMessage("Synchronization: Server online signal confirmed..."); + + // verify ReverseProxy is online + if ((this.ReverseProxyApi != null) && ((this.TargetService == "rp") || (this.TargetService == "apigw"))) + { + this.Logger.LogTraceMessage("Synchronization: Poll ReverseProxy for online signal..."); + await this.ReverseProxyApi.PollForExpectedStateAsync(nameof(State), (state => state.Online()), TimeSpan.FromMinutes(10), cancellationToken).ConfigureAwait(false); + this.Logger.LogTraceMessage("Synchronization: ReverseProxy online signal confirmed..."); + Item reverseProxyState = await this.ReverseProxyApi.GetStateAsync(nameof(State), cancellationToken).ConfigureAwait(false); + telemetryContext.AddContext(nameof(reverseProxyState), reverseProxyState); + + HttpResponseMessage reverseProxyHttpResponseMessage = await this.ReverseProxyApi.GetStateAsync("version", cancellationToken).ConfigureAwait(continueOnCapturedContext: false); + telemetryContext.AddResponseContext(reverseProxyHttpResponseMessage); + + if (reverseProxyHttpResponseMessage.IsSuccessStatusCode) + { + Item reverseProxyVersion = await reverseProxyHttpResponseMessage.FromContentAsync>(); + telemetryContext.AddContext(nameof(reverseProxyVersion), reverseProxyVersion.Definition.Properties); + } + } + + this.Logger.LogTraceMessage("Synchronization: Start client workload..."); + + string commandArguments = this.GetCommandLineArguments(cancellationToken); + await this.ExecuteWorkloadAsync(commandArguments, this.PackageDirectory, telemetryContext, cancellationToken); + } + }); + + await Task.WhenAll(clientWorkloadTask); + + if (this.WarmUp) + { + this.IsServerWarmedUp = true; + } + } + } + + /// + /// Initializes API client. + /// + protected void InitializeApiClients() + { + IApiClientManager clientManager = this.Dependencies.GetService(); + + if (!this.IsMultiRoleLayout()) + { + this.ServerApi = clientManager.GetOrCreateApiClient(IPAddress.Loopback.ToString(), IPAddress.Loopback); + } + else + { + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ServerApi); + + IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); + if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + { + this.ReverseProxyApi = null; + } + else + { + ClientInstance reverseProxyInstance = reverseProxyInstanceEnumerable.FirstOrDefault(); + this.ReverseProxyApi = clientManager.GetOrCreateApiClient(reverseProxyInstance.Name, reverseProxyInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ReverseProxyApi); + } + } + } + + /// + /// Gets Command Line Argument to start workload. + /// + protected string GetCommandLineArguments(CancellationToken cancellationToken) + { + string result = this.CommandArguments; + + Dictionary roleAndRegexKvp = new Dictionary() + { + { ClientRole.Server, new Regex(@"\{ServerIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) }, + { ClientRole.ReverseProxy, new Regex(@"\{ReverseProxyIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) }, + { ClientRole.Client, new Regex(@"\{ClientIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) } + }; + + foreach (KeyValuePair kvp in roleAndRegexKvp) + { + MatchCollection matches = kvp.Value.Matches(this.CommandArguments); + + if (matches?.Any() == true) + { + foreach (Match match in matches) + { + IEnumerable instances = this.GetLayoutClientInstances(kvp.Key, throwIfNotExists: false); + string ipAddress = instances?.FirstOrDefault()?.IPAddress ?? IPAddress.Loopback.ToString(); + result = Regex.Replace(result, match.Value, ipAddress); + } + } + } + + return result; + } + + /// + /// Execute Bombardier Executor + /// + /// Command argument to execute on workload + /// Working Directory + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected async Task ExecuteWorkloadAsync(string commandArguments, string workingDir, EventContext telemetryContext, CancellationToken cancellationToken) + { + commandArguments.ThrowIfNullOrEmpty(nameof(commandArguments)); + + string bombardierPath = this.Combine(this.PackageDirectory, this.Platform == PlatformID.Unix ? "bombardier" : "bombardier.exe"); + + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(bombardierPath); + + EventContext relatedContext = telemetryContext.Clone() + .AddContext(nameof(bombardierPath), bombardierPath) + .AddContext(nameof(commandArguments), commandArguments) + .AddContext("bindToCores", this.BindToCores) + .AddContext("coreAffinity", this.CoreAffinity); + + try + { + await (this.ClientRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => + { + try + { + DateTime startTime = DateTime.UtcNow; + IProcessProxy process = null; + ProcessAffinityConfiguration affinityConfig = null; + + if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) + { + affinityConfig = ProcessAffinityConfiguration.Create( + this.Platform, + this.CoreAffinity); + + relatedContext.AddContext("affinityMask", affinityConfig.ToString()); + + if (this.Platform == PlatformID.Win32NT) + { + process = this.SystemManagement.ProcessManager.CreateProcess( + bombardierPath, + commandArguments, + workingDir); + } + else + { + LinuxProcessAffinityConfiguration linuxConfig = (LinuxProcessAffinityConfiguration)affinityConfig; + string fullCommandLine = $"{bombardierPath} {commandArguments}"; + string wrappedCommand = linuxConfig.GetCommandWithAffinity(null, fullCommandLine); + + process = this.SystemManagement.ProcessManager.CreateProcess( + "/bin/bash", + $"-c {wrappedCommand}", + workingDir); + } + } + else + { + process = await this.ExecuteCommandAsync(bombardierPath, commandArguments, workingDir, telemetryContext, cancellationToken, runElevated: true); + } + + using (process) + { + if (affinityConfig != null) + { + if (this.Platform == PlatformID.Win32NT) + { + process.Start(); + process.ApplyAffinity((WindowsProcessAffinityConfiguration)affinityConfig); + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + } + else + { + await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "Bombardier", logToFile: true); + process.ThrowIfWorkloadFailed(); + + if (process.StandardOutput.Length == 0) + { + throw new WorkloadException($"{this.PackageName} did not write metrics to console.", ErrorReason.CriticalWorkloadFailure); + } + + if (!this.WarmUp) + { + await this.CaptureMetricsAsync(process, commandArguments, relatedContext, cancellationToken).ConfigureAwait(false); + } + } + } + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.WorkloadStartError", LogLevel.Warning, telemetryContext.Clone().AddError(exc)); + throw; + } + }); + } + catch (OperationCanceledException) + { + this.Logger.LogMessage($"{this.TypeName}.OperationCanceledException", LogLevel.Warning, telemetryContext.Clone()); + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.ExecuteWorkloadError", LogLevel.Error, telemetryContext.Clone().AddError(exc)); + throw; + } + } + + /// + /// Get Bombardier Version + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// Bombardier Version + protected async Task GetBombardierVersionAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + string bombardierPath = this.Combine(this.PackageDirectory, this.Platform == PlatformID.Unix ? "bombardier" : "bombardier.exe"); + string bombardierVersion = null; + + try + { + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(bombardierPath); + + string commandArguments = "--version"; + string versionPattern = @"bombardier\s+(?:version\s+)?v?(\d+\.\d+\.\d+)"; + Regex versionRegex = new Regex(versionPattern, RegexOptions.Compiled | RegexOptions.IgnoreCase); + + using (IProcessProxy process = await this.ExecuteCommandAsync(bombardierPath, commandArguments, workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false)) + { + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "BombardierVersion", logToFile: true).ConfigureAwait(false); + string output = process.StandardOutput.ToString(); + Match match = versionRegex.Match(output); + + if (!match.Success) + { + output = process.StandardError.ToString(); + match = versionRegex.Match(output); + } + + if (match.Success) + { + bombardierVersion = match.Groups[1].Value; + telemetryContext.AddContext("BombardierVersion", bombardierVersion); + this.Logger.LogMessage($"{this.TypeName}.BombardierVersionCaptured", LogLevel.Information, telemetryContext); + } + } + } + } + catch (Exception exc) + { + this.Logger.LogErrorMessage(exc, telemetryContext); + } + + return bombardierVersion; + } + + private async Task CaptureMetricsAsync(IProcessProxy workloadProcess, string commandArguments, EventContext context, CancellationToken cancellationToken) + { + if (!cancellationToken.IsCancellationRequested) + { + if (workloadProcess.ExitCode == 0) + { + EventContext telemetryContext = context.Clone(); + telemetryContext.AddContext(nameof(this.MetricScenario), this.MetricScenario); + telemetryContext.AddContext(nameof(this.Scenario), this.Scenario); + + BombardierMetricsParser resultsParser = new BombardierMetricsParser(workloadProcess.StandardOutput.ToString()); + IList metrics = resultsParser.Parse(); + + foreach (var metric in metrics) + { + metric.Metadata.Add("bindToCores", this.BindToCores.ToString()); + if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) + { + metric.Metadata.Add("coreAffinity", this.CoreAffinity); + } + } + + string bombardierVersion = await this.GetBombardierVersionAsync(telemetryContext, cancellationToken).ConfigureAwait(false); + + this.MetadataContract.AddForScenario( + toolName: this.PackageName, + toolArguments: commandArguments, + toolVersion: bombardierVersion, + packageName: this.PackageName, + packageVersion: null, + additionalMetadata: null); + this.MetadataContract.Apply(telemetryContext); + + this.Logger.LogMetrics( + toolName: this.PackageName, + scenarioName: this.MetricScenario ?? this.Scenario, + workloadProcess.StartTime, + workloadProcess.ExitTime, + metrics: metrics, + metricCategorization: null, + scenarioArguments: commandArguments, + tags: this.Tags, + eventContext: telemetryContext); + } + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierMetricsParser.cs similarity index 100% rename from src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs rename to src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierMetricsParser.cs diff --git a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxCommand.cs b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxCommand.cs new file mode 100644 index 0000000000..c09743d927 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxCommand.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + /// + /// Common Nginx Commands + /// Explore more here: https://nginx.org/en/docs/switches.html + /// + public enum NginxCommand + { + /// + /// "service nginx start" + /// + Start, + + /// + /// "service nginx stop" + /// shut down quickly + /// + Stop, + + /// + /// "nginx -T"; + /// -V: print nginx version, compiler version, and configure parameters. + /// -T: Test configuration file. And dump it. + /// Use standard output to read + /// + GetVersion, + + /// + /// "nginx -T"; + /// -T: Test configuration file. And dump it. + /// Use standard output to read + /// + GetConfig + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs new file mode 100644 index 0000000000..78e51700f4 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using VirtualClient.Common.Extensions; + using VirtualClient.Contracts; + + /// + /// Extension method for Nginx Workload. + /// + public static class NginxExtensions + { + /// + /// Checks if workload has expired from the state's "Timeout" property + /// + /// + /// + public static bool IsExpired(this State state) + { + return state.Timeout() < DateTime.UtcNow; + } + + /// + /// Gets or sets the 'Timeout' property value from the state. + /// + public static DateTime Timeout(this State state, DateTime? value = null) + { + state.ThrowIfNull("state"); + if (value != null) + { + state.Properties["Timeout"] = value.Value; + } + + return state.Properties.GetValue("Timeout"); + } + + /// + /// Returns nginx commands to start the process. + /// + /// + /// + /// + public static string ConvertToCommandArgs(this NginxCommand command) + { + switch (command) + { + case NginxCommand.Start: + return "systemctl restart nginx"; + + case NginxCommand.Stop: + return "systemctl stop nginx"; + + case NginxCommand.GetVersion: + return "nginx -V"; + + case NginxCommand.GetConfig: + return "nginx -T"; + + default: + throw new WorkloadException($"Unable to convert {nameof(NginxCommand)} enum into string value. Value: {Enum.GetName(command)} - {command}"); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs new file mode 100644 index 0000000000..3e1b110de6 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs @@ -0,0 +1,361 @@ +// Copyright (c) Microsoft Corporation. +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.IO.Abstractions; + using System.Linq; + using System.Net; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.Platform; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// The NGINX Server Executor + /// + [UnixCompatible] + public class NginxServerExecutor : VirtualClientComponent + { + private TimeSpan pollingInterval = TimeSpan.FromSeconds(120); + + /// + /// Constructor + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public NginxServerExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + this.SystemManagement = this.Dependencies.GetService(); + this.pollingInterval = parameters.GetTimeSpanValue(nameof(this.pollingInterval), TimeSpan.FromSeconds(60)); + } + + /// + /// Number workers used. Number cores nginx can use. Set to null or 0 to use all cores. + /// + public int Workers + { + get + { + return this.Parameters.GetValue(nameof(this.Workers), 0); + } + } + + /// + /// Polling Timeout + /// + public TimeSpan Timeout + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.Timeout), TimeSpan.FromMinutes(30)); + } + } + + /// + /// File size to transport between Client and Server + /// + public int FileSizeInKB + { + get + { + return this.Parameters.GetValue(nameof(this.FileSizeInKB), 1); + } + } + + /// + /// The role of current instance + /// + public string Role + { + get + { + return this.Parameters.GetValue(nameof(this.Role)); + } + } + + /// + /// Provides components and services necessary for interacting with the local system and environment. + /// + protected ISystemManagement SystemManagement { get; } + + /// + /// API Client that is used to communicate with self-hosted instance of the Virtual Client. + /// + protected IApiClient ServerApi { get; set; } + + /// + /// API Client that is used to communicate with client-hosted instance of the Virtual Client Client. + /// + protected IApiClient ClientApi { get; set; } + + /// + /// The path to the Nginx package. + /// + protected string PackageDirectory { get; set; } + + /// + /// Initializes the API dependencies for running Nginx Server + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + this.Logger.LogTraceMessage($"Role: {this.Roles}; Layout: {this.GetLayoutClientInstances}"); + + PlatformSpecifics.ThrowIfNotSupported(this.CpuArchitecture); + if (this.Platform != PlatformID.Unix) + { + this.Logger.LogNotSupported(this.PackageName, this.Platform, this.CpuArchitecture, EventContext.Persisted()); + throw new NotSupportedException($"The OS/system platform '{this.Platform}' is not supported."); + } + + IApiClientManager clientManager = this.Dependencies.GetService(); + + ClientInstance serverInstance = this.GetLayoutClientInstances(this.Role).First(); + IPAddress.TryParse(serverInstance.IPAddress, out IPAddress serverIPAddress); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + + ClientInstance clientInstance = this.GetLayoutClientInstances(ClientRole.Client).First(); + IPAddress.TryParse(clientInstance.IPAddress, out IPAddress clientIPAddress); + this.ClientApi = clientManager.GetOrCreateApiClient(clientInstance.Name, clientInstance); + + DependencyPath workloadPackage = await this.SystemManagement.PackageManager.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); + if (workloadPackage == null) + { + throw new DependencyException($"{this.TypeName} did not find package ({this.PackageName}) in the packages directory.", ErrorReason.WorkloadDependencyMissing); + } + + workloadPackage = this.PlatformSpecifics.ToPlatformSpecificPath(workloadPackage, this.Platform, this.CpuArchitecture); + this.PackageDirectory = workloadPackage.Path; + + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.PackageDirectory, "setup-reset.sh")); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.PackageDirectory, "setup-config.sh")); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.PackageDirectory, "setup-content.sh")); + + string resetFilePath = this.PlatformSpecifics.Combine(this.PackageDirectory, "reset.sh"); + + if (!this.SystemManagement.FileSystem.File.Exists(resetFilePath)) + { + IProcessProxy process1 = await this.ExecuteCommandAsync(command: "bash", commandArguments: "setup-reset.sh", workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true); + await this.LogProcessDetailsAsync(process1, telemetryContext, this.PackageDirectory, logToFile: true); + process1.ThrowIfWorkloadFailed(); + telemetryContext.AddContext("resetContent", process1.StandardOutput); + + using (FileSystemStream fileStream = this.SystemManagement.FileSystem.FileStream.New(resetFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite)) + { + byte[] bytedata = Encoding.Default.GetBytes(process1.StandardOutput.ToString()); + fileStream.Write(bytedata, 0, bytedata.Length); + await fileStream.FlushAsync().ConfigureAwait(false); + fileStream.Close(); + fileStream.Dispose(); + this.Logger.LogTraceMessage($"File Created...{resetFilePath}"); + } + } + + IProcessProxy process2 = await this.ExecuteCommandAsync(command: "bash", commandArguments: $"setup-content.sh {this.FileSizeInKB}", workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process2, telemetryContext, this.PackageDirectory, logToFile: true); + process2.ThrowIfWorkloadFailed(); + + ClientInstance backendInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + IPAddress.TryParse(backendInstance.IPAddress, out IPAddress backendIPAddress); + IProcessProxy process3 = await this.ExecuteCommandAsync(command: "bash", commandArguments: $"setup-config.sh {((this.Workers != 0) ? this.Workers : "auto")} {this.Role} {backendIPAddress}", workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process3, telemetryContext, this.PackageDirectory, logToFile: true); + process3.ThrowIfWorkloadFailed(); + + await this.ServerApi.DeleteStateAsync("version", cancellationToken).ConfigureAwait(false); + await this.ServerApi.DeleteStateAsync(nameof(State), cancellationToken).ConfigureAwait(false); + this.SetServerOnline(true); + this.Logger.LogTraceMessage($"{this.TypeName} Initialize Complete."); + } + + /// + /// Executes component logic + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + telemetryContext + .AddContext("currentDirectory", Environment.CurrentDirectory) + .AddContext("toolName", "nginx") + .AddContext("timeout", this.Timeout); + + if (!cancellationToken.IsCancellationRequested) + { + try + { + Dictionary nginxVersion = await this.GetNginxVersionAsync(telemetryContext, cancellationToken).ConfigureAwait(false); + State nginxState = new State(); + nginxVersion.Any(x => nginxState.Properties.TryAdd(x.Key, x.Value)); + await this.ServerApi.CreateStateAsync("version", nginxState, cancellationToken).ConfigureAwait(false); + telemetryContext.AddContext(nameof(nginxVersion), nginxVersion); + + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) + { + await this.ExecuteNginxCommandAsync(NginxCommand.Start, workingDirectory: null, telemetryContext, cancellationToken).ConfigureAwait(false); + + Item serverState = new Item(nameof(State), new State()); + serverState.Definition.Timeout(DateTime.UtcNow.Add(this.Timeout)); + serverState.Definition.Online(true); + await this.ServerApi.CreateStateAsync(nameof(State), serverState.Definition, cancellationToken).ConfigureAwait(false); + + while (!(serverState.Definition.Timeout() < DateTime.UtcNow)) + { + EventContext relatedContext = telemetryContext + .Clone() + .AddContext(nameof(serverState), serverState); + + await this.ClientApi.PollForExpectedStateAsync(nameof(State), (state => state.Online() == true), this.Timeout, cancellationToken, this.pollingInterval).ConfigureAwait(false); + Item clientState = await this.ClientApi.GetStateAsync(nameof(State), cancellationToken).ConfigureAwait(false); + relatedContext.AddContext(nameof(clientState), clientState); + + await Task.Delay(this.pollingInterval, cancellationToken); + DateTime timeout = clientState.Definition.Properties.GetValue("Timeout", serverState.Definition.Timeout()); + + serverState.Definition.Timeout(timeout); + await this.ServerApi.UpdateStateAsync(nameof(State), serverState, cancellationToken).ConfigureAwait(false); + relatedContext.AddContext(nameof(serverState), serverState); + } + } + } + catch (Exception exc) + { + this.Logger.LogErrorMessage(exc, telemetryContext); + throw; + } + finally + { + await this.ResetNginxAsync(telemetryContext, cancellationToken).ConfigureAwait(false); + } + } + } + + /// + /// Disposes of resources used by the executor including resetting server. + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + Task.Run(async () => + { + await this.ResetNginxAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + }).Wait(TimeSpan.FromSeconds(30)); + } + catch (AggregateException) + { + // Best-effort cleanup during dispose; exceptions are intentionally swallowed. + } + } + } + + /// + /// Reset Server for Nginx + /// + /// Provides context information to include with telemetry events. + /// A token that can be used to cancel the operation. + protected async Task ResetNginxAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + try + { + IProcessProxy process1 = await this.ExecuteCommandAsync(command: "bash", commandArguments: "reset.sh", workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process1, telemetryContext, this.PackageName, logToFile: true); + + await this.ExecuteNginxCommandAsync(NginxCommand.Stop, workingDirectory: null, telemetryContext, cancellationToken).ConfigureAwait(false); + + State serverState = new State(); + serverState.Online(false); + serverState.Properties["ResetTime"] = DateTime.UtcNow.ToString(); + await this.ServerApi.UpdateStateAsync(nameof(State), new Item(nameof(State), serverState), cancellationToken).ConfigureAwait(false); + } + catch + { + this.Logger.LogTraceMessage("Failed to reset server."); + } + } + + /// + /// Executes a command to get Nginx Version installed from the system. + /// + /// Provides context information to include with telemetry events. + /// A token that can be used to cancel the process execution. + /// NginxVersion + protected async Task> GetNginxVersionAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + IProcessProxy process = await this.ExecuteNginxCommandAsync(NginxCommand.GetVersion, null, telemetryContext, cancellationToken).ConfigureAwait(false); + string standardErr = process.StandardError.ToString(); + + return this.TransformNginxVersionToDictionary(standardErr); + } + + private Dictionary TransformNginxVersionToDictionary(string processOutput) + { + processOutput.ThrowIfNullOrEmpty(nameof(processOutput)); + string[] standardOutputSections = processOutput.Split(Environment.NewLine, StringSplitOptions.TrimEntries); + + string nginxVersion = + standardOutputSections + .Where(x => x.Contains("nginx version", StringComparison.OrdinalIgnoreCase)).FirstOrDefault() + .Split(":").Last().Trim(); + + nginxVersion.ThrowIfNullOrEmpty(nameof(nginxVersion), $"{nameof(processOutput)} does not contain nginx version. {nameof(processOutput)}: {processOutput}"); + + string sslVersion = + standardOutputSections + .Where(x => x.Contains("OpenSSL", StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + + string serverNameIndicationSupport = + standardOutputSections + .Where(x => x.Contains("TLS", StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + + string arguments = + standardOutputSections + .Where(x => x.Contains("configure arguments", StringComparison.OrdinalIgnoreCase)).FirstOrDefault() + ?.Split(":").Last().Trim(); + + nginxVersion.ThrowIfNullOrEmpty(nameof(nginxVersion), $"{nameof(processOutput)} does not contain nginx version. {nameof(processOutput)}: {processOutput}"); + return new Dictionary + { + { nameof(nginxVersion), nginxVersion }, + { nameof(sslVersion), sslVersion }, + { nameof(serverNameIndicationSupport), serverNameIndicationSupport }, + { nameof(arguments), arguments } + }; + } + + private Task ExecuteNginxCommandAsync(NginxCommand nginxCommand, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken) + { + nginxCommand.ThrowIfNull("nginxCommand"); + telemetryContext.ThrowIfNull("telemetryContext"); + + telemetryContext.AddContext(nameof(nginxCommand), nginxCommand); + string commandArgs = nginxCommand.ConvertToCommandArgs(); + telemetryContext.AddContext(nameof(commandArgs), $"{commandArgs}"); + + if (this.Platform != PlatformID.Unix) + { + throw new NotSupportedException($"Nginx command is not supported on '{this.Platform}' platform/architecture systems."); + } + + return this.Logger.LogMessageAsync($"{nameof(this.TypeName)}.ExecuteNginxCommand", telemetryContext, async () => + { + IProcessProxy process = await this.ExecuteCommandAsync(command: "sudo", commandArguments: commandArgs, workingDirectory: workingDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process, telemetryContext, logToFile: true); + process.ThrowIfWorkloadFailed(); + return process; + }); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/VirtualClient.Actions.csproj b/src/VirtualClient/VirtualClient.Actions/VirtualClient.Actions.csproj index 038749f5b7..31975ac868 100644 --- a/src/VirtualClient/VirtualClient.Actions/VirtualClient.Actions.csproj +++ b/src/VirtualClient/VirtualClient.Actions/VirtualClient.Actions.csproj @@ -29,6 +29,7 @@ + diff --git a/src/VirtualClient/VirtualClient.Actions/Wrk/Wrk2Executor.cs b/src/VirtualClient/VirtualClient.Actions/Wrk/Wrk2Executor.cs new file mode 100644 index 0000000000..38769028e7 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Wrk/Wrk2Executor.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// The Wrk Client executor. + /// + [SupportedPlatforms("linux-arm64,linux-x64")] + public class Wrk2Executor : WrkExecutor + { + /// + /// Constructor + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public Wrk2Executor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + } + + /// + /// Initializes the executor dependencies, package locations, server api, etc... + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + DependencyPath workloadPackage = await this.SystemManagement.PackageManager.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); + DependencyPath scriptPackage = await this.SystemManagement.PackageManager.GetPackageAsync(Wrk2Executor.WrkConfiguration, cancellationToken).ConfigureAwait(false); + + if (workloadPackage == null || this.PackageName != "wrk2") + { + throw new DependencyException($"{this.TypeName} did not find correct package in the directory. Supported Package: wrk2. Package Provided: {this.PackageName}", ErrorReason.WorkloadDependencyMissing); + } + + if (scriptPackage == null) + { + throw new DependencyException($"{this.TypeName} did not find package ({WrkExecutor.WrkConfiguration}) in the packages directory.", ErrorReason.WorkloadDependencyMissing); + } + + this.PackageDirectory = workloadPackage.Path; + this.ScriptDirectory = this.PlatformSpecifics.ToPlatformSpecificPath(scriptPackage, this.Platform, this.CpuArchitecture).Path; + + this.InitializeApiClients(); + await this.SetupWrkClient(telemetryContext, cancellationToken).ConfigureAwait(false); + } + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs new file mode 100644 index 0000000000..246130ca7d --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs @@ -0,0 +1,585 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.IO.Abstractions; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Text; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + using VirtualClient.Contracts.Metadata; + + /// + /// The Wrk Client executor. + /// + [SupportedPlatforms("linux-arm64,linux-x64")] + public class WrkExecutor : VirtualClientMultiRoleComponent + { + internal const string WrkConfiguration = "wrkconfiguration"; + internal const string WrkRunShell = "runwrk.sh"; + + /// + /// Constructor + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public WrkExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + this.ClientFlowRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(2, (retries) => TimeSpan.FromSeconds(retries * 2)); + + this.ClientRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(2, (retries) => TimeSpan.FromSeconds(retries)); + + this.SystemManagement = this.Dependencies.GetService(); + } + + /// + /// API Client that is used to communicate with server-hosted instance of the Virtual Client Server. + /// + public IApiClient ServerApi { get; set; } + + /// + /// API Client that is used to communicate with ReverseProxy instance of the Virtual Client Server. + /// + public IApiClient ReverseProxyApi { get; set; } + + /// + /// Provides components and services necessary for interacting with the local system and environment. + /// + public ISystemManagement SystemManagement { get; } + + /// + /// Option for testing webserver (default), reverse-proxy (rp), api-gateway (apigw) + /// + public string TargetService + { + get + { + switch (this.Parameters.GetValue(nameof(this.TargetService), string.Empty).ToLower()) + { + case "reverse-proxy": + case "rp": + return "rp"; + case "apigw": + case "api-gateway": + return "apigw"; + case "server": + return "server"; + default: + IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); + if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + { + return "server"; + } + else + { + return "rp"; + } + } + } + } + + /// + /// The command line argument defined in the profile. + /// + public string CommandArguments + { + get + { + return this.Parameters.GetValue(nameof(this.CommandArguments)); + } + } + + /// + /// Gets or sets whether to bind the workload to specific CPU cores. + /// + public bool BindToCores + { + get + { + return this.Parameters.GetValue(nameof(this.BindToCores), defaultValue: false); + } + } + + /// + /// Gets the CPU core affinity specification (e.g., "0-3", "0,2,4,6"). + /// + public string CoreAffinity + { + get + { + this.Parameters.TryGetValue(nameof(this.CoreAffinity), out IConvertible value); + return value?.ToString(); + } + } + + /// + /// Emit Latency Spectrum as additional metrics + /// + public bool EmitLatencySpectrum + { + get + { + return this.Parameters.GetValue(nameof(this.EmitLatencySpectrum), false); + } + } + + /// + /// Polling Timeout + /// + public TimeSpan Timeout + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.Timeout), TimeSpan.FromMinutes(5)); + } + } + + /// + /// Parameter defines true/false whether the action is meant to warm up the server. + /// We do not capture metrics on warm up operations. + /// + public bool WarmUp + { + get + { + return this.Parameters.GetValue(nameof(this.WarmUp), false); + } + } + + /// + /// The path to the Wrk package. + /// + public string PackageDirectory { get; set; } + + /// + /// The path to wrk scripts. + /// + public string ScriptDirectory { get; set; } + + /// + /// True/false whether the server instance has been warmed up. + /// + protected bool IsServerWarmedUp { get; set; } + + /// + /// The retry policy to apply to the client-side execution workflow. + /// + protected IAsyncPolicy ClientFlowRetryPolicy { get; set; } + + /// + /// The retry policy to apply to each Memtier workload instance when trying to startup + /// against a target server. + /// + protected IAsyncPolicy ClientRetryPolicy { get; set; } + + /// + /// Initializes the executor dependencies, package locations, server api, etc... + /// + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + DependencyPath workloadPackage = await this.SystemManagement.PackageManager.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); + DependencyPath scriptPackage = await this.SystemManagement.PackageManager.GetPackageAsync(WrkExecutor.WrkConfiguration, cancellationToken).ConfigureAwait(false); + + if (this.PackageName != "wrk" || workloadPackage == null) + { + throw new DependencyException($"{this.TypeName} did not find correct package in the directory. Supported Package: wrk. Package Provided: {this.PackageName}", ErrorReason.WorkloadDependencyMissing); + } + + if (scriptPackage == null) + { + throw new DependencyException($"{this.TypeName} did not find package ({WrkExecutor.WrkConfiguration}) in the packages directory.", ErrorReason.WorkloadDependencyMissing); + } + + this.PackageDirectory = workloadPackage.Path; + this.ScriptDirectory = this.PlatformSpecifics.ToPlatformSpecificPath(scriptPackage, this.Platform, this.CpuArchitecture).Path; + + this.InitializeApiClients(); + await this.SetupWrkClient(telemetryContext, cancellationToken).ConfigureAwait(false); + } + + /// + /// Validates the component parameters. + /// + protected override void Validate() + { + base.Validate(); + + if (this.BindToCores) + { + this.ThrowIfParameterNotDefined(nameof(this.CoreAffinity)); + } + } + + /// + /// Executes component logic + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + if (!this.WarmUp || !this.IsServerWarmedUp) + { + Task clientWorkloadTask; + + clientWorkloadTask = this.ClientFlowRetryPolicy.ExecuteAsync(async () => + { + if (!cancellationToken.IsCancellationRequested) + { + this.Logger.LogTraceMessage("Synchronization: Poll server API for heartbeat..."); + await this.ServerApi.PollForHeartbeatAsync(this.PollingTimeout, cancellationToken); + this.Logger.LogTraceMessage("Synchronization: Poll server for online signal..."); + + await this.ServerApi.PollForExpectedStateAsync(nameof(State), (state => state.Online()), this.Timeout, cancellationToken).ConfigureAwait(false); + + this.Logger.LogTraceMessage("Synchronization: Server online signal confirmed..."); + + // verify ReverseProxy is online + if ((this.ReverseProxyApi != null) && ((this.TargetService == "rp") || (this.TargetService == "apigw"))) + { + this.Logger.LogTraceMessage("Synchronization: Poll ReverseProxy for online signal..."); + await this.ReverseProxyApi.PollForExpectedStateAsync(nameof(State), (state => state.Online()), TimeSpan.FromMinutes(10), cancellationToken).ConfigureAwait(false); + this.Logger.LogTraceMessage("Synchronization: ReverseProxy online signal confirmed..."); + Item reverseProxyState = await this.ReverseProxyApi.GetStateAsync(nameof(State), cancellationToken).ConfigureAwait(false); + telemetryContext.AddContext(nameof(reverseProxyState), reverseProxyState); + + HttpResponseMessage reverseProxyHttpResponseMessage = await this.ReverseProxyApi.GetStateAsync("version", cancellationToken).ConfigureAwait(continueOnCapturedContext: false); + telemetryContext.AddResponseContext(reverseProxyHttpResponseMessage); + + if (reverseProxyHttpResponseMessage.IsSuccessStatusCode) + { + Item reverseProxyVersion = await reverseProxyHttpResponseMessage.FromContentAsync>(); + telemetryContext.AddContext(nameof(reverseProxyVersion), reverseProxyVersion.Definition.Properties); + } + } + + this.Logger.LogTraceMessage("Synchronization: Start client workload..."); + + string commandArguments = this.GetCommandLineArguments(cancellationToken); + await this.ExecuteWorkloadAsync(commandArguments, this.PackageDirectory, telemetryContext, cancellationToken); + } + }); + + await Task.WhenAll(clientWorkloadTask); + + if (this.WarmUp) + { + this.IsServerWarmedUp = true; + } + } + } + + /// + /// Setup Wrk Executor + /// + protected async Task SetupWrkClient(EventContext telemetryContext, CancellationToken cancellationToken) + { + this.PackageDirectory.ThrowIfNullOrWhiteSpace(nameof(this.PackageDirectory)); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.PackageDirectory, "wrk")); + await this.SystemManagement.MakeFileExecutableAsync(this.PlatformSpecifics.Combine(this.PackageDirectory, "wrk"), this.Platform, cancellationToken).ConfigureAwait(false); + + string shellScriptPath = this.PlatformSpecifics.GetScriptPath("wrk", WrkExecutor.WrkRunShell); + this.SystemManagement.FileSystem.File.Copy(shellScriptPath, this.Combine(this.PackageDirectory, WrkExecutor.WrkRunShell), true); + + this.ScriptDirectory.ThrowIfNullOrWhiteSpace(nameof(this.ScriptDirectory)); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.ScriptDirectory, "setup-reset.sh")); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.ScriptDirectory, "setup-config.sh")); + + string resetFilePath = this.PlatformSpecifics.Combine(this.ScriptDirectory, "reset.sh"); + if (!this.SystemManagement.FileSystem.File.Exists(resetFilePath)) + { + IProcessProxy process1 = await this.ExecuteCommandAsync(command: "bash", commandArguments: "setup-reset.sh", workingDirectory: this.ScriptDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process1, telemetryContext, WrkExecutor.WrkConfiguration, logToFile: true); + process1.ThrowIfWorkloadFailed(); + + using (FileSystemStream fileStream = this.SystemManagement.FileSystem.FileStream.New(resetFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite)) + { + byte[] bytedata = Encoding.Default.GetBytes(process1.StandardOutput.ToString()); + fileStream.Write(bytedata, 0, bytedata.Length); + await fileStream.FlushAsync().ConfigureAwait(false); + this.Logger.LogTraceMessage($"File Created...{resetFilePath}"); + } + } + + // set up Config + IProcessProxy process2 = await this.ExecuteCommandAsync(command: "bash", commandArguments: "setup-config.sh", workingDirectory: this.ScriptDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process2, telemetryContext, WrkExecutor.WrkConfiguration, logToFile: true); + process2.ThrowIfWorkloadFailed(); + } + + /// + /// Initializes API client. + /// + protected void InitializeApiClients() + { + IApiClientManager clientManager = this.Dependencies.GetService(); + + if (!this.IsMultiRoleLayout()) + { + this.ServerApi = clientManager.GetOrCreateApiClient(IPAddress.Loopback.ToString(), IPAddress.Loopback); + } + else + { + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ServerApi); + + IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); + if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + { + this.ReverseProxyApi = null; + } + else + { + ClientInstance reverseProxyInstance = reverseProxyInstanceEnumerable.FirstOrDefault(); + this.ReverseProxyApi = clientManager.GetOrCreateApiClient(reverseProxyInstance.Name, reverseProxyInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ReverseProxyApi); + } + } + } + + /// + /// Gets Command Line Argument to start workload. + /// + protected string GetCommandLineArguments(CancellationToken cancellationToken) + { + string result = this.CommandArguments; + + Dictionary roleAndRegexKvp = new Dictionary() + { + { ClientRole.Server, new Regex(@"\{ServerIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) }, + { ClientRole.ReverseProxy, new Regex(@"\{ReverseProxyIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) }, + { ClientRole.Client, new Regex(@"\{ClientIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) } + }; + + foreach (KeyValuePair kvp in roleAndRegexKvp) + { + MatchCollection matches = kvp.Value.Matches(this.CommandArguments); + + if (matches?.Any() == true) + { + foreach (Match match in matches) + { + IEnumerable instances = this.GetLayoutClientInstances(kvp.Key, throwIfNotExists: false); + string ipAddress = instances?.FirstOrDefault()?.IPAddress ?? IPAddress.Loopback.ToString(); + result = Regex.Replace(result, match.Value, ipAddress); + } + } + } + + return result; + } + + /// + /// Execute Wrk Executor + /// + /// Command argument ti execute on workload + /// Working Directory + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected async Task ExecuteWorkloadAsync(string commandArguments, string workingDir, EventContext telemetryContext, CancellationToken cancellationToken) + { + commandArguments.ThrowIfNullOrEmpty(nameof(commandArguments)); + string scriptPath = this.Combine(this.PackageDirectory, WrkExecutor.WrkRunShell); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(scriptPath); + string command = $"bash {scriptPath}"; + + EventContext relatedContext = telemetryContext.Clone() + .AddContext(nameof(command), command) + .AddContext(nameof(commandArguments), commandArguments) + .AddContext("bindToCores", this.BindToCores) + .AddContext("coreAffinity", this.CoreAffinity); + + try + { + await (this.ClientRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => + { + try + { + DateTime startTime = DateTime.UtcNow; + IProcessProxy process = null; + ProcessAffinityConfiguration affinityConfig = null; + + if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) + { + affinityConfig = ProcessAffinityConfiguration.Create( + this.Platform, + this.CoreAffinity); + + relatedContext.AddContext("affinityMask", affinityConfig.ToString()); + + string fullCommandLine = $"{command} \"{commandArguments}\""; + LinuxProcessAffinityConfiguration linuxConfig = (LinuxProcessAffinityConfiguration)affinityConfig; + string wrappedCommand = linuxConfig.GetCommandWithAffinity(null, fullCommandLine); + + process = this.SystemManagement.ProcessManager.CreateProcess( + "/bin/bash", + $"-c {wrappedCommand}", + workingDir); + } + else + { + process = await this.ExecuteCommandAsync(command, commandArguments: $"\"{commandArguments}\"", workingDir, telemetryContext, cancellationToken, runElevated: true); + } + + using (process) + { + if (affinityConfig != null) + { + await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); + } + + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "Wrk", logToFile: true); + process.ThrowIfWorkloadFailed(); + + if (process.StandardOutput.Length == 0) + { + throw new WorkloadException($"{this.PackageName} did not write metrics to console.", ErrorReason.CriticalWorkloadFailure); + } + + if (!this.WarmUp) + { + await this.CaptureMetricsAsync(process, commandArguments, this.EmitLatencySpectrum, relatedContext, cancellationToken).ConfigureAwait(false); + } + } + } + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.WorkloadStartError", LogLevel.Warning, telemetryContext.Clone().AddError(exc)); + throw; + } + }); + } + catch (OperationCanceledException) + { + this.Logger.LogMessage($"{this.TypeName}.OperationCanceledException", LogLevel.Warning, telemetryContext.Clone()); + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.ExecuteWorkloadError", LogLevel.Error, telemetryContext.Clone().AddError(exc)); + throw; + } + } + + /// + /// Get Wrk Version + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// Wrk Version + protected async Task GetWrkVersionAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + string wrkVersion = null; + + try + { + string scriptPath = this.Combine(this.PackageDirectory, WrkExecutor.WrkRunShell); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(scriptPath); + + string command = $"bash {scriptPath}"; + string commandArguments = "--version"; + string versionPattern = @"wrk\s(\d+\.\d+\.\d+)"; + Regex versionRegex = new Regex(versionPattern, RegexOptions.Compiled); + + using (IProcessProxy process = await this.ExecuteCommandAsync(command, commandArguments, workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false)) + { + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "WrkVersion", logToFile: true).ConfigureAwait(false); + string output = process.StandardOutput.ToString(); + Match match = versionRegex.Match(output); + + if (!match.Success) + { + output = process.StandardError.ToString(); + match = versionRegex.Match(output); + } + + if (match.Success) + { + wrkVersion = match.Groups[1].Value; + telemetryContext.AddContext("WrkVersion", wrkVersion); + this.Logger.LogMessage($"{this.TypeName}.WrkVersionCaptured", LogLevel.Information, telemetryContext); + } + } + } + } + catch (Exception exc) + { + this.Logger.LogErrorMessage(exc, telemetryContext); + } + + return wrkVersion; + } + + private async Task CaptureMetricsAsync(IProcessProxy workloadProcess, string commandArguments, bool emitLatencySpectrum, EventContext context, CancellationToken cancellationToken) + { + if (!cancellationToken.IsCancellationRequested) + { + if (workloadProcess.ExitCode == 0) + { + EventContext telemetryContext = context.Clone(); + telemetryContext.AddContext(nameof(this.MetricScenario), this.MetricScenario); + telemetryContext.AddContext(nameof(this.Scenario), this.Scenario); + + WrkMetricParser resultsParser = new WrkMetricParser(workloadProcess.StandardOutput.ToString()); + IList metrics = resultsParser.Parse(emitLatencySpectrum); + + foreach (var metric in metrics) + { + metric.Metadata.Add("bindToCores", this.BindToCores.ToString()); + if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) + { + metric.Metadata.Add("coreAffinity", this.CoreAffinity); + } + } + + string wrkVersion = await this.GetWrkVersionAsync(telemetryContext, cancellationToken).ConfigureAwait(false); + + this.MetadataContract.AddForScenario( + toolName: this.PackageName, + toolArguments: commandArguments, + toolVersion: wrkVersion, + packageName: this.PackageName, + packageVersion: null, + additionalMetadata: null); + telemetryContext.AddContext("TestConfig", resultsParser.GetTestConfig()); + this.MetadataContract.Apply(telemetryContext); + + this.Logger.LogMetrics( + toolName: this.PackageName, + scenarioName: this.MetricScenario ?? this.Scenario, + workloadProcess.StartTime, + workloadProcess.ExitTime, + metrics: metrics, + metricCategorization: null, + scenarioArguments: commandArguments, + tags: this.Tags, + eventContext: telemetryContext); + } + } + } + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/WrkMetricParser.cs b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkMetricParser.cs similarity index 99% rename from src/VirtualClient/VirtualClient.Actions/ASPNET/WrkMetricParser.cs rename to src/VirtualClient/VirtualClient.Actions/Wrk/WrkMetricParser.cs index 65367622a6..75d63fd63f 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/WrkMetricParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkMetricParser.cs @@ -43,7 +43,7 @@ public WrkMetricParser(string resultsText) /// public override IList Parse() { - return this.Parse(); + return this.Parse(emitLatencySpectrum: true); } /// diff --git a/src/VirtualClient/VirtualClient.Actions/Wrk/runwrk.sh b/src/VirtualClient/VirtualClient.Actions/Wrk/runwrk.sh new file mode 100644 index 0000000000..d7c59a5c11 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Wrk/runwrk.sh @@ -0,0 +1,2 @@ +ulimit -n 65535 +./wrk $1 \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json deleted file mode 100644 index 44f2ef4970..0000000000 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "Description": ".NET benchmarking Workload", - "Metadata": { - "RecommendedMinimumExecutionTime": "00:05:00", - "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", - "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" - }, - "Parameters": { - "DotNetVersion": "8.0.204", - "TargetFramework": "net8.0", - "EnvironmentVariables": "ASPNETCORE_threadCount=1;DOTNET_SYSTEM_NET_SOCKETS_THREAD_COUNT=1", - "AspNetCoreThreadCount": 1, - "DotNetSystemNetSocketsThreadCount": 1 - }, - "Actions": [ - { - "Type": "AspNetBenchServerExecutor", - "Parameters": { - "Role": "Server", - "Scenario": "ExecuteJsonSerializationBenchmark", - "PackageName": "aspnetbenchmarks", - "BombardierPackageName": "bombardier", - "DotNetSdkPackageName": "dotnetsdk", - "TargetFramework": "$.Parameters.TargetFramework", - "AspNetCoreThreadCount": "$.Parameters.AspNetCoreThreadCount", - "DotNetSystemNetSocketsThreadCount": "$.Parameters.DotNetSystemNetSocketsThreadCount" - } - }, - { - "Type": "AspNetBenchClientExecutor", - "Parameters": { - "Role": "Client", - "Scenario": "ExecuteJsonSerializationBenchmarkWarmUp", - "PackageName": "aspnetbenchmarks", - "BombardierPackageName": "bombardier", - "DotNetSdkPackageName": "dotnetsdk", - "TargetFramework": "$.Parameters.TargetFramework", - "WrkCommandLine": "-t 256 -c 256 -d 45s --timeout 10s http://{ipAddress}:{port}/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q = 0.9,application/xml;q=0.8,*/*;q=0.7\"" - } - }, - { - "Type": "AspNetBenchClientExecutor", - "Parameters": { - "Role": "Client", - "Scenario": "ExecuteJsonSerializationBenchmark", - "PackageName": "aspnetbenchmarks", - "BombardierPackageName": "bombardier", - "DotNetSdkPackageName": "dotnetsdk", - "TargetFramework": "$.Parameters.TargetFramework", - "WrkCommandLine": "-t 256 -c 256 -d 15s --timeout 10s http://{ipAddress}:{port}/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q = 0.9,application/xml;q=0.8,*/*;q=0.7\"" - } - } - - ], - "Dependencies": [ - { - "Type": "ChocolateyInstallation", - "Parameters": { - "Scenario": "InstallChocolatey", - "PackageName": "chocolatey" - } - }, - { - "Type": "ChocolateyPackageInstallation", - "Parameters": { - "Scenario": "InstallGit", - "PackageName": "chocolatey", - "Packages": "git" - } - }, - { - "Type": "GitRepoClone", - "Parameters": { - "Scenario": "CloneAspNetBenchmarksRepo", - "RepoUri": "https://github.com/aspnet/Benchmarks.git", - "Commit": "cf5b6ee", - "PackageName": "aspnetbenchmarks" - } - }, - { - "Type": "LinuxPackageInstallation", - "Parameters": { - "Scenario": "InstallLinuxPackages", - "Packages-Apt": "build-essential,unzip", - "Role": "Client" - } - }, - { - "Type": "GitRepoClone", - "Parameters": { - "Scenario": "CloneWrkRepo", - "RepoUri": "https://github.com/wg/wrk.git", - "PackageName": "wrk", - "Role": "Client" - } - }, - { - "Type": "ExecuteCommand", - "Parameters": { - "Scenario": "CompileWrk", - "Command": "make", - "WorkingDirectory": "{PackagePath:wrk}", - "Role": "Client" - } - }, - { - "Type": "DotNetInstallation", - "Parameters": { - "Scenario": "InstallDotNetSdk", - "DotNetVersion": "$.Parameters.DotNetVersion", - "PackageName": "dotnetsdk" - } - }, - { - "Type": "ApiServer", - "Parameters": { - "Scenario": "StartAPIServer" - } - }, - { - "Type": "SetEnvironmentVariable", - "Parameters": { - "Scenario": "SetEnvironmentVariableForAspNet", - "EnvironmentVariables": "$.Parameters.EnvironmentVariables", - "Role": "Server" - } - } - ] -} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json deleted file mode 100644 index f4baf54582..0000000000 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "Description": ".NET benchmarking Workload", - "Metadata": { - "RecommendedMinimumExecutionTime": "00:05:00", - "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", - "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" - }, - "Parameters": { - "DotNetVersion": "8.0.204", - "TargetFramework": "net8.0" - }, - "Actions": [ - { - "Type": "AspNetBenchExecutor", - "Parameters": { - "Scenario": "ExecuteJsonSerializationBenchmark", - "PackageName": "aspnetbenchmarks", - "BombardierPackageName": "bombardier", - "DotNetSdkPackageName": "dotnetsdk", - "TargetFramework": "$.Parameters.TargetFramework" - } - } - ], - "Dependencies": [ - { - "Type": "ChocolateyInstallation", - "Parameters": { - "Scenario": "InstallChocolatey", - "PackageName": "chocolatey" - } - }, - { - "Type": "ChocolateyPackageInstallation", - "Parameters": { - "Scenario": "InstallGit", - "PackageName": "chocolatey", - "Packages": "git" - } - }, - { - "Type": "GitRepoClone", - "Parameters": { - "Scenario": "CloneAspNetBenchmarksRepo", - "RepoUri": "https://github.com/aspnet/Benchmarks.git", - "Commit": "cf5b6ee", - "PackageName": "aspnetbenchmarks" - } - }, - { - "Type": "DotNetInstallation", - "Parameters": { - "Scenario": "InstallDotNetSdk", - "DotNetVersion": "$.Parameters.DotNetVersion", - "PackageName": "dotnetsdk" - } - }, - { - "Type": "DependencyPackageInstallation", - "Parameters": { - "Scenario": "InstallBombardierPackage", - "BlobContainer": "packages", - "BlobName": "bombardier.1.2.5.zip", - "PackageName": "bombardier", - "Extract": true - } - } - ] -} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json new file mode 100644 index 0000000000..63a09d94ad --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json @@ -0,0 +1,87 @@ +{ + "Description": ".NET benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:05:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" + }, + "Parameters": { + "DotNetVersion": "8.0.204", + "TargetFramework": "net8.0", + "ServerPort": 9876 + }, + "Actions": [ + { + "Type": "AspNetServerExecutor", + "Parameters": { + "Role": "Server", + "Scenario": "ExecuteJsonSerializationBenchmark", + "PackageName": "aspnetbenchmarks", + "DotNetSdkPackageName": "dotnetsdk", + "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "$.Parameters.ServerPort" + } + }, + { + "Type": "BombardierExecutor", + "Parameters": { + "Role": "Client", + "Scenario": "ExecuteJsonSerializationBenchmark", + "PackageName": "bombardier", + "CommandArguments": "--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ServerIp}:9876/json --print r --format json", + "WarmUp": false, + "Timeout": "00:05:00" + } + } + ], + "Dependencies": [ + { + "Type": "ChocolateyInstallation", + "Parameters": { + "Scenario": "InstallChocolatey", + "PackageName": "chocolatey" + } + }, + { + "Type": "ChocolateyPackageInstallation", + "Parameters": { + "Scenario": "InstallGit", + "PackageName": "chocolatey", + "Packages": "git" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneAspNetBenchmarksRepo", + "RepoUri": "https://github.com/aspnet/Benchmarks.git", + "Commit": "cf5b6ee", + "PackageName": "aspnetbenchmarks" + } + }, + { + "Type": "DotNetInstallation", + "Parameters": { + "Scenario": "InstallDotNetSdk", + "DotNetVersion": "$.Parameters.DotNetVersion", + "PackageName": "dotnetsdk" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallBombardierPackage", + "BlobContainer": "packages", + "BlobName": "bombardier.1.2.5.zip", + "PackageName": "bombardier", + "Extract": true + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-ORCHARD-WRK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-ORCHARD-WRK.json new file mode 100644 index 0000000000..84d24b06dc --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-ORCHARD-WRK.json @@ -0,0 +1,161 @@ +{ + "Description": ".NET benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedOperatingSystems": "CBL-Mariner,Ubuntu" + }, + "Parameters": { + "DotNetVersion": "9.0.203", + "TargetFramework": "net9.0", + "OrchardCoreAdminValue": "Compete@CRC1", + "ServerPort": 5014, + "TestDuration": "00:00:20", + "EmitLatencySpectrum": false, + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s http://{serverip}:{ServerPort}/about --header \"Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\" --header \"Connection:keep-alive\"" + }, + "Actions": [ + { + "Type": "AspNetOrchardServerExecutor", + "Parameters": { + "Role": "Server", + "Scenario": "OrchardCore", + "PackageName": "orchardcore", + "DotNetSdkPackageName": "dotnetsdk", + "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "$.Parameters.ServerPort" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Warmup", + "MetricScenario": "Warmup_Orchard-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "ServerPort": "$.Parameters.ServerPort", + "ThreadCount": "64", + "Connection": 128, + "TestDuration": "00:02:00", + "WarmUp": "true", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Tags": "Orchard,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Benchmark_Measurement", + "MetricScenario": "Orchard-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "ServerPort": "$.Parameters.ServerPort", + "ThreadCount": "64", + "Connection": 128, + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Tags": "Orchard,WRK" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "build-essential,libevent-dev,pkg-config,zlib1g-dev,libssl-dev,autoconf,automake,make,libpcre3-dev,gcc,unzip,openssl", + "Packages-Yum": "zlib-devel,pcre-devel,libevent-devel,openssl-devel,git,gcc-c++,make,autoconf,automake", + "Packages-Dnf": "zlib-devel,libevent-devel,openssl-devel,git,gcc,gcc-c++,make,autoconf,icu,automake,unzip,binutils,libstdc++-devel,kernel-headers,glibc-devel,perl,perl-Module-CoreList" + } + }, + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "wget,build-essential,tcl-dev,numactl", + "Packages-Yum": "wget,numactl,tcl-devel", + "Packages-Dnf": "wget,numactl,tcl-devel,iptables" + } + }, + { + "Type": "ChocolateyInstallation", + "Parameters": { + "Scenario": "InstallChocolatey", + "PackageName": "chocolatey" + } + }, + { + "Type": "ChocolateyPackageInstallation", + "Parameters": { + "Scenario": "InstallGit", + "PackageName": "chocolatey", + "Packages": "git" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneOrchardCoreRepo", + "RepoUri": "https://github.com/orchardcms/orchardcore.git", + "Commit": "4e7c47c", + "PackageName": "orchardcore", + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "DotNetInstallation", + "Parameters": { + "Scenario": "InstallDotNetSdk", + "DotNetVersion": "$.Parameters.DotNetVersion", + "PackageName": "dotnetsdk", + "Role": "Server" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + }, + { + "Type": "SetEnvironmentVariable", + "Parameters": { + "Scenario": "SetEnvironmentVariableForAspNet", + "EnvironmentVariables": "OrchardCore__OrchardCore_AutoSetup__Tenants__0__ShellName=Default;OrchardCore__OrchardCore_AutoSetup__Tenants__0__SiteName=Benchmark;OrchardCore__OrchardCore_AutoSetup__Tenants__0__SiteTimeZone=Europe/Amsterdam;OrchardCore__OrchardCore_AutoSetup__Tenants__0__AdminUsername=admin;OrchardCore__OrchardCore_AutoSetup__Tenants__0__AdminEmail=info@orchardproject.net;OrchardCore__OrchardCore_AutoSetup__Tenants__0__AdminPassword={OrchardCoreAdminValue};OrchardCore__OrchardCore_AutoSetup__Tenants__0__DatabaseProvider=Sqlite;OrchardCore__OrchardCore_AutoSetup__Tenants__0__RecipeName=Blog;DOTNET_GCDynamicAdaptationMode=0;DOTNET_HillClimbing_Disable=1", + "OrchardCoreAdminValue": "$.Parameters.OrchardCoreAdminValue", + "Role": "Server" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-WRK-AFFINITY.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-WRK-AFFINITY.json new file mode 100644 index 0000000000..9cf49811f8 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-WRK-AFFINITY.json @@ -0,0 +1,172 @@ +{ + "Description": ".NET benchmarking Workload with CPU Core Affinity (supports .NET 9 and .NET 10)", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" + }, + "Parameters": { + "DotNetVersion": "9", + "TargetFramework": "net9.0", + "ServerPort": 9876, + "ServerCoreAffinity": "0-7", + "ClientCoreAffinity": "8-15", + "TestDuration": "00:00:15", + "ServerOnlineTimeout": "00:10:00", + "EmitLatencySpectrum": false, + "WaitForServer": false, + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s http://{serverip}:{ServerPort}/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\"" + }, + "ParametersOn": [ + { + "Condition": "{calculate(\"{DotNetVersion}\" == \"9\")}", + "DotNetVersion": "9.0.111", + "TargetFramework": "net9.0" + }, + { + "Condition": "{calculate(\"{DotNetVersion}\" == \"10\")}", + "DotNetVersion": "10.0.101", + "TargetFramework": "net10.0" + } + ], + "Actions": [ + { + "Type": "AspNetServerExecutor", + "Parameters": { + "Role": "Server", + "Scenario": "ExecuteJsonSerializationBenchmarkWithAffinity", + "PackageName": "aspnetbenchmarks", + "DotNetSdkPackageName": "dotnetsdk", + "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "$.Parameters.ServerPort", + "BindToCores": true, + "CoreAffinity": "$.Parameters.ServerCoreAffinity" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "warmup", + "MetricScenario": "Warmup_ASP.NET-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "ServerPort": "$.Parameters.ServerPort", + "ThreadCount": "64", + "Connection": 4096, + "TestDuration": "00:00:45", + "WaitForServer": "$.Parameters.WaitForServer", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Timeout": "$.Parameters.ServerOnlineTimeout", + "Tags": "ASP.NET,WRK", + "BindToCores": true, + "CoreAffinity": "$.Parameters.ClientCoreAffinity" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "benchmark_measurement", + "MetricScenario": "ASP.NET-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "ServerPort": "$.Parameters.ServerPort", + "ThreadCount": "64", + "Connection": 4096, + "TestDuration": "$.Parameters.TestDuration", + "WaitForServer": "$.Parameters.WaitForServer", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Timeout": "$.Parameters.ServerOnlineTimeout", + "Tags": "ASP.NET,WRK", + "BindToCores": true, + "CoreAffinity": "$.Parameters.ClientCoreAffinity" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip", + "Packages-Yum": "zlib-devel,pcre-devel,libevent-devel,openssl-devel,git,gcc-c++,make,autoconf,automake", + "Packages-Dnf": "git,make,gcc,zlib-devel,openssl-devel,icu,glibc-devel,kernel-headers,binutils,perl,unzip" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "DisableFirewall", + "Command": "sudo iptables -P INPUT ACCEPT" + } + }, + { + "Type": "ChocolateyInstallation", + "Parameters": { + "Scenario": "InstallChocolatey", + "PackageName": "chocolatey" + } + }, + { + "Type": "ChocolateyPackageInstallation", + "Parameters": { + "Scenario": "InstallGit", + "PackageName": "chocolatey", + "Packages": "git" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneAspNetBenchmarksRepo", + "RepoUri": "https://github.com/aspnet/Benchmarks.git", + "Commit": "5643540", + "PackageName": "aspnetbenchmarks" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "DotNetInstallation", + "Parameters": { + "Scenario": "InstallDotNetSdk", + "DotNetVersion": "$.Parameters.DotNetVersion", + "PackageName": "dotnetsdk" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-WRK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-WRK.json new file mode 100644 index 0000000000..8a65c54e1f --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-WRK.json @@ -0,0 +1,164 @@ +{ + "Description": ".NET benchmarking Workload (supports .NET 9 and .NET 10)", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" + }, + "Parameters": { + "DotNetVersion": "9", + "TargetFramework": "net9.0", + "Port": 9876, + "TestDuration": "00:00:15", + "ServerOnlineTimeout": "00:10:00", + "EmitLatencySpectrum": false, + "WaitForServer": false, + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s http://{serverip}:{Port}/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\"" + }, + "ParametersOn": [ + { + "Condition": "{calculate(\"{DotNetVersion}\" == \"9\")}", + "DotNetVersion": "9.0.111", + "TargetFramework": "net9.0" + }, + { + "Condition": "{calculate(\"{DotNetVersion}\" == \"10\")}", + "DotNetVersion": "10.0.101", + "TargetFramework": "net10.0" + } + ], + "Actions": [ + { + "Type": "AspNetServerExecutor", + "Parameters": { + "Role": "Server", + "Scenario": "ExecuteJsonSerializationBenchmark", + "PackageName": "aspnetbenchmarks", + "DotNetSdkPackageName": "dotnetsdk", + "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "$.Parameters.Port" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "warmup", + "MetricScenario": "Warmup_ASP.NET-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "Port": "$.Parameters.Port", + "ThreadCount": "64", + "Connection": 4096, + "TestDuration": "00:00:45", + "WaitForServer": "$.Parameters.WaitForServer", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Timeout": "$.Parameters.ServerOnlineTimeout", + "Tags": "ASP.NET,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "benchmark_measurement", + "MetricScenario": "ASP.NET-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "Port": "$.Parameters.Port", + "ThreadCount": "64", + "Connection": 4096, + "TestDuration": "$.Parameters.TestDuration", + "WaitForServer": "$.Parameters.WaitForServer", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Timeout": "$.Parameters.ServerOnlineTimeout", + "Tags": "ASP.NET,WRK" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip", + "Packages-Yum": "zlib-devel,pcre-devel,libevent-devel,openssl-devel,git,gcc-c++,make,autoconf,automake", + "Packages-Dnf": "git,make,gcc,zlib-devel,openssl-devel,icu,glibc-devel,kernel-headers,binutils,perl,unzip" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "DisableFirewall", + "Command": "sudo iptables -P INPUT ACCEPT" + } + }, + { + "Type": "ChocolateyInstallation", + "Parameters": { + "Scenario": "InstallChocolatey", + "PackageName": "chocolatey" + } + }, + { + "Type": "ChocolateyPackageInstallation", + "Parameters": { + "Scenario": "InstallGit", + "PackageName": "chocolatey", + "Packages": "git" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneAspNetBenchmarksRepo", + "RepoUri": "https://github.com/aspnet/Benchmarks.git", + "Commit": "5643540", + "PackageName": "aspnetbenchmarks" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "DotNetInstallation", + "Parameters": { + "Scenario": "InstallDotNetSdk", + "DotNetVersion": "$.Parameters.DotNetVersion", + "PackageName": "dotnetsdk" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK-RP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK-RP.json new file mode 100644 index 0000000000..2f930f5299 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK-RP.json @@ -0,0 +1,404 @@ +{ + "Description": "Networking benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "01:00:00", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu", + "Note": "Reverse Proxy config requires three nodes for the roles Client, Reverse Proxy, and Server." + }, + "Parameters": { + "TestDuration": "00:02:30", + "Timeout": "00:30:00", + "FileSizeInKB": 1, + "EmitLatencySpectrum": false, + "Workers": null, + "TargetService": "reverse-proxy", + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s https://{reverseproxyip}/{FileSizeInKB}kb" + }, + "Actions": [ + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "Server", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "ReverseProxy", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip nginx" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "StopNginxAutostart", + "Command": "sudo systemctl disable nginx" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "ReverseProxy" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK.json new file mode 100644 index 0000000000..182783d5a6 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK.json @@ -0,0 +1,382 @@ +{ + "Description": "Networking benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "01:00:00", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu" + }, + "Parameters": { + "TestDuration": "00:02:30", + "Timeout": "00:30:00", + "FileSizeInKB": 1, + "EmitLatencySpectrum": false, + "Workers": null, + "TargetService": "server", + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s https://{serverip}/api_new/{FileSizeInKB}kb" + }, + "Actions": [ + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "Server", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip nginx" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "StopNginxAutostart", + "Command": "sudo systemctl disable nginx" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2-RP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2-RP.json new file mode 100644 index 0000000000..f82c68a429 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2-RP.json @@ -0,0 +1,421 @@ +{ + "Description": "Networking benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "01:00:00", + "SupportedPlatforms": "linux-x64", + "SupportedOperatingSystems": "Ubuntu 22", + "Note": "Reverse Proxy config requires three nodes for the roles Client, Reverse Proxy, and Server." + }, + "Parameters": { + "TestDuration": "00:02:30", + "Timeout": "00:20:00", + "FileSizeInKB": 1, + "EmitLatencySpectrum": false, + "Workers": null, + "TargetService": "reverse-proxy", + "Rate": 1000, + "CommandArguments": "--rate {Rate} --latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s https://{reverseproxyip}/{FileSizeInKB}kb" + }, + "Actions": [ + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "Server", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "ReverseProxy", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip nginx" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "StopNginxAutostart", + "Command": "sudo systemctl disable nginx" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "ReverseProxy" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk2", + "RepoUri": "https://github.com/giltene/wrk2", + "PackageName": "wrk2", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk2", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk2}", + "Role": "Client" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2.json new file mode 100644 index 0000000000..e319436f1f --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2.json @@ -0,0 +1,399 @@ +{ + "Description": "Networking benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "01:00:00", + "SupportedPlatforms": "linux-x64", + "SupportedOperatingSystems": "Ubuntu 22" + }, + "Parameters": { + "TestDuration": "00:02:30", + "Timeout": "00:20:00", + "FileSizeInKB": 1, + "EmitLatencySpectrum": false, + "Workers": null, + "TargetService": "server", + "Rate": 1000, + "CommandArguments": "--rate {Rate} --latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s https://{serverip}/api_new/{FileSizeInKB}kb" + }, + "Actions": [ + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "Server", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip nginx" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "StopNginxAutostart", + "Command": "sudo systemctl disable nginx" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk2", + "RepoUri": "https://github.com/giltene/wrk2", + "PackageName": "wrk2", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk2", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk2}", + "Role": "Client" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} \ No newline at end of file diff --git a/src/VirtualClient/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md b/src/VirtualClient/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/website/docs/dependencies/0040-install-git-repo.md b/website/docs/dependencies/0040-install-git-repo.md index 3f2244a37c..eb841193da 100644 --- a/website/docs/dependencies/0040-install-git-repo.md +++ b/website/docs/dependencies/0040-install-git-repo.md @@ -47,7 +47,7 @@ In this example, VC clones https://github.com/eembc/coremark.git into the runtim ## Example In this example, VC installs Chocolatey, installs Git on the system and then clones https://github.com/aspnet/Benchmarks.git into the runtime packages directory -* [Profile Example with Chocolatey](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json) +* [Profile Example with Chocolatey](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json)
diff --git a/website/docs/dependencies/0051-install-dotnet-sdk.md b/website/docs/dependencies/0051-install-dotnet-sdk.md index b4d162a63e..115a386fa1 100644 --- a/website/docs/dependencies/0051-install-dotnet-sdk.md +++ b/website/docs/dependencies/0051-install-dotnet-sdk.md @@ -22,7 +22,7 @@ The following section describes the parameters used by the individual component ## Example The following section describes the parameters used by the individual component in the profile. -* [Profile Example](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json) +* [Profile Example](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json) ```json { diff --git a/website/docs/overview/overview.md b/website/docs/overview/overview.md index 56bc735d95..e4d9124079 100644 --- a/website/docs/overview/overview.md +++ b/website/docs/overview/overview.md @@ -50,7 +50,7 @@ for using them. | **Workload/Benchmark** | **Specialization** | **Supported Platforms/Architectures** | **License(s)** | |------------------------|--------------------|---------------------------------------|----------------| | [7zip](https://microsoft.github.io/VirtualClient/docs/workloads/compression/7zip) | Compression | linux-x64, linux-arm64 | [GNU LGPL](https://www.7-zip.org/faq.html) | -| [AspNetBench](https://microsoft.github.io/VirtualClient/docs/workloads/aspnetbench/aspnetbench) | ASP.NET Web Server. | linux-x64, linux-arm64, win-x64, win-arm64 | [MIT (ASP.NET)](https://github.com/dotnet/aspnetcore/blob/main/LICENSE.txt)
[MIT (Bombardier)](https://github.com/codesenberg/bombardier/blob/master/LICENSE) | +| [ASP.NET Benchmarks](https://microsoft.github.io/VirtualClient/docs/workloads/aspnet-benchmarks/aspnet-benchmarks) | ASP.NET Kestrel web server throughput and latency. | linux-x64, linux-arm64, win-x64, win-arm64 | [MIT (ASP.NET)](https://github.com/dotnet/aspnetcore/blob/main/LICENSE.txt)
[MIT (Bombardier)](https://github.com/codesenberg/bombardier/blob/master/LICENSE) | | [BlenderBenchmark](https://microsoft.github.io/VirtualClient/docs/workloads/blenderbenchmark) | GPU/Graphics Rendering Performance | win-x64 | [GNU LGPL](https://projects.blender.org/infrastructure/blender-open-data/src/branch/main/LICENSE) | | [CoreMark](https://microsoft.github.io/VirtualClient/docs/workloads/coremark/coremark) | CPU Performance | linux-x64, linux-arm64 | [Apache+Custom](https://github.com/eembc/coremark/blob/main/LICENSE.md) | | [CoreMark Pro](https://microsoft.github.io/VirtualClient/docs/workloads/coremark) | Precision CPU | linux-x64, linux-arm64, win-x64, win-arm64 | [Apache+Custom](https://github.com/eembc/coremark-pro/blob/main/LICENSE.md) | diff --git a/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md b/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md new file mode 100644 index 0000000000..669fbff13c --- /dev/null +++ b/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks-profiles.md @@ -0,0 +1,173 @@ +# ASP.NET Benchmarks Workload Profiles +The following profiles run customer-representative or benchmarking scenarios using ASP.NET server workloads. + +* [Workload Details](./aspnet-benchmarks.md) + +## PERF-WEB-ASPNET-WRK.json +Runs the ASP.NET TechEmpower JSON serialization benchmark using Wrk. Includes a warm-up pass before the benchmark measurement. +Supports .NET 9 and .NET 10 via the `ParametersOn` conditional parameter system. + +* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-WRK.json) + +* **Supported Platform/Architectures** + * linux-x64 + * linux-arm64 + * win-x64 + * win-arm64 + +* **Supports Disconnected Scenarios** + * No. Internet connection required. + +* **Dependencies** + The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. + * Internet connection. + + Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: + * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) + +* **Profile Parameters** + + | Parameter | Purpose | Default Value | + |---------------------------|-------------------------------------------------------------------|---------------| + | DotNetVersion | The major .NET version to use. Triggers `ParametersOn` conditional resolution. | 9 | + | TargetFramework | The .NET target framework to run. | net9.0 | + | TestDuration | Duration of each test run. | 00:00:15 | + | EmitLatencySpectrum | When `true`, emits fine-grained latency spectrum data. | false | + +* **Profile Runtimes** + See the 'Metadata' section of the profile for estimated runtimes. + +* **Usage Examples** + + ```bash + # Run with .NET 9 (default) + ./VirtualClient --profile=PERF-WEB-ASPNET-WRK.json --system=Demo --timeout=1440 + + # Run with .NET 10 + ./VirtualClient --profile=PERF-WEB-ASPNET-WRK.json --system=Demo --timeout=1440 --parameters="DotNetVersion=10" + ``` + +## PERF-WEB-ASPNET-WRK-AFFINITY.json +Runs the ASP.NET TechEmpower JSON serialization benchmark using Wrk with CPU core affinity for both server and client. +Includes a warm-up pass before the benchmark measurement. Supports .NET 9 and .NET 10. + +* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-WRK-AFFINITY.json) + +* **Supported Platform/Architectures** + * linux-x64 + * linux-arm64 + * win-x64 + * win-arm64 + +* **Supports Disconnected Scenarios** + * No. Internet connection required. + +* **Dependencies** + The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. + * Internet connection. + + Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: + * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) + +* **Profile Parameters** + + | Parameter | Purpose | Default Value | + |---------------------------|-------------------------------------------------------------------|---------------| + | DotNetVersion | The major .NET version to use. Triggers `ParametersOn` conditional resolution. | 9 | + | TargetFramework | The .NET target framework to run. | net9.0 | + | ServerCoreAffinity | CPU core affinity for the server process. | 0-7 | + | ClientCoreAffinity | CPU core affinity for the client process. | 8-15 | + | TestDuration | Duration of each test run. | 00:00:15 | + | EmitLatencySpectrum | When `true`, emits fine-grained latency spectrum data. | false | + +* **Profile Runtimes** + See the 'Metadata' section of the profile for estimated runtimes. + +* **Usage Examples** + + ```bash + # Run with default affinity and .NET 9 + ./VirtualClient --profile=PERF-WEB-ASPNET-WRK-AFFINITY.json --system=Demo --timeout=1440 + + # Run with .NET 10 and custom core affinity + ./VirtualClient --profile=PERF-WEB-ASPNET-WRK-AFFINITY.json --system=Demo --timeout=1440 --parameters="DotNetVersion=10,,,ServerCoreAffinity=0-3,,,ClientCoreAffinity=4-7" + ``` + +## PERF-WEB-ASPNET-BOMBARDIER.json +Runs the ASP.NET TechEmpower JSON serialization benchmark using Bombardier as the HTTP client +with 256 concurrent connections. Uses .NET 8 SDK. + +* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json) + +* **Supported Platform/Architectures** + * linux-x64 + * linux-arm64 + * win-x64 + * win-arm64 + +* **Supports Disconnected Scenarios** + * No. Internet connection required. + +* **Dependencies** + The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. + * Internet connection. + + Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: + * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) + +* **Profile Parameters** + + | Parameter | Purpose | Default Value | + |---------------------------|-------------------------------------------------------------------|---------------| + | DotNetVersion | The version of the .NET SDK to download and install. | 8.0.204 | + | TargetFramework | The .NET target framework to run. | net8.0 | + | ServerPort | The port the Kestrel server listens on. | 9876 | + +* **Profile Runtimes** + See the 'Metadata' section of the profile for estimated runtimes. + +* **Usage Examples** + + ```bash + # Execute the workload profile + ./VirtualClient --profile=PERF-WEB-ASPNET-BOMBARDIER.json --system=Demo --timeout=1440 + ``` +## PERF-WEB-ASPNET-ORCHARD-WRK.json +Runs the ASP.NET OrchardCore CMS benchmark using Wrk. The server runs the OrchardCore.Cms.Web application +and Wrk benchmarks the `/about` endpoint. Includes a warm-up pass before the benchmark measurement. + +* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-ORCHARD-WRK.json) + +* **Supported Platform/Architectures** + * linux-x64 + * linux-arm64 + +* **Supports Disconnected Scenarios** + * No. Internet connection required. + +* **Dependencies** + The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. + * Internet connection. + + Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: + * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) + +* **Profile Parameters** + + | Parameter | Purpose | Default Value | + |---------------------------|-------------------------------------------------------------------|---------------| + | DotNetVersion | The version of the .NET SDK to download and install. | 9.0.203 | + | TargetFramework | The .NET target framework to run. | net9.0 | + | ServerPort | The port the OrchardCore server listens on. | 5014 | + | TestDuration | Duration of each test run. | 00:00:20 | + | EmitLatencySpectrum | When `true`, emits fine-grained latency spectrum data. | false | + +* **Profile Runtimes** + See the 'Metadata' section of the profile for estimated runtimes. + +* **Usage Examples** + + ```bash + # Execute the workload profile + ./VirtualClient --profile=PERF-WEB-ASPNET-ORCHARD-WRK.json --system=Demo --timeout=1440 + ``` diff --git a/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks.md b/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks.md new file mode 100644 index 0000000000..089425f7c4 --- /dev/null +++ b/website/docs/workloads/aspnet-benchmarks/aspnet-benchmarks.md @@ -0,0 +1,174 @@ +# ASP.NET Benchmarks +The ASP.NET benchmarks measure the throughput and latency of ASP.NET Kestrel web applications under sustained HTTP load. +The workloads use a client-server architecture where the server runs an ASP.NET application and the client generates +HTTP requests using [Bombardier](../bombardier/bombardier.md) or [Wrk](../wrk/wrk.md). + +Two server workloads are supported: + +- **TechEmpower JSON Serialization** — Based on the [ASP.NET Benchmarks](https://github.com/aspnet/benchmarks) project + (derived from the [TechEmpower framework](https://www.techempower.com/benchmarks/)), the server exposes a `/json` + endpoint that serializes a simple JSON object. Supports Bombardier and Wrk as client load generators. +- **OrchardCore CMS** — Runs the [OrchardCore](https://github.com/orchardcms/orchardcore) content management system + with a Blog recipe, benchmarked with Wrk against the `/about` endpoint. + +Both server configurations support **CPU core affinity** via the `BindToCores` and `CoreAffinity` parameters, allowing the +server process to be pinned to specific CPU cores for controlled performance measurement. + +* [ASP.NET Benchmarks GitHub](https://github.com/aspnet/benchmarks) +* [OrchardCore GitHub](https://github.com/orchardcms/orchardcore) +* [TechEmpower Framework Benchmarks](https://www.techempower.com/benchmarks/) +* [Bombardier Documentation](../bombardier/bombardier.md) +* [Wrk/Wrk2 Documentation](../wrk/wrk.md) + +## Deployment Modes +The ASP.NET benchmark workloads support two deployment modes: + +- **Multi-VM (Client-Server)** — The server and client run on separate machines connected via an + [environment layout](../../guides/0020-client-server.md) file. This is the recommended mode for production benchmarking + as it isolates server and client resource consumption. +- **Single-VM** — When no layout file is provided, both server and client actions run sequentially on the same machine. + The client connects to the server via the loopback address (`127.0.0.1`). This mode is useful for development, + validation, and quick smoke testing. + +CPU core affinity profiles (e.g., `PERF-WEB-ASPNET-WRK-AFFINITY.json`) work in +both modes. In single-VM mode, core affinity is especially useful to prevent the server and client from contending +for the same CPU cores. + +## What is Being Measured? +The client tools (Bombardier or Wrk) generate concurrent HTTP requests against the ASP.NET server and capture latency +percentile distributions and throughput statistics. + +### TechEmpower JSON Serialization +The server exposes a `/json` endpoint that serializes a simple JSON object. This scenario measures raw HTTP request processing +performance of the Kestrel server with minimal application logic overhead. + +### OrchardCore CMS +The server runs a full OrchardCore CMS application with a Blog recipe. The `/about` endpoint exercises the full CMS rendering +pipeline including routing, middleware, and content rendering. + +## Workload Metrics +The following metrics are examples of those captured by the Virtual Client when running the ASP.NET benchmark workloads. + +### Bombardier Metrics +When Bombardier is used as the client (e.g., `PERF-WEB-ASPNET-BOMBARDIER.json`): + +| Name | Example Value | Unit | Description | +|------------------------|--------------------|-------------|-----------------------------------------------------| +| Latency Average | 133.3698688 | milliseconds| Average HTTP response latency | +| Latency Max | 7123.06304 | milliseconds| Maximum HTTP response latency | +| Latency P50 | 83.39392 | milliseconds| HTTP response latency (50th percentile) | +| Latency P75 | 160.14336 | milliseconds| HTTP response latency (75th percentile) | +| Latency P90 | 286.4128 | milliseconds| HTTP response latency (90th percentile) | +| Latency P95 | 367.4112 | milliseconds| HTTP response latency (95th percentile) | +| Latency P99 | 637.534208 | milliseconds| HTTP response latency (99th percentile) | +| RequestPerSecond Avg | 32768.449018 | Reqs/sec | ASP.NET Web Requests per second (average) | +| RequestPerSecond Stddev| 6446.822354105378 | Reqs/sec | ASP.NET Web Requests per second (standard deviation)| +| RequestPerSecond P50 | 31049.462844 | Reqs/sec | ASP.NET Web Requests per second (P50) | +| RequestPerSecond P75 | 35597.436614 | Reqs/sec | ASP.NET Web Requests per second (P75) | +| RequestPerSecond P90 | 39826.205746 | Reqs/sec | ASP.NET Web Requests per second (P90) | +| RequestPerSecond P95 | 41662.542962 | Reqs/sec | ASP.NET Web Requests per second (P95) | +| RequestPerSecond P99 | 48600.556224 | Reqs/sec | ASP.NET Web Requests per second (P99) | + +### Wrk Metrics +When Wrk is used as the client (e.g., `PERF-WEB-ASPNET-WRK.json`, `PERF-WEB-ASPNET-ORCHARD-WRK.json`): + +| Name | Example Value | Unit | Description | +|--------------------|---------------|---------------|-----------------------------------------------| +| latency_p50 | 1.427 | milliseconds | HTTP response latency (50th percentile) | +| latency_p75 | 1.982 | milliseconds | HTTP response latency (75th percentile) | +| latency_p90 | 2.683 | milliseconds | HTTP response latency (90th percentile) | +| latency_p99 | 3.960 | milliseconds | HTTP response latency (99th percentile) | +| latency_p99_9 | 6.930 | milliseconds | HTTP response latency (99.9th percentile) | +| latency_p100 | 9.770 | milliseconds | HTTP response latency (100th percentile) | +| requests/sec | 16305.17 | requests/sec | Aggregate throughput | +| transfers/sec | 20.01 | megabytes/sec | Data transfer rate | + +See the [Wrk/Wrk2 documentation](../wrk/wrk.md) and [Bombardier documentation](../bombardier/bombardier.md) for the complete list of client metrics. + +## Profiles +The following profiles are available for the ASP.NET benchmark workloads. See the [profile details](./aspnet-benchmarks-profiles.md) +page for per-profile parameters, dependencies, and usage examples. + +| Profile Name | Description | Client Tool | Server | Platforms | +|------------------------------------------|------------------------------------------------------------------------------------|--------------------|-------------------------------|--------------------------------------------| +| PERF-WEB-ASPNET-WRK.json | TechEmpower JSON serialization benchmark using Wrk with warm-up pass. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-WRK-AFFINITY.json | TechEmpower JSON serialization benchmark using Wrk with CPU core affinity. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-BOMBARDIER.json | TechEmpower JSON serialization benchmark using Bombardier with 256 connections. | BombardierExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-ORCHARD-WRK.json | OrchardCore CMS benchmark using Wrk with warm-up pass. | WrkExecutor | AspNetOrchardServerExecutor | linux-x64, linux-arm64 | + +## Server Parameters +The following tables describe the key parameters supported by the ASP.NET server executors. + +### AspNetServerExecutor + +| Parameter | Description | Default | +|----------------------------------|-----------------------------------------------------------------------------|------------| +| PackageName | The name of the ASP.NET Benchmarks dependency package. | *required* | +| DotNetSdkPackageName | The name of the .NET SDK dependency package. | `dotnetsdk`| +| TargetFramework | The .NET target framework (e.g., `net8.0`, `net9.0`). | *required* | +| ServerPort | The port the Kestrel server listens on. | `9876` | +| AspNetCoreThreadCount | The ASPNETCORE thread count environment variable. | `1` | +| DotNetSystemNetSocketsThreadCount| The DOTNET_SYSTEM_NET_SOCKETS_THREAD_COUNT environment variable. | `1` | +| BindToCores | When `true`, the server process is pinned to specific CPU cores. | `false` | +| CoreAffinity | CPU core affinity specification (e.g., `0-7`). Required when `BindToCores` is `true`. | *none* | + +### AspNetOrchardServerExecutor + +| Parameter | Description | Default | +|--------------------|-----------------------------------------------------------------------------|------------| +| PackageName | The name of the OrchardCore dependency package. | *required* | +| DotNetSdkPackageName | The name of the .NET SDK dependency package. | `dotnetsdk`| +| TargetFramework | The .NET target framework (e.g., `net9.0`). | *required* | +| ServerPort | The port the OrchardCore server listens on. | `5014` | +| BindToCores | When `true`, the server process is pinned to specific CPU cores. | `false` | +| CoreAffinity | CPU core affinity specification (e.g., `0-7`). Required when `BindToCores` is `true`. | *none* | + +## Packaging and Setup +The following section covers how to create the custom Virtual Client dependency packages required to execute the workload +and toolset(s). This section is meant to provide guidance for users who would like to create their own packages with the +software for use with the Virtual Client. For example, users may want to bring in new versions of the software. +See the documentation on [Dependency Packages](https://microsoft.github.io/VirtualClient/docs/developing/0040-vc-packages/) +for more information on the concepts. + +### TechEmpower JSON Serialization Setup +1. Virtual Client installs the .NET SDK via the `DotNetInstallation` dependency. +2. Virtual Client clones the [ASP.NET Benchmarks](https://github.com/aspnet/Benchmarks) GitHub repository. +3. The `src/Benchmarks` project is built using `dotnet build`. +4. The server is started using `dotnet run`: + +```bash +dotnet /Benchmarks.dll \ + --nonInteractive true \ + --scenarios json \ + --urls http://*:9876 \ + --server Kestrel \ + --kestrelTransport Sockets \ + --protocol http \ + --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" \ + --header "Connection: keep-alive" +``` +5. The client tool (Bombardier or Wrk) generates HTTP load against the server: + +```bash +# Bombardier example +bombardier -d 15s -c 256 -t 2s --fasthttp --insecure -l http://:9876/json --print r --format json + +# Wrk example +wrk --latency --threads 64 --connections 4096 --duration 15s --timeout 10s http://:9876/json +``` + +### OrchardCore CMS Setup +1. Virtual Client installs the .NET SDK via the `DotNetInstallation` dependency. +2. Virtual Client clones the [OrchardCore](https://github.com/OrchardCMS/OrchardCore) GitHub repository. +3. The `OrchardCore.Cms.Web` project is published using `dotnet publish`. +4. The server is started: + +```bash +nohup /OrchardCore.Cms.Web --urls http://*:5014 +``` + +5. Wrk generates HTTP load against the `/about` endpoint: + +```bash +wrk -t 64 -c 128 -d 20s http://:5014/about +``` diff --git a/website/docs/workloads/aspnetbench/aspnetbench-profiles.md b/website/docs/workloads/aspnetbench/aspnetbench-profiles.md deleted file mode 100644 index 353c53c889..0000000000 --- a/website/docs/workloads/aspnetbench/aspnetbench-profiles.md +++ /dev/null @@ -1,49 +0,0 @@ -# AspNetBench Workload Profiles -The following profiles run customer-representative or benchmarking scenarios using the AspNetBench workload. - -* [Workload Details](./aspnetbench.md) - -## PERF-ASPNETBENCH.json -Runs the AspNetBench benchmark workload to assess the performance of an ASP.NET Server. - -* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json) - -* **Supported Platform/Architectures** - * linux-x64 - * linux-arm64 - * win-x64 - * win-arm64 - -* **Supports Disconnected Scenarios** - * No. Internet connection required. - -* **Dependencies** - The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. - * Internet connection. - - Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: - * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) - -* **Profile Parameters** - The following parameters can be optionally supplied on the command line to modify the behaviors of the workload. - - | Parameter | Purpose | Default Value | - |---------------------------|-------------------------------------------------------------------|---------------| - | DotNetVersion | Optional. The version of the [.NET SDK to download and install](https://dotnet.microsoft.com/en-us/download/visual-studio-sdks). | 7.0.100 | - | TargetFramework | Optional. The [.NET target framework](https://learn.microsoft.com/en-us/dotnet/standard/frameworks) to run (e.g. net6.0, net7.0). | net7.0 | - -* **Profile Runtimes** - See the 'Metadata' section of the profile for estimated runtimes. These timings represent the length of time required to run a single round of profile - actions. These timings can be used to determine minimum required runtimes for the Virtual Client in order to get results. These are often estimates based on the - number of system cores. - -* **Usage Examples** - The following section provides a few basic examples of how to use the workload profile. - - ``` bash - # Execute the workload profile - VirtualClient.exe --profile=PERF-ASPNETBENCH.json --system=Demo --timeout=1440 - - # Override the profile default parameters to use a different .NET SDK version - VirtualClient.exe --profile=PERF-ASPNETBENCH.json --system=Demo --timeout=1440 --parameters="DotNetVersion=7.0.307" - ``` \ No newline at end of file diff --git a/website/docs/workloads/aspnetbench/aspnetbench.md b/website/docs/workloads/aspnetbench/aspnetbench.md deleted file mode 100644 index 9f89941e05..0000000000 --- a/website/docs/workloads/aspnetbench/aspnetbench.md +++ /dev/null @@ -1,56 +0,0 @@ -# AspNetBenchmark -AspNetBenchmark is a benchmark developed by MSFT ASPNET team, based on open source benchmark TechEmpower. -This workload has server and client part, on the same test machine. The server part is started as a ASPNET service. The client calls server using open source bombardier binaries. -Bombardier binaries could be downloaded from Github release, or directly compile from source using "go build ." - -* [AspNetBenchmarks Github](https://github.com/aspnet/benchmarks) -* [Bombardier Github](https://github.com/codesenberg/bombardier) -* [Bombardier Release](https://github.com/codesenberg/bombardier/releases/tag/v1.2.5) - -## Workload Metrics -The following metrics are examples of those captured by the Virtual Client when running the AspNetBenchmark workload. - -[Bombardier output example](https://github.com/codesenberg/bombardier#examples) - -The following metrics are examples of those captured during the operations of the AspNetBench workload. - -| Name | Example | Unit | Description | -|--------------------------|--------------------|-------------|----------------------------------------| -| Latency Max | 178703 | microsecond | ASP.NET Web Request latency (max) | -| Latency Average | 8270.807963429836 | microsecond | ASP.NET Web Request latency (avg) | -| Latency Stddev | 6124.356473307014 | microsecond | ASP.NET Web Request latency (standard deviation) | -| Latency P50 | 6058 | microsecond | ASP.NET Web Request latency (P50) | -| Latency P75 | 10913 | microsecond | ASP.NET Web Request latency (P75) | -| Latency P90 | 17949 | microsecond | ASP.NET Web Request latency (P90) | -| Latency P95 | 23318 | microsecond | ASP.NET Web Request latency (P95) | -| Latency P99 | 35856 | microsecond | ASP.NET Web Request latency (P99) | -| RequestPerSecond Max | 61221.282458945345 | Reqs/sec | ASP.NET Web Request per second (max) | -| RequestPerSecond Average | 31211.609987720527 | Reqs/sec | ASP.NET Web Request per second (avg) | -| RequestPerSecond Stddev | 6446.822354105378 | Reqs/sec | ASP.NET Web Request per second (standard deviation) | -| RequestPerSecond P50 | 31049.462844 | Reqs/sec | ASP.NET Web Request per second (P50) | -| RequestPerSecond P75 | 35597.436614 | Reqs/sec | ASP.NET Web Request per second (P75) | -| RequestPerSecond P90 | 39826.205746 | Reqs/sec | ASP.NET Web Request per second (P90) | -| RequestPerSecond P95 | 41662.542962 | Reqs/sec | ASP.NET Web Request per second (P95) | -| RequestPerSecond P99 | 48600.556224 | Reqs/sec | ASP.NET Web Request per second (P99) | - -## Packaging and Setup -The following section covers how to create the custom Virtual Client dependency packages required to execute the workload and toolset(s). This section -is meant to provide guidance for users that would like to create their own packages with the software for use with the Virtual Client. For example, users -may want to bring in new versions of the software. See the documentation on '[Dependency Packages](https://microsoft.github.io/VirtualClient/docs/developing/0040-vc-packages/)' -for more information on the concepts. - -1. VC installs dotnet SDK -2. VC clones AspNetBenchmarks github repo -3. dotnet build src/benchmarks project in AspNetBenchmarks repo -4. Use dotnet to start server - -``` -dotnet \Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:5000 --server Kestrel --kestrelTransport Sockets --protocol http --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" --header "Connection: keep-alive" -``` - -5. Use bombardier to start client -``` -bombardier-windows-amd64.exe -d 15s -c 256 -t 2s --fasthttp --insecure -l http://localhost:5000/json --print r --format json -``` - - diff --git a/website/docs/workloads/bombardier/bombardier.md b/website/docs/workloads/bombardier/bombardier.md new file mode 100644 index 0000000000..9c219cb721 --- /dev/null +++ b/website/docs/workloads/bombardier/bombardier.md @@ -0,0 +1,90 @@ +# Bombardier HTTP Benchmarking +Bombardier is a fast, cross-platform HTTP(S) benchmarking tool written in Go. It uses the fasthttp library for high-performance +HTTP client operations and supports configurable concurrency, request duration, timeouts, and output in JSON format for automated +metric collection. + +In Virtual Client, Bombardier serves as the HTTP client load generator for ASP.NET server workloads. It runs on a dedicated +client machine (or the same machine in single-VM mode) and sends requests to the ASP.NET Kestrel server, producing latency +and throughput statistics in JSON format that are parsed into standardized metrics. + +* [Bombardier GitHub](https://github.com/codesenberg/bombardier) +* [Bombardier Releases](https://github.com/codesenberg/bombardier/releases) + +## Deployment Modes +The Bombardier executor supports two deployment modes: + +- **Multi-VM (Client-Server)** — The client and server run on separate machines connected via a layout file. + This isolates load generation from server processing for accurate benchmarking. +- **Single-VM** — When no layout file is provided, Bombardier connects to the server via the loopback address + (`127.0.0.1`). Both server and client actions run sequentially on the same machine. + +## What is Being Measured? +Bombardier generates sustained HTTP request traffic against a target server endpoint and captures the following: + +- **Latency** — average, standard deviation, maximum, and percentile distributions (P50, P75, P90, P95, P99) of response latency in microseconds. +- **Requests per second** — average, standard deviation, maximum, and percentile distributions (P50, P75, P90, P95, P99) of throughput. + +Bombardier outputs results in JSON format, which is parsed by the `BombardierMetricsParser` to extract structured metrics. + +## Workload Metrics +The following metrics are examples of those captured by the Virtual Client when running the Bombardier workload against an +ASP.NET server. + +### Latency Metrics + +| Metric Name | Example Value | Unit | Description | +|--------------------|---------------------|-------------|--------------------------------------------| +| Latency Max | 178703 | microsecond | HTTP response latency (maximum) | +| Latency Average | 8270.807963429836 | microsecond | HTTP response latency (average) | +| Latency Stddev | 6124.356473307014 | microsecond | HTTP response latency (standard deviation) | +| Latency P50 | 6058 | microsecond | HTTP response latency (50th percentile) | +| Latency P75 | 10913 | microsecond | HTTP response latency (75th percentile) | +| Latency P90 | 17949 | microsecond | HTTP response latency (90th percentile) | +| Latency P95 | 23318 | microsecond | HTTP response latency (95th percentile) | +| Latency P99 | 35856 | microsecond | HTTP response latency (99th percentile) | + +### Throughput Metrics + +| Metric Name | Example Value | Unit | Description | +|------------------------|----------------------|----------|----------------------------------------------------| +| RequestPerSecond Max | 67321.282458945348 | Reqs/sec | HTTP requests per second (maximum) | +| RequestPerSecond Average| 31211.609987720527 | Reqs/sec | HTTP requests per second (average) | +| RequestPerSecond Stddev | 6446.822354105378 | Reqs/sec | HTTP requests per second (standard deviation) | +| RequestPerSecond P50 | 31049.462844 | Reqs/sec | HTTP requests per second (50th percentile) | +| RequestPerSecond P75 | 35597.436614 | Reqs/sec | HTTP requests per second (75th percentile) | +| RequestPerSecond P90 | 39826.205746 | Reqs/sec | HTTP requests per second (90th percentile) | +| RequestPerSecond P95 | 41662.542962 | Reqs/sec | HTTP requests per second (95th percentile) | +| RequestPerSecond P99 | 49625.656227 | Reqs/sec | HTTP requests per second (99th percentile) | + +## Profiles +The following profiles use Bombardier as the client load generator. + +| Profile Name | Description | Server | Platforms | +|------------------------------|-------------------------------------------------------------------------------------------------|------------------------|----------------------------------------------| +| PERF-WEB-ASPNET-BOMBARDIER.json | ASP.NET JSON serialization benchmark using Bombardier with 256 connections. | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | + +## Parameters +The following table describes the key parameters supported by the Bombardier executor. + +| Parameter | Description | Default | +|------------------|--------------------------------------------------------------------------------------------------|---------------| +| PackageName | The name of the Bombardier dependency package. | *required* | +| CommandArguments | The Bombardier command-line arguments (duration, connections, timeout, URL, output format, etc.). | *required* | +| TargetService | The target service type: `server`, `rp` (reverse-proxy), or `apigw` (API gateway). | auto-detected | +| Timeout | Maximum time to wait for server availability. | 5 minutes | +| WarmUp | When `true`, the run is a warm-up pass and metrics are not captured. | `false` | +| BindToCores | When `true`, the Bombardier process is pinned to specific CPU cores. | `false` | +| CoreAffinity | CPU core affinity specification (e.g., `8-15`). Required when `BindToCores` is `true`. | *none* | + +## Packaging and Setup +Virtual Client handles installation of Bombardier automatically through profile dependencies. The typical setup flow is: + +1. VC installs Bombardier from the registered dependency package. +2. On Unix systems, VC makes the `bombardier` binary executable. +3. VC waits for the server (e.g., ASP.NET Kestrel) to come online. +4. Bombardier is invoked with the configured parameters against the target URL. + +Example Bombardier command: +``` +bombardier -d 15s -c 256 -t 10s --fasthttp --insecure -l http://localhost:9876/json --print r --format json +``` diff --git a/website/docs/workloads/nginx/nginx.md b/website/docs/workloads/nginx/nginx.md new file mode 100644 index 0000000000..b7bd08d147 --- /dev/null +++ b/website/docs/workloads/nginx/nginx.md @@ -0,0 +1,86 @@ +# Nginx +Nginx is a high-performance open-source web server, reverse proxy, and load balancer. In Virtual Client, the Nginx workload measures web server performance by serving static content of configurable sizes over HTTPS and measuring throughput and latency under sustained load. The HTTP load is generated by the Wrk or Wrk2 client tool. + +The workload supports two deployment topologies: + +- **Client-Server (two-node)** — A dedicated client machine runs wrk/wrk2 against a dedicated server machine running Nginx. This separates resource consumption between the web server and the load generator for accurate measurements. +- **Reverse Proxy (three-node)** — A client sends requests to an Nginx reverse-proxy instance, which forwards them to a backend Nginx server. This topology is used to benchmark Nginx reverse-proxy performance. + +The `NginxServerExecutor` manages the Nginx server lifecycle on the server (and reverse-proxy) instances. It handles configuration generation, SSL setup, static content creation of configurable file sizes, and server start/stop/reset operations via the `NginxCommand` enum (`Start`, `Stop`, `GetVersion`, `GetConfig`). + +* [Nginx Official Site](https://nginx.org/en/) +* [Nginx GitHub](https://github.com/nginx/nginx) +* [Wrk GitHub](https://github.com/wg/wrk) +* [Wrk2 GitHub](https://github.com/giltene/wrk2) + +## Deployment Modes +The Nginx workload requires a **multi-VM layout** and does not support single-VM mode. This is because the +`NginxServerExecutor` depends on separate client and server instances connected via a layout file. + +- **Client-Server (two-node)** — A dedicated client machine runs wrk/wrk2, and a dedicated server machine runs Nginx. +- **Reverse Proxy (three-node)** — A client machine, a reverse-proxy machine, and a backend server machine. + +## What is Being Measured? +The Nginx workload measures the throughput and latency of an Nginx web server serving static files of a configurable size. The Wrk (or Wrk2) client tool generates concurrent HTTP/HTTPS requests across different connection counts and thread configurations to characterize server performance at varying load levels. + +The following scenarios are included in the standard profile: + +| Scenario | Connections | Description | +|---------------------------------------|-------------|-----------------------------------------------------------------------| +| Latency_100_Connections | 100 | Low-concurrency baseline measurement at full thread count. | +| Latency_1K_Connections | 1,000 | Medium-concurrency measurement at full thread count. | +| Latency_5K_Connections | 5,000 | High-concurrency measurement at full thread count. | +| Latency_10K_Connections | 10,000 | Very high-concurrency measurement at full thread count. | +| Latency_100_Connections_Thread/2 | 100 | Low-concurrency measurement at half the logical core count. | +| Latency_1K_Connections_Thread/2 | 1,000 | Medium-concurrency measurement at half the logical core count. | +| Latency_5K_Connections_Thread/2 | 5,000 | High-concurrency measurement at half the logical core count. | +| Latency_10K_Connections_Thread/2 | 10,000 | Very high-concurrency at half the logical core count. | +| Latency_100_Connections_Thread/4 | 100 | Low-concurrency measurement at one quarter of the logical core count. | +| Latency_1K_Connections_Thread/4 | 1,000 | Medium-concurrency measurement at one quarter of the logical core count. | +| Latency_5K_Connections_Thread/4 | 5,000 | High-concurrency measurement at one quarter of the logical core count. | +| Latency_10K_Connections_Thread/4 | 10,000 | Very high-concurrency at one quarter of the logical core count. | +| Latency_100_Connections_Thread/8 | 100 | Low-concurrency measurement at one eighth of the logical core count. | +| Latency_1K_Connections_Thread/8 | 1,000 | Medium-concurrency measurement at one eighth of the logical core count. | +| Latency_5K_Connections_Thread/8 | 5,000 | High-concurrency measurement at one eighth of the logical core count. | +| Latency_10K_Connections_Thread/8 | 10,000 | Very high-concurrency at one eighth of the logical core count. | + +## Workload Metrics +The following metrics are examples of those captured by the Virtual Client when running the Nginx workload. Latency values are normalized to milliseconds by the parser regardless of the unit reported by Wrk (nanoseconds, microseconds, milliseconds, or seconds). + +| Tool Name | Metric Name | Example Value | Unit | +|-----------|-------------------|---------------|---------------| +| Wrk | latency_p50 | 1.234 | milliseconds | +| Wrk | latency_p75 | 2.456 | milliseconds | +| Wrk | latency_p90 | 3.678 | milliseconds | +| Wrk | latency_p99 | 8.901 | milliseconds | +| Wrk | latency_p99_9 | 15.432 | milliseconds | +| Wrk | latency_p99_99 | 20.987 | milliseconds | +| Wrk | latency_p99_999 | 30.123 | milliseconds | +| Wrk | latency_p100 | 45.678 | milliseconds | +| Wrk | requests/sec | 25432.56 | requests/sec | +| Wrk | transfers/sec | 312.45 | megabytes/sec | + +When wrk2 is used, an additional set of uncorrected latency metrics is emitted (e.g., `uncorrected_latency_p50`, `uncorrected_latency_p99`). +See the [Wrk/Wrk2 documentation](../wrk/wrk.md) for the complete list of wrk metrics. + +## Profiles +The following profiles are available for the Nginx workload. + +| Profile Name | Description | Client Tool | Topology | Platforms | +|----------------------------|------------------------------------------------------------------------------------------|--------------|-------------------|------------------------| +| PERF-WEB-NGINX-WRK.json | Nginx web server benchmark using wrk across multiple connection and thread counts. | WrkExecutor | Client → Server | linux-x64, linux-arm64 | +| PERF-WEB-NGINX-WRK2.json | Nginx web server benchmark using wrk2 with constant request rate and corrected latency. | Wrk2Executor | Client → Server | linux-x64 | +| PERF-WEB-NGINX-WRK-RP.json | Nginx reverse-proxy benchmark using wrk across a three-node layout. | WrkExecutor | Client → RP → Server | linux-x64, linux-arm64 | +| PERF-WEB-NGINX-WRK2-RP.json | Nginx reverse-proxy benchmark using wrk2 across a three-node layout. | Wrk2Executor | Client → RP → Server | linux-x64 | + +## Server Parameters +The following table describes the key parameters supported by the `NginxServerExecutor`. + +| Parameter | Description | Default | +|-----------------|-----------------------------------------------------------------------------|-------------| +| PackageName | The name of the Nginx dependency package. | *required* | +| Role | The role of the current instance: `Server` or `ReverseProxy`. | *required* | +| Workers | Number of Nginx worker processes. Set to `0` or omit to use all cores. | `0` (auto) | +| FileSizeInKB | Size of the static file to serve (in kilobytes). | `1` | +| Timeout | Maximum time to keep the server online before resetting. | 30 minutes | +| pollingInterval | Interval between server state polling cycles. | 60 seconds | diff --git a/website/docs/workloads/wrk/wrk.md b/website/docs/workloads/wrk/wrk.md new file mode 100644 index 0000000000..e065bad146 --- /dev/null +++ b/website/docs/workloads/wrk/wrk.md @@ -0,0 +1,166 @@ +# Wrk/Wrk2 HTTP Benchmarking +Wrk is a modern HTTP benchmarking tool capable of generating significant load when run on a single multi-core CPU. It combines a multithreaded +design with scalable event notification systems such as epoll and kqueue to produce high request throughput with low resource consumption. + +Wrk2 is a variant of wrk that adds constant-throughput, correct-latency recording using a modified version of Gil Tene's wrk2 rate-limiting +approach. Unlike wrk (which measures only coordinated-omission-free throughput), wrk2 takes a target request rate and produces an +HdrHistogram-corrected latency distribution, making it suitable for latency-sensitive benchmarks. + +In Virtual Client, wrk and wrk2 serve as the HTTP client load generators for Nginx and ASP.NET web server workloads. +They run on a dedicated client machine and send requests to a server machine running the target web server. + +* [Wrk GitHub](https://github.com/wg/wrk) +* [Wrk2 GitHub](https://github.com/giltene/wrk2) +* [How NOT to Measure Latency — Gil Tene (video)](https://www.youtube.com/watch?v=lJ8ydIuPFeU) +* [On Coordinated Omission — ScyllaDB](https://www.scylladb.com/2021/04/22/on-coordinated-omission/) + +## Deployment Modes +The Wrk/Wrk2 executors support two deployment modes when used with ASP.NET server workloads: + +- **Multi-VM (Client-Server)** — The client and server run on separate machines connected via a layout file. + This isolates load generation from server processing for accurate benchmarking. +- **Single-VM** — When no layout file is provided, wrk connects to the server via the loopback address + (`127.0.0.1`). Both server and client actions run sequentially on the same machine. + +Note: Nginx workloads (`PERF-WEB-NGINX-*.json`) require a multi-VM layout and do not support single-VM mode. + +## What is Being Measured? +The Wrk/Wrk2 toolset generates sustained HTTP/HTTPS request traffic against a target web server and captures the following: + +- **Latency percentile distribution** — full HdrHistogram percentiles (P50 through P100) of response latency. Values are normalized to milliseconds. +- **Requests per second** — the aggregate throughput achieved during the test run. +- **Transfer rate** — the data transfer rate achieved during the test run (megabytes/sec). + +Wrk2 additionally captures an *uncorrected latency distribution* that records raw measured latency without accounting for coordinated omission. +Both tools optionally emit a detailed percentile spectrum for fine-grained latency analysis (controlled by the `EmitLatencySpectrum` parameter). + +## Workload Metrics +The following metrics are examples of those captured by the Virtual Client when running the Wrk workload. +Latency values are normalized to milliseconds by the parser regardless of the unit reported by wrk (nanoseconds, microseconds, milliseconds, or seconds). + +### Latency Distribution Metrics + +| Tool Name | Metric Name | Example Value | Unit | +|-----------|------------------------|---------------|--------------| +| Wrk | latency_p50 | 1.427 | milliseconds | +| Wrk | latency_p75 | 1.982 | milliseconds | +| Wrk | latency_p90 | 2.683 | milliseconds | +| Wrk | latency_p99 | 3.960 | milliseconds | +| Wrk | latency_p99_9 | 6.930 | milliseconds | +| Wrk | latency_p99_99 | 8.990 | milliseconds | +| Wrk | latency_p99_999 | 9.770 | milliseconds | +| Wrk | latency_p100 | 9.770 | milliseconds | + +When wrk2 is used, an additional set of uncorrected latency metrics is emitted: + +| Tool Name | Metric Name | Example Value | Unit | +|-----------|---------------------------------|---------------|--------------| +| Wrk2 | uncorrected_latency_p50 | 0.483 | milliseconds | +| Wrk2 | uncorrected_latency_p75 | 1.120 | milliseconds | +| Wrk2 | uncorrected_latency_p90 | 1.710 | milliseconds | +| Wrk2 | uncorrected_latency_p99 | 2.870 | milliseconds | +| Wrk2 | uncorrected_latency_p99_9 | 5.760 | milliseconds | +| Wrk2 | uncorrected_latency_p99_99 | 8.020 | milliseconds | +| Wrk2 | uncorrected_latency_p99_999 | 8.410 | milliseconds | +| Wrk2 | uncorrected_latency_p100 | 8.410 | milliseconds | + +### Throughput Metrics + +| Tool Name | Metric Name | Example Value | Unit | +|-----------|----------------|---------------|---------------| +| Wrk | requests/sec | 16305.17 | requests/sec | +| Wrk | transfers/sec | 20.01 | megabytes/sec | + +### Latency Spectrum Metrics (Optional) +When the `EmitLatencySpectrum` parameter is set to `true`, the parser emits fine-grained percentile spectrum data points. +These are useful for visualizing full HdrHistogram latency distributions. + +| Tool Name | Metric Name | Example Value | Unit | Description | +|-----------|--------------------------------|---------------|------|---------------------------------| +| Wrk | latency_spectrum_p0_000000 | 0.175 | | TotalCount:1 | +| Wrk | latency_spectrum_p0_100000 | 0.566 | | TotalCount:3954 | +| Wrk | latency_spectrum_p0_500000 | 1.427 | | TotalCount:19773 | +| Wrk | latency_spectrum_p0_900000 | 2.683 | | TotalCount:35553 | +| Wrk | latency_spectrum_p0_990000 | 3.960 | | TotalCount:39130 | +| Wrk | latency_spectrum_p0_999000 | 7.011 | | TotalCount:39462 | +| Wrk | latency_spectrum_p1_000000 | 9.767 | | TotalCount:39500 | + +### Error Metrics + +| Tool Name | Metric Name | Example Value | Unit | +|-----------|--------------------------|---------------|------| +| Wrk | Non-2xx or 3xx responses | 58902 | | + +If socket errors are detected, the parser raises a `WorkloadException` with a `Socket Error` metric. + +## Profiles +The following profiles are available for the Wrk/Wrk2 workloads. + +| Profile Name | Description | Client Tool | Server | Platforms | +|---------------------------|---------------------------------------------------------------------------------------------|-------------|----------------------------|----------------------| +| PERF-WEB-NGINX-WRK.json | Nginx web server benchmark using wrk. Tests 100 to 10K connections at multiple thread counts. | WrkExecutor | NginxServerExecutor | linux-x64, linux-arm64 | +| PERF-WEB-NGINX-WRK2.json | Nginx web server benchmark using wrk2 with constant request rate and corrected latency. | Wrk2Executor | NginxServerExecutor | linux-x64 | +| PERF-WEB-NGINX-WRK-RP.json | Nginx reverse-proxy benchmark using wrk. Uses a three-node layout (Client → Reverse Proxy → Server). | WrkExecutor | NginxServerExecutor (×2) | linux-x64, linux-arm64 | +| PERF-WEB-NGINX-WRK2-RP.json | Nginx reverse-proxy benchmark using wrk2. Uses a three-node layout (Client → Reverse Proxy → Server). | Wrk2Executor | NginxServerExecutor (×2) | linux-x64 | +| PERF-WEB-ASPNET-WRK.json | ASP.NET TechEmpower JSON serialization benchmark using wrk. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-WRK-AFFINITY.json | ASP.NET TechEmpower JSON serialization benchmark using wrk with CPU core affinity. | WrkExecutor | AspNetServerExecutor | linux-x64, linux-arm64, win-x64, win-arm64 | +| PERF-WEB-ASPNET-ORCHARD-WRK.json | ASP.NET OrchardCore CMS benchmark using wrk. | WrkExecutor | AspNetOrchardServerExecutor | linux-x64, linux-arm64 | + +## Parameters +The following table describes the key parameters supported by the Wrk/Wrk2 executors. + +| Parameter | Description | Default | +|----------------------|-----------------------------------------------------------------------------------------------|-----------------| +| PackageName | The name of the wrk or wrk2 dependency package. | *required* | +| CommandArguments | The wrk/wrk2 command-line arguments (threads, connections, duration, URL, etc.). | *required* | +| TargetService | The target service type: `server`, `rp` (reverse-proxy), or `apigw` (API gateway). | auto-detected | +| TestDuration | Duration of the test run (e.g., `00:02:30`). | profile-defined | +| Timeout | Maximum time to wait for server availability. | 5 minutes | +| WarmUp | When `true`, the run is a warm-up pass and metrics are not captured. | `false` | +| EmitLatencySpectrum | When `true`, the fine-grained latency spectrum is emitted as additional metrics. | `false` | +| BindToCores | When `true`, the wrk process is pinned to specific CPU cores. | `false` | +| CoreAffinity | CPU core affinity specification (e.g., `0-3`, `0,2,4,6`). Required when `BindToCores` is `true`. | *none* | + +## Wrk Command Line Options +The following are the key command-line options for wrk and wrk2. These are referenced in the `CommandArguments` +parameter in Virtual Client profiles. + +| Option | Description | +|--------------------|--------------------------------------------------------------------------------------------------| +| `-t, --threads` | Total number of threads to use. | +| `-c, --connections` | Total number of HTTP connections to keep open (each thread handles N = connections/threads). | +| `-d, --duration` | Duration of the test (e.g., `2s`, `2m`, `2h`). | +| `-R, --rate` | Total requests per second (wrk2 only). Enables constant-throughput, corrected-latency recording. | +| `-L, --latency` | Print detailed latency statistics (HdrHistogram percentile distribution). | +| `--timeout` | Record a timeout if a response is not received within this amount of time. | + +## Translating a Profile to a Command +Virtual Client profiles define wrk parameters declaratively. The executor translates them into a wrk command line +at runtime. For example, the following profile action: + +```json +{ + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "benchmark_measurement", + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s http://{serverip}:{ServerPort}/json", + "ThreadCount": "64", + "Connection": 4096, + "TestDuration": "00:00:15", + "ServerPort": 9876, + "Role": "Client" + } +} +``` + +Translates to the following wrk command (assuming server IP `10.0.0.5`): + +``` +wrk --latency --threads 64 --connections 4096 --duration 15s --timeout 10s http://10.0.0.5:9876/json +``` + +## Performance Notes +Generally, the more concurrent connections you configure, the higher the load on the server. At some point, increasing +connections further will result in diminishing returns in requests per second and increased latency, as the server becomes +saturated. This saturation point is useful for characterizing the maximum throughput capacity of the server under test.