From 067028925f14355d2710c81e0dea09d8836d306a Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Tue, 10 Feb 2026 20:49:26 -0800 Subject: [PATCH 1/8] Commit changes --- .../DiskSpd/DiskSpdMetricsParser.cs | 52 +++-- .../FIO/FioMetricsParser.cs | 215 ++++++++++-------- .../Memtier/MemtierMetricsParser.cs | 34 ++- .../MetricExtensionsTests.cs | 136 ++++++++++- .../Extensions/MetricExtensions.cs | 66 ++++-- .../VirtualClient.Contracts/Metric.cs | 14 +- website/docs/guides/0011-profiles.md | 98 ++++++++ 7 files changed, 445 insertions(+), 170 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions/DiskSpd/DiskSpdMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/DiskSpd/DiskSpdMetricsParser.cs index a67abe6426..b3ee8c0913 100644 --- a/src/VirtualClient/VirtualClient.Actions/DiskSpd/DiskSpdMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/DiskSpd/DiskSpdMetricsParser.cs @@ -182,9 +182,9 @@ private void ParseCPUResult() DataTable cpuUsage = DataTableExtensions.ConvertToDataTable( this.Sections[sectionName], DiskSpdMetricsParser.DiskSpdDataTableDelimiter, sectionName, columnNames: null); - this.metrics.AddRange(cpuUsage.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "percentage", namePrefix: $"cpu {cpuUsage.Columns[1].ColumnName.ToLower()} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 2)); - this.metrics.AddRange(cpuUsage.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "percentage", namePrefix: $"cpu {cpuUsage.Columns[2].ColumnName.ToLower()} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 2)); - this.metrics.AddRange(cpuUsage.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "percentage", namePrefix: $"cpu {cpuUsage.Columns[3].ColumnName.ToLower()} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 2)); + this.metrics.AddRange(cpuUsage.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "percentage", namePrefix: $"cpu {cpuUsage.Columns[1].ColumnName.ToLower()} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 5)); + this.metrics.AddRange(cpuUsage.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "percentage", namePrefix: $"cpu {cpuUsage.Columns[2].ColumnName.ToLower()} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 5)); + this.metrics.AddRange(cpuUsage.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "percentage", namePrefix: $"cpu {cpuUsage.Columns[3].ColumnName.ToLower()} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 5)); } private void ParseTotalIoResult() @@ -193,15 +193,16 @@ private void ParseTotalIoResult() DataTable totalIo = DataTableExtensions.ConvertToDataTable( this.Sections[sectionName], DiskSpdMetricsParser.DiskSpdDataTableDelimiter, sectionName, columnNames: null); List metrics = new List(); - metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "bytes", namePrefix: $"total {totalIo.Columns[1].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 2)); - metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "I/Os", namePrefix: $"total {totalIo.Columns[2].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 2)); - metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "MiB/s", namePrefix: $"total {totalIo.Columns[3].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter)); - metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 4, unit: "iops", namePrefix: $"total {totalIo.Columns[4].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter)); - metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 5, unit: "ms", namePrefix: $"total {totalIo.Columns[5].ColumnName} ", metricRelativity: MetricRelativity.LowerIsBetter)); + metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "bytes", namePrefix: $"total {totalIo.Columns[1].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 5)); + metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "I/Os", namePrefix: $"total {totalIo.Columns[2].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 5)); + metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "MiB/s", namePrefix: $"total {totalIo.Columns[3].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 1)); + metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 4, unit: "iops", namePrefix: $"total {totalIo.Columns[4].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 1)); + metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 5, unit: "ms", namePrefix: $"total {totalIo.Columns[5].ColumnName} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 1)); + // Total metrics ending with "total" remain at level 1 (most important) foreach (var metric in metrics.Where(m => m.Name.EndsWith("total") && (m.Unit == "iops" || m.Unit == "ms" || m.Unit == "MiB/s"))) { - metric.Verbosity = 0; + metric.Verbosity = 1; // Keep at 1 for critical } this.metrics.AddRange(metrics); @@ -213,15 +214,15 @@ private void ParseReadIoResult() DataTable readIo = DataTableExtensions.ConvertToDataTable( this.Sections[sectionName], DiskSpdMetricsParser.DiskSpdDataTableDelimiter, sectionName, columnNames: null); List metrics = new List(); - metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "bytes", namePrefix: $"read {readIo.Columns[1].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 2)); - metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "I/Os", namePrefix: $"read {readIo.Columns[2].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 2)); - metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "MiB/s", namePrefix: $"read {readIo.Columns[3].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter)); - metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 4, unit: "iops", namePrefix: $"read {readIo.Columns[4].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter)); - metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 5, unit: "ms", namePrefix: $"read {readIo.Columns[5].ColumnName} ", metricRelativity: MetricRelativity.LowerIsBetter)); + metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "bytes", namePrefix: $"read {readIo.Columns[1].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 5)); + metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "I/Os", namePrefix: $"read {readIo.Columns[2].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 5)); + metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "MiB/s", namePrefix: $"read {readIo.Columns[3].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 1)); + metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 4, unit: "iops", namePrefix: $"read {readIo.Columns[4].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 1)); + metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 5, unit: "ms", namePrefix: $"read {readIo.Columns[5].ColumnName} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 1)); foreach (var metric in metrics.Where(m => m.Name.EndsWith("total") && (m.Unit == "iops" || m.Unit == "ms" || m.Unit == "MiB/s"))) { - metric.Verbosity = 0; + metric.Verbosity = 1; } this.metrics.AddRange(metrics); @@ -233,14 +234,15 @@ private void ParseWriteIoResult() DataTable writeIo = DataTableExtensions.ConvertToDataTable( this.Sections[sectionName], DiskSpdMetricsParser.DiskSpdDataTableDelimiter, sectionName, columnNames: null); List metrics = new List(); - metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "bytes", namePrefix: $"write {writeIo.Columns[1].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 2)); - metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "I/Os", namePrefix: $"write {writeIo.Columns[2].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 2)); - metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "MiB/s", namePrefix: $"write {writeIo.Columns[3].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter)); - metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 4, unit: "iops", namePrefix: $"write {writeIo.Columns[4].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter)); - metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 5, unit: "ms", namePrefix: $"write {writeIo.Columns[5].ColumnName} ", metricRelativity: MetricRelativity.LowerIsBetter)); + metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "bytes", namePrefix: $"write {writeIo.Columns[1].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 5)); + metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "I/Os", namePrefix: $"write {writeIo.Columns[2].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 5)); + metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "MiB/s", namePrefix: $"write {writeIo.Columns[3].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 1)); + metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 4, unit: "iops", namePrefix: $"write {writeIo.Columns[4].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 1)); + metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 5, unit: "ms", namePrefix: $"write {writeIo.Columns[5].ColumnName} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 1)); + foreach (var metric in metrics.Where(m => m.Name.EndsWith("total") && (m.Unit == "iops" || m.Unit == "ms" || m.Unit == "MiB/s"))) { - metric.Verbosity = 0; + metric.Verbosity = 1; } this.metrics.AddRange(metrics); @@ -254,20 +256,20 @@ private void ParseLatencyResult() List metrics = new List(); if (this.readWriteMode != ReadWriteMode.WriteOnly) { - metrics.AddRange(latency.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "ms", namePrefix: "read latency ", metricRelativity: MetricRelativity.LowerIsBetter)); + metrics.AddRange(latency.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "ms", namePrefix: "read latency ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 1)); } if (this.readWriteMode != ReadWriteMode.ReadOnly) { - metrics.AddRange(latency.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "ms", namePrefix: "write latency ", metricRelativity: MetricRelativity.LowerIsBetter)); + metrics.AddRange(latency.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "ms", namePrefix: "write latency ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 1)); } - metrics.AddRange(latency.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "ms", namePrefix: "total latency ", metricRelativity: MetricRelativity.LowerIsBetter)); + metrics.AddRange(latency.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "ms", namePrefix: "total latency ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 1)); string[] criticalMetrics = { "total latency 50th", "total latency 90th", "total latency 99th" }; foreach (var metric in metrics.Where(m => criticalMetrics.Contains(m.Name))) { - metric.Verbosity = 0; + metric.Verbosity = 1; // Keep at 1 for critical percentiles } this.metrics.AddRange(metrics); diff --git a/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs index 9e11dbd5fa..308d694600 100644 --- a/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs @@ -122,29 +122,30 @@ private void AddLatencyHistogramMeasurements(IList metrics, JToken resul { foreach (JToken job in jobs) { - this.AddMeasurement(metrics, job, $"latency_us.2", "h000000_002", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.4", "h000000_004", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.10", "h000000_010", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.20", "h000000_020", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.50", "h000000_050", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.100", "h000000_100", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.250", "h000000_250", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.500", "h000000_500", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.750", "h000000_750", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.1000", "h000001_000", verbosity: 2); - - this.AddMeasurement(metrics, job, $"latency_ms.2", "h000002_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.4", "h000004_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.10", "h000010_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.20", "h000020_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.50", "h000050_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.100", "h000100_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.250", "h000250_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.500", "h000500_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.750", "h000750_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.1000", "h001000_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.2000", "h002000_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.['>=2000']", "hgt002000_000", verbosity: 2); + // Level 5 - Verbose: Histogram buckets (diagnostic metrics) + this.AddMeasurement(metrics, job, $"latency_us.2", "h000000_002", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.4", "h000000_004", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.10", "h000000_010", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.20", "h000000_020", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.50", "h000000_050", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.100", "h000000_100", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.250", "h000000_250", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.500", "h000000_500", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.750", "h000000_750", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.1000", "h000001_000", verbosity: 5); + + this.AddMeasurement(metrics, job, $"latency_ms.2", "h000002_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.4", "h000004_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.10", "h000010_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.20", "h000020_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.50", "h000050_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.100", "h000100_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.250", "h000250_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.500", "h000500_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.750", "h000750_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.1000", "h001000_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.2000", "h002000_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.['>=2000']", "hgt002000_000", verbosity: 5); } } } @@ -162,44 +163,52 @@ private void AddReadMeasurements(IList metrics, JToken resultsJson) { foreach (JToken job in jobs) { - this.AddMeasurement(metrics, job, $"read.io_bytes", "read_bytes", null, MetricRelativity.HigherIsBetter, verbosity: 2); - this.AddMeasurement(metrics, job, $"read.total_ios", "read_ios", null, MetricRelativity.HigherIsBetter, verbosity: 2); - this.AddMeasurement(metrics, job, $"read.short_ios", "read_ios_short", null, MetricRelativity.LowerIsBetter, verbosity: 2); - this.AddMeasurement(metrics, job, $"read.drop_ios", "read_ios_dropped", null, MetricRelativity.LowerIsBetter, verbosity: 2); - - this.AddMeasurement(metrics, job, $"read.bw", "read_bandwidth", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.bw_min", "read_bandwidth_min", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.bw_max", "read_bandwidth_max", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.bw_mean", "read_bandwidth_mean", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0); - this.AddMeasurement(metrics, job, $"read.bw_dev", "read_bandwidth_stdev", MetricUnit.KibibytesPerSecond, MetricRelativity.LowerIsBetter, verbosity: 2); - - this.AddMeasurement(metrics, job, $"read.iops", "read_iops", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.iops_min", "read_iops_min", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.iops_max", "read_iops_max", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.iops_mean", "read_iops_mean", null, MetricRelativity.HigherIsBetter, verbosity: 0); - this.AddMeasurement(metrics, job, $"read.iops_stddev", "read_iops_stdev", null, MetricRelativity.LowerIsBetter, verbosity: 2); - - this.AddMeasurement(metrics, job, $"read.lat_ns.min", "read_latency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.lat_ns.max", "read_latency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.lat_ns.mean", "read_latency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.lat_ns.stddev", "read_latency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); - - this.AddMeasurement(metrics, job, $"read.clat_ns.min", "read_completionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.clat_ns.max", "read_completionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.clat_ns.mean", "read_completionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.clat_ns.stddev", "read_completionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); - - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['50.000000']", "read_completionlatency_p50", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 0); - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['70.000000']", "read_completionlatency_p70", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['90.000000']", "read_completionlatency_p90", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['99.000000']", "read_completionlatency_p99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 0); - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['99.990000']", "read_completionlatency_p99_99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - - this.AddMeasurement(metrics, job, $"read.slat_ns.min", "read_submissionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.slat_ns.max", "read_submissionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.slat_ns.mean", "read_submissionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.slat_ns.stddev", "read_submissionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); - + // Level 5 - Verbose/Diagnostic metrics + this.AddMeasurement(metrics, job, $"read.io_bytes", "read_bytes", null, MetricRelativity.HigherIsBetter, verbosity: 5); + this.AddMeasurement(metrics, job, $"read.total_ios", "read_ios", null, MetricRelativity.HigherIsBetter, verbosity: 5); + this.AddMeasurement(metrics, job, $"read.short_ios", "read_ios_short", null, MetricRelativity.LowerIsBetter, verbosity: 5); + this.AddMeasurement(metrics, job, $"read.drop_ios", "read_ios_dropped", null, MetricRelativity.LowerIsBetter, verbosity: 5); + + // Level 1 - Standard/Critical bandwidth metrics (most important) + this.AddMeasurement(metrics, job, $"read.bw", "read_bandwidth", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.bw_min", "read_bandwidth_min", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.bw_max", "read_bandwidth_max", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.bw_mean", "read_bandwidth_mean", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.bw_dev", "read_bandwidth_stdev", MetricUnit.KibibytesPerSecond, MetricRelativity.LowerIsBetter, verbosity: 5); + + // Level 1 - Standard/Critical IOPS metrics + this.AddMeasurement(metrics, job, $"read.iops", "read_iops", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.iops_min", "read_iops_min", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.iops_max", "read_iops_max", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.iops_mean", "read_iops_mean", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.iops_stddev", "read_iops_stdev", null, MetricRelativity.LowerIsBetter, verbosity: 5); + + // Level 1 - Standard latency metrics + this.AddMeasurement(metrics, job, $"read.lat_ns.min", "read_latency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.lat_ns.max", "read_latency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.lat_ns.mean", "read_latency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.lat_ns.stddev", "read_latency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 5); + + // Level 1 - Standard completion latency metrics + this.AddMeasurement(metrics, job, $"read.clat_ns.min", "read_completionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.clat_ns.max", "read_completionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.clat_ns.mean", "read_completionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.clat_ns.stddev", "read_completionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 5); + + // Level 1 - Critical percentiles + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['50.000000']", "read_completionlatency_p50", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['99.000000']", "read_completionlatency_p99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + + // Level 3 - Detailed: Other percentiles + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['70.000000']", "read_completionlatency_p70", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 3); + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['90.000000']", "read_completionlatency_p90", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 3); + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['99.990000']", "read_completionlatency_p99_99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 3); + + // Level 1 - Standard submission latency metrics + this.AddMeasurement(metrics, job, $"read.slat_ns.min", "read_submissionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.slat_ns.max", "read_submissionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.slat_ns.mean", "read_submissionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.slat_ns.stddev", "read_submissionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 5); } } } @@ -217,44 +226,52 @@ private void AddWriteMeasurements(IList metrics, JToken resultsJson) { foreach (JToken job in jobs) { - this.AddMeasurement(metrics, job, $"write.io_bytes", "write_bytes", null, MetricRelativity.HigherIsBetter, verbosity: 2); - this.AddMeasurement(metrics, job, $"write.total_ios", "write_ios", null, MetricRelativity.HigherIsBetter, verbosity: 2); - this.AddMeasurement(metrics, job, $"write.short_ios", "write_ios_short", null, MetricRelativity.LowerIsBetter, verbosity: 2); - this.AddMeasurement(metrics, job, $"write.drop_ios", "write_ios_dropped", null, MetricRelativity.LowerIsBetter, verbosity: 2); - - this.AddMeasurement(metrics, job, $"write.bw", "write_bandwidth", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.bw_min", "write_bandwidth_min", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.bw_max", "write_bandwidth_max", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.bw_mean", "write_bandwidth_mean", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0); - this.AddMeasurement(metrics, job, $"write.bw_dev", "write_bandwidth_stdev", MetricUnit.KibibytesPerSecond, MetricRelativity.LowerIsBetter, verbosity: 2); - - this.AddMeasurement(metrics, job, $"write.iops", "write_iops", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.iops_min", "write_iops_min", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.iops_max", "write_iops_max", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.iops_mean", "write_iops_mean", null, MetricRelativity.HigherIsBetter, verbosity: 0); - this.AddMeasurement(metrics, job, $"write.iops_stddev", "write_iops_stdev", null, MetricRelativity.LowerIsBetter, verbosity: 2); - - this.AddMeasurement(metrics, job, $"write.lat_ns.min", "write_latency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.lat_ns.max", "write_latency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.lat_ns.mean", "write_latency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.lat_ns.stddev", "write_latency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); - - this.AddMeasurement(metrics, job, $"write.clat_ns.min", "write_completionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.clat_ns.max", "write_completionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.clat_ns.mean", "write_completionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.clat_ns.stddev", "write_completionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); - - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['50.000000']", "write_completionlatency_p50", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 0); - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['70.000000']", "write_completionlatency_p70", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['90.000000']", "write_completionlatency_p90", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['99.000000']", "write_completionlatency_p99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 0); - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['99.990000']", "write_completionlatency_p99_99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - - this.AddMeasurement(metrics, job, $"write.slat_ns.min", "write_submissionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.slat_ns.max", "write_submissionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.slat_ns.mean", "write_submissionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.slat_ns.stddev", "write_submissionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); - + // Level 5 - Verbose/Diagnostic metrics + this.AddMeasurement(metrics, job, $"write.io_bytes", "write_bytes", null, MetricRelativity.HigherIsBetter, verbosity: 5); + this.AddMeasurement(metrics, job, $"write.total_ios", "write_ios", null, MetricRelativity.HigherIsBetter, verbosity: 5); + this.AddMeasurement(metrics, job, $"write.short_ios", "write_ios_short", null, MetricRelativity.LowerIsBetter, verbosity: 5); + this.AddMeasurement(metrics, job, $"write.drop_ios", "write_ios_dropped", null, MetricRelativity.LowerIsBetter, verbosity: 5); + + // Level 1 - Standard/Critical bandwidth metrics + this.AddMeasurement(metrics, job, $"write.bw", "write_bandwidth", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.bw_min", "write_bandwidth_min", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.bw_max", "write_bandwidth_max", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.bw_mean", "write_bandwidth_mean", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.bw_dev", "write_bandwidth_stdev", MetricUnit.KibibytesPerSecond, MetricRelativity.LowerIsBetter, verbosity: 5); + + // Level 1 - Standard/Critical IOPS metrics + this.AddMeasurement(metrics, job, $"write.iops", "write_iops", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.iops_min", "write_iops_min", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.iops_max", "write_iops_max", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.iops_mean", "write_iops_mean", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.iops_stddev", "write_iops_stdev", null, MetricRelativity.LowerIsBetter, verbosity: 5); + + // Level 1 - Standard latency metrics + this.AddMeasurement(metrics, job, $"write.lat_ns.min", "write_latency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.lat_ns.max", "write_latency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.lat_ns.mean", "write_latency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.lat_ns.stddev", "write_latency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 5); + + // Level 1 - Standard completion latency metrics + this.AddMeasurement(metrics, job, $"write.clat_ns.min", "write_completionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.clat_ns.max", "write_completionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.clat_ns.mean", "write_completionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.clat_ns.stddev", "write_completionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 5); + + // Level 1 - Critical percentiles + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['50.000000']", "write_completionlatency_p50", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['99.000000']", "write_completionlatency_p99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + + // Level 3 - Detailed: Other percentiles + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['70.000000']", "write_completionlatency_p70", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 3); + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['90.000000']", "write_completionlatency_p90", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 3); + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['99.990000']", "write_completionlatency_p99_99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 3); + + // Level 1 - Standard submission latency metrics + this.AddMeasurement(metrics, job, $"write.slat_ns.min", "write_submissionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.slat_ns.max", "write_submissionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.slat_ns.mean", "write_submissionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.slat_ns.stddev", "write_submissionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 5); } } } diff --git a/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs index 687c3237ee..a831a23155 100644 --- a/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs @@ -308,93 +308,103 @@ private static IDictionary CreateMetricsBin(string metr { return new Dictionary { - // e.g. - // Throughput-Req/sec - // GET_Throughput-Req/sec - // GET_Throughput-Req/sec - // Latency-Avg - // Latency-Avg (Gets) - // Latency-Avg (Sets) + // Level 1 - Critical: Throughput { "Ops/sec", new MetricAggregate( metricNamePrefix == null ? "Throughput" : $"{metricNamePrefix}-Throughput", metricUnit: MetricUnit.RequestsPerSec, relativity: MetricRelativity.HigherIsBetter, - verbosity: 0, + verbosity: 1, // Level 1 - Most important description: "Total number of requests/operations per second during the period of time.") }, + // Level 1 - Standard: Hits/Misses { "Hits/sec", new MetricAggregate( metricNamePrefix == null ? "Hits/sec" : $"{metricNamePrefix}-Hits/sec", + metricUnit: MetricUnit.RequestsPerSec, relativity: MetricRelativity.HigherIsBetter, + verbosity: 1, description: "Total number of cache hits per second during the period of time.") }, { "Misses/sec", new MetricAggregate( metricNamePrefix == null ? "Misses/sec" : $"{metricNamePrefix}-Misses/sec", + metricUnit: MetricUnit.RequestsPerSec, relativity: MetricRelativity.LowerIsBetter, - description: "Total number of cache misses per second during a period of time. This is an indication of data evictions due to reaching memory limits.") + verbosity: 1, + description: "Total number of cache misses per second during a period of time.") }, + // Level 1 - Standard: Average latency { "AvgLatency", new MetricAggregate( metricNamePrefix == null ? "Latency-Avg" : $"{metricNamePrefix}-Latency-Avg", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, + verbosity: 1, description: "Average latency for requests/operations during the period of time.") }, + // Level 1 - Critical: P50 latency { "p50Latency", new MetricAggregate( metricNamePrefix == null ? "Latency-P50" : $"{metricNamePrefix}-Latency-P50", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, - verbosity: 0, + verbosity: 1, description: "The latency for 50% of all requests was at or under this value.") }, + // Level 3 - Detailed: P90 latency { "p90Latency", new MetricAggregate( metricNamePrefix == null ? "Latency-P90" : $"{metricNamePrefix}-Latency-P90", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, + verbosity: 3, description: "The latency for 90% of all requests was at or under this value.") }, + // Level 3 - Detailed: P95 latency { "p95Latency", new MetricAggregate( metricNamePrefix == null ? "Latency-P95" : $"{metricNamePrefix}-Latency-P95", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, + verbosity: 3, description: "The latency for 95% of all requests was at or under this value.") }, + // Level 1 - Critical: P99 latency { "p99Latency", new MetricAggregate( metricNamePrefix == null ? "Latency-P99" : $"{metricNamePrefix}-Latency-P99", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, - verbosity: 0, + verbosity: 1, description: "The latency for 99% of all requests was at or under this value.") }, + // Level 3 - Detailed: P99.9 latency { "p99.9Latency", new MetricAggregate( metricNamePrefix == null ? "Latency-P99.9" : $"{metricNamePrefix}-Latency-P99.9", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, + verbosity: 3, description: "The latency for 99.9% of all requests was at or under this value.") }, + // Level 1 - Critical: Bandwidth { "KB/sec", new MetricAggregate( metricNamePrefix == null ? "Bandwidth" : $"{metricNamePrefix}-Bandwidth", metricUnit: MetricUnit.KilobytesPerSecond, relativity: MetricRelativity.HigherIsBetter, - verbosity: 0, + verbosity: 1, description: "Total amount of data transferred per second during the period of time.") } }; diff --git a/src/VirtualClient/VirtualClient.Contracts.UnitTests/MetricExtensionsTests.cs b/src/VirtualClient/VirtualClient.Contracts.UnitTests/MetricExtensionsTests.cs index f16a0c3fa2..835577f5b9 100644 --- a/src/VirtualClient/VirtualClient.Contracts.UnitTests/MetricExtensionsTests.cs +++ b/src/VirtualClient/VirtualClient.Contracts.UnitTests/MetricExtensionsTests.cs @@ -38,11 +38,11 @@ public void SetupTest() new Metric("write_completionlatency_p99_99", 3267543), new Metric("write_submissionlatency_mean", 15.35467863), - new Metric("verbose_test_1", 123, "unit", MetricRelativity.HigherIsBetter, verbosity: 0), - new Metric("verbose_test_2", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 0), - new Metric("verbose_test_3", 123, "unit", MetricRelativity.HigherIsBetter, verbosity: 2), - new Metric("verbose_test_4", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 2), - new Metric("verbose_test_5", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 2), + new Metric("verbose_test_1", 123, "unit", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("verbose_test_2", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("verbose_test_3", 123, "unit", MetricRelativity.HigherIsBetter, verbosity: 3), + new Metric("verbose_test_4", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 3), + new Metric("verbose_test_5", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 5), }; } @@ -71,11 +71,11 @@ public void MetricFiltersCorrectFiltersVerbosity() IEnumerable filter = new List { "Verbosity:1" }; CollectionAssert.AreEquivalent(this.metrics.Where(m => m.Verbosity <= 1).Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); - filter = new List { "Verbosity:0" }; - CollectionAssert.AreEquivalent(this.metrics.Where(m => m.Verbosity == 0).Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); + filter = new List { "Verbosity:3" }; + CollectionAssert.AreEquivalent(this.metrics.Where(m => m.Verbosity <= 3).Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); - filter = new List { "Verbosity:2" }; - CollectionAssert.AreEquivalent(this.metrics.Where(m => m.Verbosity <= 2).Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); + filter = new List { "Verbosity:5" }; + CollectionAssert.AreEquivalent(this.metrics.Where(m => m.Verbosity <= 5).Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); filter = new List { "Verbosity:others" }; CollectionAssert.AreEquivalent(Enumerable.Empty().Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); @@ -172,7 +172,7 @@ public void FilterByExtensionReturnsTheExpectedFilteredWithBothVerbosityAndText( List filters = new List { "test_2", - "verbosity:0", + "verbosity:1", }; IEnumerable expectedMetrics = this.metrics.Where(m => m.Name == "verbose_test_2"); @@ -182,5 +182,121 @@ public void FilterByExtensionReturnsTheExpectedFilteredWithBothVerbosityAndText( Assert.IsNotEmpty(actualMetrics); CollectionAssert.AreEquivalent(expectedMetrics, actualMetrics); } + + [Test] + public void FilterByExtensionSupportsAllVerbosityLevels() + { + // Setup metrics with different verbosity levels (1, 3, 5 only) + var metrics = new List + { + new Metric("critical_metric_1a", 1, "unit", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("critical_metric_1b", 2, "unit", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("detailed_metric_3a", 3, "unit", MetricRelativity.HigherIsBetter, verbosity: 3), + new Metric("detailed_metric_3b", 4, "unit", MetricRelativity.HigherIsBetter, verbosity: 3), + new Metric("verbose_metric_5a", 5, "unit", MetricRelativity.HigherIsBetter, verbosity: 5), + new Metric("verbose_metric_5b", 6, "unit", MetricRelativity.HigherIsBetter, verbosity: 5) + }; + + // Test verbosity:1 - should return only level 1 metrics + var filter1 = new List { "verbosity:1" }; + var result1 = metrics.FilterBy(filter1); + Assert.AreEqual(2, result1.Count()); + Assert.IsTrue(result1.All(m => m.Verbosity == 1)); + + // Test verbosity:3 - should return level 1 + level 3 metrics + var filter3 = new List { "verbosity:3" }; + var result3 = metrics.FilterBy(filter3); + Assert.AreEqual(4, result3.Count()); + Assert.IsTrue(result3.All(m => m.Verbosity <= 3)); + + // Test verbosity:5 - should return all metrics + var filter5 = new List { "verbosity:5" }; + var result5 = metrics.FilterBy(filter5); + Assert.AreEqual(6, result5.Count()); + } + + [Test] + public void FilterByExtensionHandlesInvalidVerbosityValues() + { + var metrics = new List + { + new Metric("test_metric", 1, "unit", MetricRelativity.HigherIsBetter, verbosity: 1) + }; + + // Test invalid verbosity format + var invalidFilter1 = new List { "verbosity:invalid" }; + var result1 = metrics.FilterBy(invalidFilter1); + Assert.AreEqual(0, result1.Count()); + + // Test out of range verbosity (> 5) + var invalidFilter2 = new List { "verbosity:10" }; + var result2 = metrics.FilterBy(invalidFilter2); + Assert.AreEqual(0, result2.Count()); + + // Test verbosity less than 1 + var invalidFilter3 = new List { "verbosity:0" }; + var result3 = metrics.FilterBy(invalidFilter3); + Assert.AreEqual(0, result3.Count()); + } + + [Test] + public void FilterByExtensionSupportsExclusionFilters() + { + var metrics = new List + { + new Metric("h000_metric", 1), + new Metric("h001_metric", 2), + new Metric("bandwidth", 3), + new Metric("iops", 4) + }; + + // Test exclusion filter + var exclusionFilter = new List { "-h000*", "-h001*" }; + var result = metrics.FilterBy(exclusionFilter); + Assert.AreEqual(2, result.Count()); + Assert.IsFalse(result.Any(m => m.Name.StartsWith("h00"))); + Assert.IsTrue(result.Any(m => m.Name == "bandwidth")); + Assert.IsTrue(result.Any(m => m.Name == "iops")); + } + + [Test] + public void FilterByExtensionComposesVerbosityAndNameFilters() + { + var metrics = new List + { + new Metric("bandwidth_read", 1, "MB/s", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("bandwidth_write", 2, "MB/s", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("iops_read", 3, "ops/s", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("iops_write", 4, "ops/s", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("latency_p99", 5, "ms", MetricRelativity.LowerIsBetter, verbosity: 3) + }; + + // Filter: verbosity <= 1 AND name contains "bandwidth" + var composedFilter = new List { "verbosity:1", "bandwidth" }; + var result = metrics.FilterBy(composedFilter); + Assert.AreEqual(2, result.Count()); + Assert.IsTrue(result.All(m => m.Name.Contains("bandwidth") && m.Verbosity <= 1)); + } + + [Test] + public void FilterByExtensionSupportsComplexFilterCombinations() + { + var metrics = new List + { + new Metric("h000_latency", 1, "ms", MetricRelativity.LowerIsBetter, verbosity: 3), + new Metric("h001_latency", 2, "ms", MetricRelativity.LowerIsBetter, verbosity: 3), + new Metric("bandwidth_read", 3, "MB/s", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("bandwidth_write", 4, "MB/s", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("iops", 5, "ops/s", MetricRelativity.HigherIsBetter, verbosity: 1) + }; + + // Complex filter: verbosity <= 3, exclude h00* metrics, include only bandwidth or iops + var complexFilter = new List { "verbosity:3", "-h00*", "bandwidth|iops" }; + var result = metrics.FilterBy(complexFilter); + + Assert.IsTrue(result.All(m => m.Verbosity <= 3)); + Assert.IsFalse(result.Any(m => m.Name.StartsWith("h00"))); + Assert.IsTrue(result.All(m => m.Name.Contains("bandwidth") || m.Name.Contains("iops"))); + } } } diff --git a/src/VirtualClient/VirtualClient.Contracts/Extensions/MetricExtensions.cs b/src/VirtualClient/VirtualClient.Contracts/Extensions/MetricExtensions.cs index 402022c4b5..d7f60f2bc9 100644 --- a/src/VirtualClient/VirtualClient.Contracts/Extensions/MetricExtensions.cs +++ b/src/VirtualClient/VirtualClient.Contracts/Extensions/MetricExtensions.cs @@ -33,14 +33,29 @@ public static void AddMetadata(this IEnumerable metrics, IDictionary - /// Filters the set of metrics down to those whose names match or contain the filter terms - /// provided (case-insensitive). + /// Filters the set of metrics down to those that match the filter criteria provided. + /// Supports both verbosity-based filtering and regex-based name filtering (case-insensitive). /// /// The set of metrics to filter down. - /// A set of terms to match against the metric names. + /// + /// A set of filter terms. Can include: + /// - Verbosity filters: "verbosity:N" where N is 1-5 (filters metrics with verbosity less than or equal to N) + /// - Name filters: Any regex pattern to match against metric names (case-insensitive) + /// - Exclusion filters: Prefix with "-" to exclude matching metrics (e.g., "-h000*") + /// /// - /// A set of metrics whose names match or contain the filter terms (case-insensitive). + /// A filtered set of metrics matching the criteria. /// + /// + /// Verbosity levels (only 1, 3, and 5 are currently used; 2 and 4 are reserved): + /// - 1 (Standard/Critical): Most important metrics - bandwidth, throughput, IOPS, key latency percentiles (p50, p99) + /// - 2 (Reserved): Reserved for future expansion + /// - 3 (Detailed): Additional detailed metrics - supplementary percentiles (p70, p90, p95, p99.9) + /// - 4 (Reserved): Reserved for future expansion + /// - 5 (Verbose): All diagnostic/internal metrics - histogram buckets, standard deviations, byte counts, I/O counts + /// + /// Filters are composable - verbosity filtering is applied first, then name-based filtering. + /// public static IEnumerable FilterBy(this IEnumerable metrics, IEnumerable filterTerms) { metrics.ThrowIfNull(nameof(metrics)); @@ -49,32 +64,39 @@ public static IEnumerable FilterBy(this IEnumerable metrics, IEn if (filterTerms?.Any() == true) { - string verbosityFilter = filterTerms.Where(f => f.Contains("verbosity", StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); - if (!string.IsNullOrEmpty(verbosityFilter)) + // Step 1: Handle verbosity filtering first + string verbosityFilter = filterTerms.FirstOrDefault(f => f.Contains("verbosity", StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrEmpty(verbosityFilter)) { - switch (verbosityFilter.ToLower()) + // Extract the verbosity level from the filter (e.g., "verbosity:3" -> 3) + string[] parts = verbosityFilter.Split(':'); + if (parts.Length == 2 && int.TryParse(parts[1].Trim(), out int maxVerbosity) && maxVerbosity >= 1 && maxVerbosity <= 5) { - case "verbosity:0": - filteredMetrics = filteredMetrics.Where(m => ((int)m.Verbosity) <= 0); - break; - case "verbosity:1": - filteredMetrics = filteredMetrics.Where(m => ((int)m.Verbosity) <= 1); - break; - case "verbosity:2": - filteredMetrics = filteredMetrics.Where(m => ((int)m.Verbosity) <= 2); - break; - - default: - filteredMetrics = Enumerable.Empty(); - break; + // Filter metrics to include only those with verbosity <= maxVerbosity + filteredMetrics = filteredMetrics.Where(m => m.Verbosity <= maxVerbosity); + } + else + { + // Invalid verbosity format or out of range - return empty set + filteredMetrics = Enumerable.Empty(); } + // Remove verbosity filter from remaining filters filterTerms = filterTerms.Where(f => !f.Contains("verbosity", StringComparison.OrdinalIgnoreCase)); } + // Step 2: Handle exclusion filters (prefix with "-") + var exclusionFilters = filterTerms.Where(f => f.StartsWith("-")).Select(f => f.Substring(1)).ToList(); + if (exclusionFilters.Any()) + { + filteredMetrics = filteredMetrics.Where(m => !exclusionFilters.Contains(m.Name, MetricFilterComparer.Instance)); + filterTerms = filterTerms.Where(f => !f.StartsWith("-")); + } + + // Step 3: Handle inclusion/regex name filtering if (filterTerms?.Any() == true) { - filteredMetrics = metrics.Where(m => filterTerms.Contains(m.Name, MetricFilterComparer.Instance)).ToList(); + filteredMetrics = filteredMetrics.Where(m => filterTerms.Contains(m.Name, MetricFilterComparer.Instance)); } } @@ -102,7 +124,7 @@ public static void LogConsole(this IEnumerable metrics, string scenario, table.Columns.Add("Value", typeof(double)); table.Columns.Add("Unit", typeof(string)); - IEnumerable metricsToPrint = criticalOnly ? metrics.Where(m => m.Verbosity == 0).ToList() : metrics; + IEnumerable metricsToPrint = criticalOnly ? metrics.Where(m => m.Verbosity == 1).ToList() : metrics; foreach (Metric metric in metricsToPrint) { diff --git a/src/VirtualClient/VirtualClient.Contracts/Metric.cs b/src/VirtualClient/VirtualClient.Contracts/Metric.cs index fa152ef9d6..aebdd9ce14 100644 --- a/src/VirtualClient/VirtualClient.Contracts/Metric.cs +++ b/src/VirtualClient/VirtualClient.Contracts/Metric.cs @@ -140,8 +140,18 @@ public Metric(string name, double value, string unit, MetricRelativity relativit public IDictionary Metadata { get; } /// - /// Metric verbosity to descript importance of metric. Default to 1, which means standard. - /// Verbosity 0: Critical. Verbosity 1: Standard. Verbosity 2: Informational. + /// Metric verbosity to describe importance/priority of the metric. + /// + /// Verbosity Levels (only 1, 3, and 5 are currently used): + /// - 1 (Standard/Critical): Most important metrics for decision making - bandwidth, throughput, IOPS, key latency percentiles (p50, p99) + /// - 2 (Reserved): Reserved for future expansion + /// - 3 (Detailed): Additional detailed metrics - supplementary percentiles (p70, p90, p95, p99.9) + /// - 4 (Reserved): Reserved for future expansion + /// - 5 (Verbose): All diagnostic/internal metrics - histogram buckets, standard deviations, byte counts, I/O counts + /// + /// Currently, only levels 1, 3, and 5 are actively used. Levels 2 and 4 are reserved for future use. + /// + /// Default = 1 (Standard). /// public int Verbosity { get; set; } = 1; diff --git a/website/docs/guides/0011-profiles.md b/website/docs/guides/0011-profiles.md index 2aaa0b7e7c..11c7b3aa5f 100644 --- a/website/docs/guides/0011-profiles.md +++ b/website/docs/guides/0011-profiles.md @@ -382,6 +382,104 @@ e.g. ] ``` +## Metric Filtering +The Virtual Client supports filtering of metrics emitted by workloads and monitors using the `MetricFilters` parameter. This allows users to control which metrics are captured and +logged, reducing telemetry volume and focusing on metrics of interest. Metric filtering supports three filtering strategies: **verbosity-based filtering**, **regex-based inclusion +filtering**, and **exclusion filtering**. + +### Verbosity-Based Filtering +Metrics in the Virtual Client are assigned a verbosity level that indicates their importance for decision making: + +| Verbosity Level | Description | Example Metrics | +|-----------------|-------------|-----------------| +| 1 (Standard/Critical) | Most important metrics for decision making | bandwidth, throughput, IOPS, p50, p99 | +| 2 (Reserved) | Reserved for future expansion | N/A | +| 3 (Detailed) | Additional detailed metrics | p70, p90, p95, p99.9 | +| 4 (Reserved) | Reserved for future expansion | N/A | +| 5 (Verbose) | All diagnostic/internal metrics | histogram buckets, standard deviations, byte counts, I/O counts | + +To filter metrics by verbosity level, use the `verbosity:N` filter format, where N is a value between 1 and 5. This will include all metrics with a verbosity level less than or equal to N. + +``` json +"Actions": [ + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "RandomWrite_4k_BlockSize", + "PackageName": "fio", + "CommandLine": "--name=fio_test --size=10G --rw=randwrite --bs=4k", + "MetricFilters": "verbosity:1" + } + } +] +``` + +### Regex-Based Inclusion Filtering +Metrics can be filtered using regular expression patterns (case-insensitive). Multiple filter terms can be combined, and a metric will be included if it matches any of the patterns. + +``` json +"Actions": [ + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "RandomWrite_4k_BlockSize", + "PackageName": "fio", + "CommandLine": "--name=fio_test --size=10G --rw=randwrite --bs=4k", + "MetricFilters": "bandwidth,iops,_p99" + } + } +] +``` + +The filter above will include metrics whose names contain "bandwidth", "iops", or "_p99". More complex regex patterns are also supported: + +``` json +"MetricFilters": "(read|write)_(bandwidth|iops)" +``` + +### Exclusion Filtering +Metrics can be explicitly excluded by prefixing the filter term with a minus sign (`-`). This is useful for removing verbose or diagnostic metrics that are not needed. + +``` json +"Actions": [ + { + "Type": "RedisServerExecutor", + "Parameters": { + "Scenario": "Server", + "PackageName": "redis", + "CommandLine": "--protected-mode no", + "MetricFilters": "-h000*,-h001*" + } + } +] +``` + +### Combining Filter Types +Multiple filter types can be combined. Verbosity filtering is applied first, followed by exclusion filters, and then inclusion filters. + +``` json +"Actions": [ + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "RandomWrite_4k_BlockSize", + "PackageName": "fio", + "CommandLine": "--name=fio_test --size=10G --rw=randwrite --bs=4k", + "MetricFilters": "verbosity:3,-histogram*,bandwidth|iops|latency" + } + } +] +``` + +The example above will: +1. Include only metrics with verbosity ? 3 +2. Exclude metrics matching the pattern "histogram*" +3. From the remaining metrics, include only those matching "bandwidth", "iops", or "latency" + +### Default Behavior +When the `MetricFilters` parameter is not specified, all metrics emitted by the workload or monitor are captured and logged. This ensures backward compatibility with existing +profiles and allows users to see all available metrics during initial testing before applying filters to reduce telemetry volume. + ## Actions The section 'Actions' within the profile defines a set of 1 or more toolsets to execute on the system. The term 'workload' is often used synonymously with 'Actions' to describe the software that is the focus of the work Virtual Client will perform on the system. For example, the section might include an action responsible for executing From 671a303563a88fe34fee08b6c02399952d5283e9 Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Wed, 18 Feb 2026 20:37:31 -0800 Subject: [PATCH 2/8] update 0 --- .../3DMark/ThreeDMarkMetricsParser.cs | 6 ++-- .../7zip/Compression7zipMetricsParser.cs | 4 +-- .../ASPNET/BombardierMetricsParser.cs | 24 +++++++------- .../HPLinpack/HPLinpackMetricsParser.cs | 4 +-- .../HammerDB/HammerDBMetricsParser.cs | 4 +-- .../CPS/CPSMetricsParser.cs | 32 +++++++++---------- .../Latte/LatteMetricsParser.cs | 4 +-- .../NTttcp/NTttcpMetricsParser.cs | 16 +++++----- .../SockPerf/SockPerfMetricsParser.cs | 32 +++++++++---------- .../OpenSSL/OpenSslMetricsParser.cs | 14 ++++---- .../OpenSSL/TlsOpenSslMetricsParser.cs | 24 +++++++------- .../Redis/RedisBenchmarkMetricsParser.cs | 4 +-- .../SPECjbb/SpecJbbMetricsParser.cs | 2 +- .../SPECjvm/SpecJvmMetricsParser.cs | 4 +-- .../StressNg/StressNgMetricsParser.cs | 12 +++---- .../Extensibility/MetricDataPoint.cs | 8 +++-- 16 files changed, 99 insertions(+), 95 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkMetricsParser.cs index fac39805d9..e428e6bdcc 100644 --- a/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkMetricsParser.cs @@ -41,15 +41,15 @@ public override IList Parse() { if (this.Defintion == "custom_TSGT1.3dmdef") { - metrics.Add(new Metric("timespy.graphics.1 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest1"), "fps", MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric("timespy.graphics.1 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest1"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); } else if (this.Defintion == "custom_TSGT2.3dmdef") { - metrics.Add(new Metric("timespy.graphics.2 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric("timespy.graphics.2 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); } else if (this.Defintion == "custom_TSCT.3dmdef") { - metrics.Add(new Metric("timespy.cpu [fps]", this.ParseXMLTag("TimeSpyPerformanceCpuSection2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric("timespy.cpu [fps]", this.ParseXMLTag("TimeSpyPerformanceCpuSection2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); } } catch (Exception exc) diff --git a/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs index 85525a3b56..19d4934a59 100644 --- a/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs @@ -66,7 +66,7 @@ public override IList Parse() (Convert.ToDouble(this.SizeAndTime.Rows[2].ItemArray[2]) / Convert.ToDouble(this.SizeAndTime.Rows[0].ItemArray[2])) * 100, "precentage", relativity: MetricRelativity.LowerIsBetter, - verbosity: 2)); + verbosity: 5)); double compressionTime = 0; @@ -75,7 +75,7 @@ public override IList Parse() compressionTime += Convert.ToDouble(this.SizeAndTime.Rows[i].ItemArray[2]); } - metrics.Add(new Metric("Compression_Time", compressionTime, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 0)); + metrics.Add(new Metric("Compression_Time", compressionTime, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 1)); return metrics; } diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs index 9aa2736eb9..6adbbc58a7 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs @@ -32,18 +32,18 @@ public override IList Parse() this.metrics = new List(); Root root = JsonSerializer.Deserialize(this.PreprocessedText); - this.metrics.Add(new Metric("Latency Max", root.Result.Latency.Max, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); - this.metrics.Add(new Metric("Latency Average", root.Result.Latency.Mean, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency Stddev", root.Result.Latency.Stddev, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); - this.metrics.Add(new Metric("Latency P50", root.Result.Latency.Percentiles.P50, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 0)); - this.metrics.Add(new Metric("Latency P75", root.Result.Latency.Percentiles.P75, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency P90", root.Result.Latency.Percentiles.P90, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency P95", root.Result.Latency.Percentiles.P95, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency P99", root.Result.Latency.Percentiles.P99, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 0)); - - this.metrics.Add(new Metric("RequestPerSecond Max", root.Result.Rps.Max, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); - this.metrics.Add(new Metric("RequestPerSecond Average", root.Result.Rps.Mean, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 0)); - this.metrics.Add(new Metric("RequestPerSecond Stddev", root.Result.Rps.Stddev, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); + this.metrics.Add(new Metric("Latency Max", root.Result.Latency.Max, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + this.metrics.Add(new Metric("Latency Average", root.Result.Latency.Mean, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); + this.metrics.Add(new Metric("Latency Stddev", root.Result.Latency.Stddev, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + this.metrics.Add(new Metric("Latency P50", root.Result.Latency.Percentiles.P50, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); + this.metrics.Add(new Metric("Latency P75", root.Result.Latency.Percentiles.P75, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + this.metrics.Add(new Metric("Latency P90", root.Result.Latency.Percentiles.P90, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + this.metrics.Add(new Metric("Latency P95", root.Result.Latency.Percentiles.P95, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + this.metrics.Add(new Metric("Latency P99", root.Result.Latency.Percentiles.P99, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); + + this.metrics.Add(new Metric("RequestPerSecond Max", root.Result.Rps.Max, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 5)); + this.metrics.Add(new Metric("RequestPerSecond Average", root.Result.Rps.Mean, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 1)); + this.metrics.Add(new Metric("RequestPerSecond Stddev", root.Result.Rps.Stddev, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 5)); this.metrics.Add(new Metric("RequestPerSecond P50", root.Result.Rps.Percentiles.P50, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter)); this.metrics.Add(new Metric("RequestPerSecond P75", root.Result.Rps.Percentiles.P75, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter)); this.metrics.Add(new Metric("RequestPerSecond P90", root.Result.Rps.Percentiles.P90, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter)); diff --git a/src/VirtualClient/VirtualClient.Actions/HPLinpack/HPLinpackMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/HPLinpack/HPLinpackMetricsParser.cs index b0b2a8e583..242080260b 100644 --- a/src/VirtualClient/VirtualClient.Actions/HPLinpack/HPLinpackMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/HPLinpack/HPLinpackMetricsParser.cs @@ -81,8 +81,8 @@ public override IList Parse() { $"Q_W{st[0]}", st[4] }, }; - this.metrics.Add(new Metric($"Time", Convert.ToDouble(st[5]), "secs", MetricRelativity.Undefined, metadata: metadata, verbosity: 2)); - this.metrics.Add(new Metric($"GFlops", Convert.ToDouble(st[6]), "Gflops", metadata: metadata, relativity: MetricRelativity.HigherIsBetter, verbosity: 0)); + this.metrics.Add(new Metric($"Time", Convert.ToDouble(st[5]), "secs", MetricRelativity.Undefined, metadata: metadata, verbosity: 5)); + this.metrics.Add(new Metric($"GFlops", Convert.ToDouble(st[6]), "Gflops", metadata: metadata, relativity: MetricRelativity.HigherIsBetter, verbosity: 1)); } } diff --git a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBMetricsParser.cs index 48a17b8eef..a973c73062 100644 --- a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBMetricsParser.cs @@ -58,14 +58,14 @@ public override IList Parse() metrics.First(m => m.Name == "Operations/min").Value / 60, MetricUnit.OperationsPerSec, relativity: MetricRelativity.HigherIsBetter, - verbosity: 0)); + verbosity: 1)); metrics.Add(new Metric( "Transactions/sec", metrics.First(m => m.Name == "Transactions/min").Value / 60, MetricUnit.TransactionsPerSec, relativity: MetricRelativity.HigherIsBetter, - verbosity: 0)); + verbosity: 1)); return metrics; } diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs index 1d1489957b..90d4f5031e 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs @@ -82,24 +82,24 @@ private static void AddStatisticalMetrics(IList metrics, List ti double lowerCI = mean - t; double upperCI = mean + t; - metrics.Add(new Metric("ConnectsPerSec_Min", connectsPerSec.Min())); - metrics.Add(new Metric("ConnectsPerSec_Max", connectsPerSec.Max())); - metrics.Add(new Metric("ConnectsPerSec_Med", connectsPerSec.Median())); - metrics.Add(new Metric("ConnectsPerSec_Avg", connectsPerSec.Average(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric("ConnectsPerSec_P25", connectsPerSec.Percentile(25))); - metrics.Add(new Metric("ConnectsPerSec_P50", connectsPerSec.Percentile(50))); - metrics.Add(new Metric("ConnectsPerSec_P75", connectsPerSec.Percentile(75))); - metrics.Add(new Metric("ConnectsPerSec_P90", connectsPerSec.Percentile(90))); - metrics.Add(new Metric("ConnectsPerSec_P99", connectsPerSec.Percentile(99))); - metrics.Add(new Metric("ConnectsPerSec_P99_9", Statistics.QuantileCustom(connectsPerSec, 1d - 0.001d, QuantileDefinition.R3))); - metrics.Add(new Metric("ConnectsPerSec_P99_99", Statistics.QuantileCustom(connectsPerSec, 1d - 0.0001d, QuantileDefinition.R3))); - metrics.Add(new Metric("ConnectsPerSec_P99_999", Statistics.QuantileCustom(connectsPerSec, 1d - 0.00001d, QuantileDefinition.R3))); + metrics.Add(new Metric("ConnectsPerSec_Min", connectsPerSec.Min(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 3)); + metrics.Add(new Metric("ConnectsPerSec_Max", connectsPerSec.Max(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 5)); + metrics.Add(new Metric("ConnectsPerSec_Med", connectsPerSec.Median(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 3)); + metrics.Add(new Metric("ConnectsPerSec_Avg", connectsPerSec.Average(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric("ConnectsPerSec_P25", connectsPerSec.Percentile(25), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 3)); + metrics.Add(new Metric("ConnectsPerSec_P50", connectsPerSec.Percentile(50), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric("ConnectsPerSec_P75", connectsPerSec.Percentile(75), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 3)); + metrics.Add(new Metric("ConnectsPerSec_P90", connectsPerSec.Percentile(90), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 3)); + metrics.Add(new Metric("ConnectsPerSec_P99", connectsPerSec.Percentile(99), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric("ConnectsPerSec_P99_9", Statistics.QuantileCustom(connectsPerSec, 1d - 0.001d, QuantileDefinition.R3), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 3)); + metrics.Add(new Metric("ConnectsPerSec_P99_99", Statistics.QuantileCustom(connectsPerSec, 1d - 0.0001d, QuantileDefinition.R3), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 5)); + metrics.Add(new Metric("ConnectsPerSec_P99_999", Statistics.QuantileCustom(connectsPerSec, 1d - 0.00001d, QuantileDefinition.R3), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 5)); double median = Statistics.Median(connectsPerSec); double[] absoluteDeviations = connectsPerSec.Select(x => Math.Abs(x - median)).ToArray(); - metrics.Add(new Metric("ConnectsPerSec_Mad", Statistics.Median(absoluteDeviations), MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); - metrics.Add(new Metric("ConnectsPerSec_StandardErrorMean", sem, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); - metrics.Add(new Metric("ConnectsPerSec_LowerCI", lowerCI, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); - metrics.Add(new Metric("ConnectsPerSec_UpperCI", upperCI, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_Mad", Statistics.Median(absoluteDeviations), MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("ConnectsPerSec_StandardErrorMean", sem, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("ConnectsPerSec_LowerCI", lowerCI, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("ConnectsPerSec_UpperCI", upperCI, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 5)); } /// diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteMetricsParser.cs index c09ac0905a..36b4772caa 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteMetricsParser.cs @@ -144,7 +144,7 @@ private void AddLatencyPercentileMetrics(IList metrics, IEnumerable entry.FrequencyInTotal <= totalObservations * .75) @@ -178,7 +178,7 @@ private void AddLatencyPercentileMetrics(IList metrics, IEnumerable entry.FrequencyInTotal <= totalObservations * .999) diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs index b0f0d7c913..1894f72c42 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs @@ -51,14 +51,14 @@ public override IList Parse() NTttcpMetric throughputMetric = this.result.Throughput.First(t => t.Units.Equals("mbps")); IList metricList = new List() { - new Metric("TotalBytesMB", this.result.TotalBytesMB.Value, this.result.TotalBytesMB.Units, MetricRelativity.HigherIsBetter, verbosity: 2), - new Metric("AvgBytesPerCompl", this.result.AverageBytesPerCompletion.Value, this.result.AverageBytesPerCompletion.Units, MetricRelativity.Undefined, verbosity: 2), - new Metric("AvgFrameSize", this.result.AverageFrameSize.Value, this.result.AverageFrameSize.Units, MetricRelativity.Undefined, verbosity: 2), - new Metric("ThroughputMbps", throughputMetric.Value, throughputMetric.Units, MetricRelativity.HigherIsBetter, verbosity: 0), - new Metric("AvgPacketsPerInterrupt", this.result.AveragePacketsPerInterrupt.Value, this.result.AveragePacketsPerInterrupt.Units, MetricRelativity.Undefined, verbosity: 2), - new Metric("InterruptsPerSec", this.result.Interrupts.Value, this.result.Interrupts.Units, MetricRelativity.Undefined), - new Metric("PacketsRetransmitted", this.result.PacketsRetransmitted, MetricUnit.Count, MetricRelativity.LowerIsBetter, verbosity: 2), - new Metric("Errors", this.result.Errors, MetricUnit.Count, MetricRelativity.LowerIsBetter), + new Metric("TotalBytesMB", this.result.TotalBytesMB.Value, this.result.TotalBytesMB.Units, MetricRelativity.HigherIsBetter, verbosity: 5), + new Metric("AvgBytesPerCompl", this.result.AverageBytesPerCompletion.Value, this.result.AverageBytesPerCompletion.Units, MetricRelativity.Undefined, verbosity: 5), + new Metric("AvgFrameSize", this.result.AverageFrameSize.Value, this.result.AverageFrameSize.Units, MetricRelativity.Undefined, verbosity: 5), + new Metric("ThroughputMbps", throughputMetric.Value, throughputMetric.Units, MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("AvgPacketsPerInterrupt", this.result.AveragePacketsPerInterrupt.Value, this.result.AveragePacketsPerInterrupt.Units, MetricRelativity.Undefined, verbosity: 5), + new Metric("InterruptsPerSec", this.result.Interrupts.Value, this.result.Interrupts.Units, MetricRelativity.Undefined, verbosity: 3), + new Metric("PacketsRetransmitted", this.result.PacketsRetransmitted, MetricUnit.Count, MetricRelativity.LowerIsBetter, verbosity: 5), + new Metric("Errors", this.result.Errors, MetricUnit.Count, MetricRelativity.LowerIsBetter, verbosity: 3), }; if (this.result.TcpAverageRtt != null) diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs index 248f29e5d9..f3d070b6f6 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs @@ -107,24 +107,24 @@ private static void AddStatisticalMetrics(IList metrics, List pa double lowerCI = mean - t; double upperCI = mean + t; - metrics.Add(new Metric("Latency-Min", packetsLatencyValues.Min(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-Max", packetsLatencyValues.Max(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-Avg", packetsLatencyValues.Average(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-P25", packetsLatencyValues.Percentile(25), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-P50", packetsLatencyValues.Percentile(50), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 0)); - metrics.Add(new Metric("Latency-P75", packetsLatencyValues.Percentile(75), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-P90", packetsLatencyValues.Percentile(90), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-P99", packetsLatencyValues.Percentile(99), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 0)); - metrics.Add(new Metric("Latency-P99.9", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-P99.99", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.0001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-P99.999", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.00001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); + metrics.Add(new Metric("Latency-Min", packetsLatencyValues.Min(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + metrics.Add(new Metric("Latency-Max", packetsLatencyValues.Max(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("Latency-Avg", packetsLatencyValues.Average(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); + metrics.Add(new Metric("Latency-P25", packetsLatencyValues.Percentile(25), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + metrics.Add(new Metric("Latency-P50", packetsLatencyValues.Percentile(50), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); + metrics.Add(new Metric("Latency-P75", packetsLatencyValues.Percentile(75), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + metrics.Add(new Metric("Latency-P90", packetsLatencyValues.Percentile(90), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + metrics.Add(new Metric("Latency-P99", packetsLatencyValues.Percentile(99), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); + metrics.Add(new Metric("Latency-P99.9", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + metrics.Add(new Metric("Latency-P99.99", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.0001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("Latency-P99.999", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.00001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); double median = Statistics.Median(packetsLatencyValues); double[] absoluteDeviations = packetsLatencyValues.Select(x => Math.Abs(x - median)).ToArray(); - metrics.Add(new Metric("Latency-Mad", Statistics.Median(absoluteDeviations), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); - metrics.Add(new Metric("Latency-StandardErrorMean", sem, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); - metrics.Add(new Metric("Latency-Stdev", sd, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); - metrics.Add(new Metric("Latency-LowerCI", lowerCI, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); - metrics.Add(new Metric("Latency-UpperCI", upperCI, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("Latency-Mad", Statistics.Median(absoluteDeviations), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("Latency-StandardErrorMean", sem, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("Latency-Stdev", sd, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("Latency-LowerCI", lowerCI, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("Latency-UpperCI", upperCI, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); } } } \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs index c1a348b6a9..e1a8e0d658 100644 --- a/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs @@ -171,7 +171,7 @@ public override IList Parse() // aes-128-cbc -1040172814.93 1589805.17k 1657786.91k 1674053.70k 1677365.52k 1633415.99k if (metricValue >= 0) { - metrics.Add(new Metric(metricName, metricValue, MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric(metricName, metricValue, MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); } } } @@ -187,11 +187,11 @@ public override IList Parse() { if (metricName.Contains("/")) { - metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 1)); } else { - metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter)); + metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 3)); } } } @@ -208,11 +208,11 @@ public override IList Parse() { if (metricName.Contains("/")) { - metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 1)); } else { - metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter)); + metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 3)); } } } @@ -229,11 +229,11 @@ public override IList Parse() { if (metricName.Contains("/")) { - metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 1)); } else { - metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter)); + metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 3)); } } } diff --git a/src/VirtualClient/VirtualClient.Actions/OpenSSL/TlsOpenSslMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/OpenSSL/TlsOpenSslMetricsParser.cs index 779a7ccd05..00a0b0b824 100644 --- a/src/VirtualClient/VirtualClient.Actions/OpenSSL/TlsOpenSslMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/OpenSSL/TlsOpenSslMetricsParser.cs @@ -61,20 +61,20 @@ public override IList Parse() // add metrics to the list - // set 1 - new connection metrics - metrics.Add(new Metric(TotalBytesRead, parsedMetrics[TotalBytesRead], MetricUnit.Bytes, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(NoOfConnections, parsedMetrics[NoOfConnections], MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(Duration, parsedMetrics[Duration], MetricUnit.Seconds)); - metrics.Add(new Metric(BytesperConnection, parsedMetrics[BytesperConnection], MetricUnit.BytesPerConnection, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(NewConnThroughput, newConnThroughput, MetricUnit.BytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(NewConnPersec, newConnPerSec, MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric(TotalBytesRead, parsedMetrics[TotalBytesRead], MetricUnit.Bytes, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(NoOfConnections, parsedMetrics[NoOfConnections], MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(Duration, parsedMetrics[Duration], MetricUnit.Seconds, MetricRelativity.Undefined, verbosity: 3)); + metrics.Add(new Metric(BytesperConnection, parsedMetrics[BytesperConnection], MetricUnit.BytesPerConnection, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(NewConnThroughput, newConnThroughput, MetricUnit.BytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(NewConnPersec, newConnPerSec, MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 1)); // set 2 - reuse connection metrics - metrics.Add(new Metric(ReuseTotalBytesRead, parsedMetrics[ReuseTotalBytesRead], MetricUnit.Bytes, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(ReuseNoOfConnections, parsedMetrics[ReuseNoOfConnections], MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(ReuseDuration, parsedMetrics[ReuseDuration], MetricUnit.Seconds)); - metrics.Add(new Metric(ReuseBytesperConnection, parsedMetrics[ReuseBytesperConnection], MetricUnit.BytesPerConnection, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(ReuseConnThroughput, reuseConnThroughput, MetricUnit.BytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(ReuseConnPerSec, reuseConnPerSec, MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric(ReuseTotalBytesRead, parsedMetrics[ReuseTotalBytesRead], MetricUnit.Bytes, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(ReuseNoOfConnections, parsedMetrics[ReuseNoOfConnections], MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(ReuseDuration, parsedMetrics[ReuseDuration], MetricUnit.Seconds, MetricRelativity.Undefined, verbosity: 3)); + metrics.Add(new Metric(ReuseBytesperConnection, parsedMetrics[ReuseBytesperConnection], MetricUnit.BytesPerConnection, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(ReuseConnThroughput, reuseConnThroughput, MetricUnit.BytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(ReuseConnPerSec, reuseConnPerSec, MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 1)); return metrics; } diff --git a/src/VirtualClient/VirtualClient.Actions/Redis/RedisBenchmarkMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Redis/RedisBenchmarkMetricsParser.cs index 5c7904bd3e..52d55a75f8 100644 --- a/src/VirtualClient/VirtualClient.Actions/Redis/RedisBenchmarkMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Redis/RedisBenchmarkMetricsParser.cs @@ -108,7 +108,7 @@ private void AddMetricsFromCsv(List metrics) reqPerSec, MetricUnit.RequestsPerSec, relativity: MetricRelativity.HigherIsBetter, - verbosity: 0, + verbosity: 1, description: "Total number of requests/operations per second during the period of time.")); } @@ -159,7 +159,7 @@ private void AddMetricsFromCsv(List metrics) p99Latency, MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, - verbosity: 0, + verbosity: 1, description: "The latency for 99% of all requests was at or under this value.")); } diff --git a/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs index 9d671963f6..bec610a828 100644 --- a/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs @@ -49,7 +49,7 @@ public override IList Parse() } else { - this.metrics.Add(new Metric(name.Trim(), Convert.ToDouble(value), SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); + this.metrics.Add(new Metric(name.Trim(), Convert.ToDouble(value), SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); } } diff --git a/src/VirtualClient/VirtualClient.Actions/SPECjvm/SpecJvmMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/SPECjvm/SpecJvmMetricsParser.cs index f546bfaffd..4b0401fcf7 100644 --- a/src/VirtualClient/VirtualClient.Actions/SPECjvm/SpecJvmMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/SPECjvm/SpecJvmMetricsParser.cs @@ -46,7 +46,7 @@ public override IList Parse() string[] nameAndValue = Regex.Split(line, columnRegex.ToString(), columnRegex.Options); string metricName = nameAndValue[0].Trim(); double metricValue = Convert.ToDouble(nameAndValue[1].Replace(SpecJvmMetricsParser.operationPerSecond, string.Empty)); - this.metrics.Add(new Metric(metricName, metricValue, SpecJvmMetricsParser.operationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); + this.metrics.Add(new Metric(metricName, metricValue, SpecJvmMetricsParser.operationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); } else { @@ -55,7 +55,7 @@ public override IList Parse() string metricName = nameAndValue[0].Trim(); string[] a = line.Split(" "); double metricValue = Convert.ToDouble(nameAndValue[1]); - this.metrics.Add(new Metric(metricName, metricValue, SpecJvmMetricsParser.operationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); + this.metrics.Add(new Metric(metricName, metricValue, SpecJvmMetricsParser.operationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); } } diff --git a/src/VirtualClient/VirtualClient.Actions/StressNg/StressNgMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/StressNg/StressNgMetricsParser.cs index 8f8a008a2f..6e04c1df51 100644 --- a/src/VirtualClient/VirtualClient.Actions/StressNg/StressNgMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/StressNg/StressNgMetricsParser.cs @@ -42,12 +42,12 @@ public override IList Parse() foreach (StressNgStressorResult stressor in parsedResult.Metrics) { - this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops", stressor.BogoOps, "BogoOps", MetricRelativity.HigherIsBetter, verbosity: 0)); - this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops-per-second-usr-sys-time", stressor.BogoOpsPerSecondUsrSysTime, BogusOperationsPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); - this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops-per-second-real-time", stressor.BogoOpsPerSecondRealTime, BogusOperationsPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); - this.metrics.Add(new Metric($"{stressor.Stressor}-wall-clock-time", stressor.WallClockTime, "second", MetricRelativity.LowerIsBetter, verbosity: 2)); - this.metrics.Add(new Metric($"{stressor.Stressor}-user-time", stressor.UserTime, "second", MetricRelativity.LowerIsBetter, verbosity: 2)); - this.metrics.Add(new Metric($"{stressor.Stressor}-system-time", stressor.SystemTime, "second", MetricRelativity.LowerIsBetter, verbosity: 2)); + this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops", stressor.BogoOps, "BogoOps", MetricRelativity.HigherIsBetter, verbosity: 1)); + this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops-per-second-usr-sys-time", stressor.BogoOpsPerSecondUsrSysTime, BogusOperationsPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); + this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops-per-second-real-time", stressor.BogoOpsPerSecondRealTime, BogusOperationsPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); + this.metrics.Add(new Metric($"{stressor.Stressor}-wall-clock-time", stressor.WallClockTime, "second", MetricRelativity.LowerIsBetter, verbosity: 5)); + this.metrics.Add(new Metric($"{stressor.Stressor}-user-time", stressor.UserTime, "second", MetricRelativity.LowerIsBetter, verbosity: 5)); + this.metrics.Add(new Metric($"{stressor.Stressor}-system-time", stressor.SystemTime, "second", MetricRelativity.LowerIsBetter, verbosity: 5)); } return this.metrics; diff --git a/src/VirtualClient/VirtualClient.Contracts/Extensibility/MetricDataPoint.cs b/src/VirtualClient/VirtualClient.Contracts/Extensibility/MetricDataPoint.cs index c9b2c2475d..2ef9547e7d 100644 --- a/src/VirtualClient/VirtualClient.Contracts/Extensibility/MetricDataPoint.cs +++ b/src/VirtualClient/VirtualClient.Contracts/Extensibility/MetricDataPoint.cs @@ -77,12 +77,16 @@ public MetricDataPoint(TelemetryDataPoint dataPoint) public double? MetricValue; /// - /// The priority/verbosity of the metric. Recommended Values = 0 (Critical), 1 (Standard), 2 (Informational) etc.. + /// The priority/verbosity of the metric. Recommended Values: + /// - 1 (Standard/Critical): Most important metrics + /// - 3 (Detailed): Additional detailed metrics + /// - 5 (Verbose): All diagnostic/internal metrics /// /// /// Allows the user to ascribe different levels of priority/verbosity to a set of metrics that can /// be used for queries/filtering. Lower values indicate higher priority. For example, metrics considered - /// to be the most critical for decision making would be set with verbosity = 0 (Critical). + /// to be the most critical for decision making would be set with verbosity = 1 (Critical). + /// For backward compatibility, verbosity = 0 is mapped to verbosity = 1. /// [JsonProperty("metricVerbosity", Required = Required.Default)] [YamlMember(Alias = "metricVerbosity", ScalarStyle = ScalarStyle.Plain)] From 6b36979af20d899af953567fb8c09bedb2526792 Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Wed, 18 Feb 2026 20:38:32 -0800 Subject: [PATCH 3/8] Update 1 --- .../CoreMark/CoreMarkMetricsParser.cs | 12 ++++++------ .../GeekBench/GeekBenchMetricsParser.cs | 2 +- .../SPECcpu/SpecCpuMetricsParser.cs | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions/CoreMark/CoreMarkMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/CoreMark/CoreMarkMetricsParser.cs index 6ad353945d..cdc9107e5b 100644 --- a/src/VirtualClient/VirtualClient.Actions/CoreMark/CoreMarkMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/CoreMark/CoreMarkMetricsParser.cs @@ -46,23 +46,23 @@ public override IList Parse() metrics.AddRange(this.CoreMarkResult.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "NA", namePrefix: string.Empty, ignoreFormatError: true)); // CoreMark result doesn't define the unit so needs manually assign units. metrics.Where(m => m.Name == "CoreMark Size").FirstOrDefault().Unit = "bytes"; - metrics.Where(m => m.Name == "CoreMark Size").FirstOrDefault().Verbosity = 2; + metrics.Where(m => m.Name == "CoreMark Size").FirstOrDefault().Verbosity = 5; metrics.Where(m => m.Name == "Total ticks").FirstOrDefault().Unit = "ticks"; - metrics.Where(m => m.Name == "Total ticks").FirstOrDefault().Verbosity = 2; + metrics.Where(m => m.Name == "Total ticks").FirstOrDefault().Verbosity = 5; metrics.Where(m => m.Name == "Total time (secs)").FirstOrDefault().Unit = "secs"; - metrics.Where(m => m.Name == "Total time (secs)").FirstOrDefault().Verbosity = 2; + metrics.Where(m => m.Name == "Total time (secs)").FirstOrDefault().Verbosity = 5; metrics.Where(m => m.Name == "Iterations/Sec").FirstOrDefault().Unit = "iterations/sec"; metrics.Where(m => m.Name == "Iterations/Sec").FirstOrDefault().Relativity = MetricRelativity.HigherIsBetter; - metrics.Where(m => m.Name == "Iterations/Sec").FirstOrDefault().Verbosity = 0; + metrics.Where(m => m.Name == "Iterations/Sec").FirstOrDefault().Verbosity = 1; metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Unit = "iterations"; metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Relativity = MetricRelativity.Undefined; - metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Verbosity = 2; + metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Verbosity = 5; // This line won't be there if it's running single thread. if (metrics.Any(m => m.Name == "Parallel PThreads")) { metrics.Where(m => m.Name == "Parallel PThreads").FirstOrDefault().Unit = "threads"; - metrics.Where(m => m.Name == "Parallel PThreads").FirstOrDefault().Verbosity = 2; + metrics.Where(m => m.Name == "Parallel PThreads").FirstOrDefault().Verbosity = 5; } return metrics; diff --git a/src/VirtualClient/VirtualClient.Actions/GeekBench/GeekBenchMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/GeekBench/GeekBenchMetricsParser.cs index 740cb33262..5a8dcff803 100644 --- a/src/VirtualClient/VirtualClient.Actions/GeekBench/GeekBenchMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/GeekBench/GeekBenchMetricsParser.cs @@ -79,7 +79,7 @@ public override IList Parse() metrics.AddRange(this.SingleCoreSummary.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "Score", namePrefix: "SingleCoreSummary-", metricRelativity: MetricRelativity.HigherIsBetter)); metrics.AddRange(this.MultiCoreSummary.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "Score", namePrefix: "MultiCoreSummary-", metricRelativity: MetricRelativity.HigherIsBetter)); - metrics.ForEach(metric => metric.Verbosity = 0); + metrics.ForEach(metric => metric.Verbosity = 1); return metrics; } diff --git a/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuMetricsParser.cs index bb045d4ca8..fe75f85113 100644 --- a/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuMetricsParser.cs @@ -68,7 +68,7 @@ public override IList Parse() this.metrics.AddRange(this.SpecCpuSummary.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "Score", namePrefix: string.Empty, ignoreFormatError: true, metricRelativity: MetricRelativity.HigherIsBetter)); // Every score in SPECcpu is critical metric. - this.metrics.ForEach(m => m.Verbosity = 0); + this.metrics.ForEach(m => m.Verbosity = 1); return this.metrics; } From d067aeb0fe0600f107ef374511d37036fade76aa Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Wed, 18 Feb 2026 21:41:36 -0800 Subject: [PATCH 4/8] Commit changes --- .../3DMark/ThreeDMarkMetricsParserTests.cs | 97 +++++++++++-------- .../7zip/Compression7zipMetricsParser.cs | 6 +- .../ASPNET/BombardierMetricsParser.cs | 6 +- .../DiskSpd/DiskSpdMetricsParser.cs | 2 +- .../FIO/FioMetricsParser.cs | 12 +-- .../Memtier/MemtierMetricsParser.cs | 8 +- .../CPS/CPSMetricsParser.cs | 14 +-- .../NTttcp/NTttcpMetricsParser.cs | 8 +- .../SockPerf/SockPerfMetricsParser.cs | 10 +- .../OpenSSL/OpenSslMetricsParser.cs | 6 +- .../OpenSSL/TlsOpenSslMetricsParser.cs | 8 +- .../MetricExtensionsTests.cs | 63 ++++++++---- .../Extensions/MetricExtensions.cs | 22 +++-- .../VirtualClient.Contracts/Metric.cs | 10 +- 14 files changed, 158 insertions(+), 114 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/3DMark/ThreeDMarkMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/3DMark/ThreeDMarkMetricsParserTests.cs index c1df1a2e73..e428e6bdcc 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/3DMark/ThreeDMarkMetricsParserTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/3DMark/ThreeDMarkMetricsParserTests.cs @@ -3,62 +3,75 @@ namespace VirtualClient.Actions { + using System; using System.Collections.Generic; - using System.Diagnostics; + using System.ComponentModel; using System.IO; using System.Linq; - using System.Reflection; + using System.Reflection.Metadata; using System.Text; - using System.Threading.Tasks; - using NUnit.Framework; - using VirtualClient.Common; + using System.Text.RegularExpressions; + using System.Xml.Linq; + using System.Xml.Serialization; using VirtualClient.Contracts; - [TestFixture] - [Category("Unit")] - public class ThreeDMarkMetricsParserTests + /// + /// Parser for the NTttcp workload. + /// + public class ThreeDMarkMetricsParser : MetricsParser { - [Test] - public void ThreeDMarkMetricsParserTestsCorrectly_ScenarioTSGT1() + /// + /// Parser for the 3DMark workload + /// + /// The raw text from the 3DMark export process. + /// The 3dmark definition. + public ThreeDMarkMetricsParser(string rawText, string definition) + : base(rawText) { - string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - string outputPath = Path.Combine(workingDirectory, "Examples", "3DMark", "result_tsgt1.xml"); - string rawText = File.ReadAllText(outputPath); - - ThreeDMarkMetricsParser testParser = new ThreeDMarkMetricsParser(rawText, "custom_TSGT1.3dmdef"); - IList metrics = testParser.Parse(); - - Assert.AreEqual(1, metrics.Count); - MetricAssert.Exists(metrics, "timespy.graphics.1 [fps]", 59.65, "fps"); + this.Defintion = definition; } - [Test] - public void ThreeDMarkMetricsParserTestsCorrectly_ScenarioTSGT2() - { - string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - string outputPath = Path.Combine(workingDirectory, "Examples", "3DMark", "result_tsgt2.xml"); - string rawText = File.ReadAllText(outputPath); + private string Defintion { get; set; } - ThreeDMarkMetricsParser testParser = new ThreeDMarkMetricsParser(rawText, "custom_TSGT2.3dmdef"); - IList metrics = testParser.Parse(); + /// + public override IList Parse() + { + IList metrics = new List(); + try + { + if (this.Defintion == "custom_TSGT1.3dmdef") + { + metrics.Add(new Metric("timespy.graphics.1 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest1"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); + } + else if (this.Defintion == "custom_TSGT2.3dmdef") + { + metrics.Add(new Metric("timespy.graphics.2 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); + } + else if (this.Defintion == "custom_TSCT.3dmdef") + { + metrics.Add(new Metric("timespy.cpu [fps]", this.ParseXMLTag("TimeSpyPerformanceCpuSection2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); + } + } + catch (Exception exc) + { + throw new WorkloadException($"Results not found. The workload '3DMark' did not produce any valid results.", exc, ErrorReason.WorkloadFailed); + } - Assert.AreEqual(1, metrics.Count); - MetricAssert.Exists(metrics, "timespy.graphics.2 [fps]", 58.10, "fps"); + return metrics; } - [Test] - public void ThreeDMarkMetricsParserTestsCorrectly_ScenarioTSCT() + /// + /// Gets the value sandwiched between the given XML tags + /// + /// + /// + private double ParseXMLTag(string tagName) { - string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - string outputPath = Path.Combine(workingDirectory, "Examples", "3DMark", "result_tsct.xml"); - string rawText = File.ReadAllText(outputPath); - - ThreeDMarkMetricsParser testParser = new ThreeDMarkMetricsParser(rawText, "custom_TSCT.3dmdef"); - IList metrics = testParser.Parse(); - - Assert.AreEqual(1, metrics.Count); - MetricAssert.Exists(metrics, "timespy.cpu [fps]", 28.50, "fps"); + string pattern = $"<{tagName}.*>((.|\\n)*?)<\\/{tagName}>"; + Match m = Regex.Match(this.RawText, pattern); + XElement tag = XElement.Parse(m.Value); + double val = double.Parse(tag.Value); + return val; } - } -} \ No newline at end of file +} diff --git a/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs index 19d4934a59..519b03f2d5 100644 --- a/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs @@ -62,10 +62,10 @@ public override IList Parse() int rows = this.SizeAndTime.Rows.Count; metrics.Add(new Metric( - "Compression_Ratio", - (Convert.ToDouble(this.SizeAndTime.Rows[2].ItemArray[2]) / Convert.ToDouble(this.SizeAndTime.Rows[0].ItemArray[2])) * 100, + "Compression_Ratio", + (Convert.ToDouble(this.SizeAndTime.Rows[2].ItemArray[2]) / Convert.ToDouble(this.SizeAndTime.Rows[0].ItemArray[2])) * 100, "precentage", - relativity: MetricRelativity.LowerIsBetter, + relativity: MetricRelativity.LowerIsBetter, verbosity: 5)); double compressionTime = 0; diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs index 6adbbc58a7..0e99a9e05c 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs @@ -36,9 +36,9 @@ public override IList Parse() this.metrics.Add(new Metric("Latency Average", root.Result.Latency.Mean, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); this.metrics.Add(new Metric("Latency Stddev", root.Result.Latency.Stddev, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); this.metrics.Add(new Metric("Latency P50", root.Result.Latency.Percentiles.P50, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); - this.metrics.Add(new Metric("Latency P75", root.Result.Latency.Percentiles.P75, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); - this.metrics.Add(new Metric("Latency P90", root.Result.Latency.Percentiles.P90, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); - this.metrics.Add(new Metric("Latency P95", root.Result.Latency.Percentiles.P95, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + this.metrics.Add(new Metric("Latency P75", root.Result.Latency.Percentiles.P75, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + this.metrics.Add(new Metric("Latency P90", root.Result.Latency.Percentiles.P90, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + this.metrics.Add(new Metric("Latency P95", root.Result.Latency.Percentiles.P95, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); this.metrics.Add(new Metric("Latency P99", root.Result.Latency.Percentiles.P99, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); this.metrics.Add(new Metric("RequestPerSecond Max", root.Result.Rps.Max, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 5)); diff --git a/src/VirtualClient/VirtualClient.Actions/DiskSpd/DiskSpdMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/DiskSpd/DiskSpdMetricsParser.cs index b3ee8c0913..9368d28571 100644 --- a/src/VirtualClient/VirtualClient.Actions/DiskSpd/DiskSpdMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/DiskSpd/DiskSpdMetricsParser.cs @@ -239,7 +239,7 @@ private void ParseWriteIoResult() metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "MiB/s", namePrefix: $"write {writeIo.Columns[3].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 1)); metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 4, unit: "iops", namePrefix: $"write {writeIo.Columns[4].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 1)); metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 5, unit: "ms", namePrefix: $"write {writeIo.Columns[5].ColumnName} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 1)); - + foreach (var metric in metrics.Where(m => m.Name.EndsWith("total") && (m.Unit == "iops" || m.Unit == "ms" || m.Unit == "MiB/s"))) { metric.Verbosity = 1; diff --git a/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs index 308d694600..07946c2189 100644 --- a/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs @@ -200,9 +200,9 @@ private void AddReadMeasurements(IList metrics, JToken resultsJson) this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['99.000000']", "read_completionlatency_p99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); // Level 3 - Detailed: Other percentiles - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['70.000000']", "read_completionlatency_p70", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 3); - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['90.000000']", "read_completionlatency_p90", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 3); - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['99.990000']", "read_completionlatency_p99_99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 3); + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['70.000000']", "read_completionlatency_p70", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['90.000000']", "read_completionlatency_p90", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['99.990000']", "read_completionlatency_p99_99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); // Level 1 - Standard submission latency metrics this.AddMeasurement(metrics, job, $"read.slat_ns.min", "read_submissionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); @@ -263,9 +263,9 @@ private void AddWriteMeasurements(IList metrics, JToken resultsJson) this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['99.000000']", "write_completionlatency_p99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); // Level 3 - Detailed: Other percentiles - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['70.000000']", "write_completionlatency_p70", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 3); - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['90.000000']", "write_completionlatency_p90", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 3); - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['99.990000']", "write_completionlatency_p99_99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 3); + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['70.000000']", "write_completionlatency_p70", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['90.000000']", "write_completionlatency_p90", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['99.990000']", "write_completionlatency_p99_99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); // Level 1 - Standard submission latency metrics this.AddMeasurement(metrics, job, $"write.slat_ns.min", "write_submissionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); diff --git a/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs index a831a23155..ae3d1980de 100644 --- a/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs @@ -25,7 +25,7 @@ public class MemtierMetricsParser : MetricsParser // Sets 4827.17 --- --- 2.64323 2.83100 3.93500 4.47900 7.45500 29.56700 329.45 // Waits 0.00 --- --- --- --- --- --- --- --- --- // Totals 48271.29 43444.12 0.00 2.62213 2.75100 3.90300 4.41500 7.42300 29.31100 4053.46 - + private static readonly Regex BandwidthExpression = new Regex(@"Bandwidth|Throughput", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex GetsExpression = new Regex(@"(?<=Gets).*(?=\n)", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex GetLatencyP80Expression = new Regex(@"(?<=GET)\s+([0-9\.]+)\s+.*80[\.0]+", RegexOptions.Compiled | RegexOptions.IgnoreCase); @@ -364,7 +364,7 @@ private static IDictionary CreateMetricsBin(string metr metricNamePrefix == null ? "Latency-P90" : $"{metricNamePrefix}-Latency-P90", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, - verbosity: 3, + verbosity: 2, description: "The latency for 90% of all requests was at or under this value.") }, // Level 3 - Detailed: P95 latency @@ -374,7 +374,7 @@ private static IDictionary CreateMetricsBin(string metr metricNamePrefix == null ? "Latency-P95" : $"{metricNamePrefix}-Latency-P95", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, - verbosity: 3, + verbosity: 2, description: "The latency for 95% of all requests was at or under this value.") }, // Level 1 - Critical: P99 latency @@ -394,7 +394,7 @@ private static IDictionary CreateMetricsBin(string metr metricNamePrefix == null ? "Latency-P99.9" : $"{metricNamePrefix}-Latency-P99.9", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, - verbosity: 3, + verbosity: 2, description: "The latency for 99.9% of all requests was at or under this value.") }, // Level 1 - Critical: Bandwidth diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs index 90d4f5031e..5cc60cd807 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs @@ -82,16 +82,16 @@ private static void AddStatisticalMetrics(IList metrics, List ti double lowerCI = mean - t; double upperCI = mean + t; - metrics.Add(new Metric("ConnectsPerSec_Min", connectsPerSec.Min(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 3)); + metrics.Add(new Metric("ConnectsPerSec_Min", connectsPerSec.Min(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); metrics.Add(new Metric("ConnectsPerSec_Max", connectsPerSec.Max(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 5)); - metrics.Add(new Metric("ConnectsPerSec_Med", connectsPerSec.Median(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 3)); + metrics.Add(new Metric("ConnectsPerSec_Med", connectsPerSec.Median(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); metrics.Add(new Metric("ConnectsPerSec_Avg", connectsPerSec.Average(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 1)); - metrics.Add(new Metric("ConnectsPerSec_P25", connectsPerSec.Percentile(25), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 3)); + metrics.Add(new Metric("ConnectsPerSec_P25", connectsPerSec.Percentile(25), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); metrics.Add(new Metric("ConnectsPerSec_P50", connectsPerSec.Percentile(50), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 1)); - metrics.Add(new Metric("ConnectsPerSec_P75", connectsPerSec.Percentile(75), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 3)); - metrics.Add(new Metric("ConnectsPerSec_P90", connectsPerSec.Percentile(90), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 3)); + metrics.Add(new Metric("ConnectsPerSec_P75", connectsPerSec.Percentile(75), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_P90", connectsPerSec.Percentile(90), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); metrics.Add(new Metric("ConnectsPerSec_P99", connectsPerSec.Percentile(99), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 1)); - metrics.Add(new Metric("ConnectsPerSec_P99_9", Statistics.QuantileCustom(connectsPerSec, 1d - 0.001d, QuantileDefinition.R3), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 3)); + metrics.Add(new Metric("ConnectsPerSec_P99_9", Statistics.QuantileCustom(connectsPerSec, 1d - 0.001d, QuantileDefinition.R3), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); metrics.Add(new Metric("ConnectsPerSec_P99_99", Statistics.QuantileCustom(connectsPerSec, 1d - 0.0001d, QuantileDefinition.R3), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 5)); metrics.Add(new Metric("ConnectsPerSec_P99_999", Statistics.QuantileCustom(connectsPerSec, 1d - 0.00001d, QuantileDefinition.R3), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 5)); double median = Statistics.Median(connectsPerSec); @@ -109,7 +109,7 @@ private static KeyValuePair, List> GetTimestampsAndConnecti { bool appendResult = true; int valueIndex = 0; - + // The timestamps and connectionsPerSecond co-relate on the index. List timestamps = new List(); List connectionsPerSec = new List(); diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs index 1894f72c42..ca9c9ccd56 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs @@ -19,7 +19,7 @@ public class NTttcpMetricsParser : MetricsParser { private readonly bool isClient; private NTttcpResult result; - + /// /// Parser for the NTttcp workload /// @@ -56,9 +56,9 @@ public override IList Parse() new Metric("AvgFrameSize", this.result.AverageFrameSize.Value, this.result.AverageFrameSize.Units, MetricRelativity.Undefined, verbosity: 5), new Metric("ThroughputMbps", throughputMetric.Value, throughputMetric.Units, MetricRelativity.HigherIsBetter, verbosity: 1), new Metric("AvgPacketsPerInterrupt", this.result.AveragePacketsPerInterrupt.Value, this.result.AveragePacketsPerInterrupt.Units, MetricRelativity.Undefined, verbosity: 5), - new Metric("InterruptsPerSec", this.result.Interrupts.Value, this.result.Interrupts.Units, MetricRelativity.Undefined, verbosity: 3), + new Metric("InterruptsPerSec", this.result.Interrupts.Value, this.result.Interrupts.Units, MetricRelativity.Undefined, verbosity: 2), new Metric("PacketsRetransmitted", this.result.PacketsRetransmitted, MetricUnit.Count, MetricRelativity.LowerIsBetter, verbosity: 5), - new Metric("Errors", this.result.Errors, MetricUnit.Count, MetricRelativity.LowerIsBetter, verbosity: 3), + new Metric("Errors", this.result.Errors, MetricUnit.Count, MetricRelativity.LowerIsBetter, verbosity: 2), }; if (this.result.TcpAverageRtt != null) @@ -133,7 +133,7 @@ public override IList Parse() /// do not use outside the context of /// public class NTttcpResult - { + { /// /// A collection of key value pairs passed into the NTttcp workload. /// diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs index f3d070b6f6..63d0d0e8a0 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs @@ -107,15 +107,15 @@ private static void AddStatisticalMetrics(IList metrics, List pa double lowerCI = mean - t; double upperCI = mean + t; - metrics.Add(new Metric("Latency-Min", packetsLatencyValues.Min(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + metrics.Add(new Metric("Latency-Min", packetsLatencyValues.Min(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); metrics.Add(new Metric("Latency-Max", packetsLatencyValues.Max(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); metrics.Add(new Metric("Latency-Avg", packetsLatencyValues.Average(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); - metrics.Add(new Metric("Latency-P25", packetsLatencyValues.Percentile(25), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + metrics.Add(new Metric("Latency-P25", packetsLatencyValues.Percentile(25), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); metrics.Add(new Metric("Latency-P50", packetsLatencyValues.Percentile(50), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); - metrics.Add(new Metric("Latency-P75", packetsLatencyValues.Percentile(75), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); - metrics.Add(new Metric("Latency-P90", packetsLatencyValues.Percentile(90), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + metrics.Add(new Metric("Latency-P75", packetsLatencyValues.Percentile(75), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("Latency-P90", packetsLatencyValues.Percentile(90), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); metrics.Add(new Metric("Latency-P99", packetsLatencyValues.Percentile(99), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); - metrics.Add(new Metric("Latency-P99.9", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + metrics.Add(new Metric("Latency-P99.9", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); metrics.Add(new Metric("Latency-P99.99", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.0001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); metrics.Add(new Metric("Latency-P99.999", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.00001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); double median = Statistics.Median(packetsLatencyValues); diff --git a/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs index e1a8e0d658..a33780bbd8 100644 --- a/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs @@ -191,7 +191,7 @@ public override IList Parse() } else { - metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 2)); } } } @@ -212,7 +212,7 @@ public override IList Parse() } else { - metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 2)); } } } @@ -233,7 +233,7 @@ public override IList Parse() } else { - metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 3)); + metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 2)); } } } diff --git a/src/VirtualClient/VirtualClient.Actions/OpenSSL/TlsOpenSslMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/OpenSSL/TlsOpenSslMetricsParser.cs index 00a0b0b824..a5c68fdcc8 100644 --- a/src/VirtualClient/VirtualClient.Actions/OpenSSL/TlsOpenSslMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/OpenSSL/TlsOpenSslMetricsParser.cs @@ -63,7 +63,7 @@ public override IList Parse() // set 1 - new connection metrics metrics.Add(new Metric(TotalBytesRead, parsedMetrics[TotalBytesRead], MetricUnit.Bytes, MetricRelativity.HigherIsBetter, verbosity: 1)); metrics.Add(new Metric(NoOfConnections, parsedMetrics[NoOfConnections], MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 1)); - metrics.Add(new Metric(Duration, parsedMetrics[Duration], MetricUnit.Seconds, MetricRelativity.Undefined, verbosity: 3)); + metrics.Add(new Metric(Duration, parsedMetrics[Duration], MetricUnit.Seconds, MetricRelativity.Undefined, verbosity: 2)); metrics.Add(new Metric(BytesperConnection, parsedMetrics[BytesperConnection], MetricUnit.BytesPerConnection, MetricRelativity.HigherIsBetter, verbosity: 1)); metrics.Add(new Metric(NewConnThroughput, newConnThroughput, MetricUnit.BytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); metrics.Add(new Metric(NewConnPersec, newConnPerSec, MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 1)); @@ -71,7 +71,7 @@ public override IList Parse() // set 2 - reuse connection metrics metrics.Add(new Metric(ReuseTotalBytesRead, parsedMetrics[ReuseTotalBytesRead], MetricUnit.Bytes, MetricRelativity.HigherIsBetter, verbosity: 1)); metrics.Add(new Metric(ReuseNoOfConnections, parsedMetrics[ReuseNoOfConnections], MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 1)); - metrics.Add(new Metric(ReuseDuration, parsedMetrics[ReuseDuration], MetricUnit.Seconds, MetricRelativity.Undefined, verbosity: 3)); + metrics.Add(new Metric(ReuseDuration, parsedMetrics[ReuseDuration], MetricUnit.Seconds, MetricRelativity.Undefined, verbosity: 2)); metrics.Add(new Metric(ReuseBytesperConnection, parsedMetrics[ReuseBytesperConnection], MetricUnit.BytesPerConnection, MetricRelativity.HigherIsBetter, verbosity: 1)); metrics.Add(new Metric(ReuseConnThroughput, reuseConnThroughput, MetricUnit.BytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); metrics.Add(new Metric(ReuseConnPerSec, reuseConnPerSec, MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 1)); @@ -101,10 +101,10 @@ Now timing with session id reuse. 1696 connections in 0.86s; 1972.09 connections/user sec, bytes read 547808 1696 connections in 6 real seconds, 323 bytes read per connection */ - + // ignore the first connections and connections per user metric as it is conflicting with // total connections in n seconds - + // Initial run var match1 = Regex.Match(input, @"(\d+) connections in ([\d.]+)s; ([\d.]+) connections/user sec, bytes read (\d+)"); if (match1.Success) diff --git a/src/VirtualClient/VirtualClient.Contracts.UnitTests/MetricExtensionsTests.cs b/src/VirtualClient/VirtualClient.Contracts.UnitTests/MetricExtensionsTests.cs index 835577f5b9..fdea920376 100644 --- a/src/VirtualClient/VirtualClient.Contracts.UnitTests/MetricExtensionsTests.cs +++ b/src/VirtualClient/VirtualClient.Contracts.UnitTests/MetricExtensionsTests.cs @@ -40,8 +40,8 @@ public void SetupTest() new Metric("verbose_test_1", 123, "unit", MetricRelativity.HigherIsBetter, verbosity: 1), new Metric("verbose_test_2", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 1), - new Metric("verbose_test_3", 123, "unit", MetricRelativity.HigherIsBetter, verbosity: 3), - new Metric("verbose_test_4", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 3), + new Metric("verbose_test_3", 123, "unit", MetricRelativity.HigherIsBetter, verbosity: 2), + new Metric("verbose_test_4", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 2), new Metric("verbose_test_5", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 5), }; } @@ -71,8 +71,8 @@ public void MetricFiltersCorrectFiltersVerbosity() IEnumerable filter = new List { "Verbosity:1" }; CollectionAssert.AreEquivalent(this.metrics.Where(m => m.Verbosity <= 1).Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); - filter = new List { "Verbosity:3" }; - CollectionAssert.AreEquivalent(this.metrics.Where(m => m.Verbosity <= 3).Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); + filter = new List { "Verbosity:2" }; + CollectionAssert.AreEquivalent(this.metrics.Where(m => m.Verbosity <= 2).Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); filter = new List { "Verbosity:5" }; CollectionAssert.AreEquivalent(this.metrics.Where(m => m.Verbosity <= 5).Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); @@ -186,13 +186,13 @@ public void FilterByExtensionReturnsTheExpectedFilteredWithBothVerbosityAndText( [Test] public void FilterByExtensionSupportsAllVerbosityLevels() { - // Setup metrics with different verbosity levels (1, 3, 5 only) + // Setup metrics with different verbosity levels (1, 2, 5 only) var metrics = new List { new Metric("critical_metric_1a", 1, "unit", MetricRelativity.HigherIsBetter, verbosity: 1), new Metric("critical_metric_1b", 2, "unit", MetricRelativity.HigherIsBetter, verbosity: 1), - new Metric("detailed_metric_3a", 3, "unit", MetricRelativity.HigherIsBetter, verbosity: 3), - new Metric("detailed_metric_3b", 4, "unit", MetricRelativity.HigherIsBetter, verbosity: 3), + new Metric("detailed_metric_2a", 3, "unit", MetricRelativity.HigherIsBetter, verbosity: 2), + new Metric("detailed_metric_2b", 4, "unit", MetricRelativity.HigherIsBetter, verbosity: 2), new Metric("verbose_metric_5a", 5, "unit", MetricRelativity.HigherIsBetter, verbosity: 5), new Metric("verbose_metric_5b", 6, "unit", MetricRelativity.HigherIsBetter, verbosity: 5) }; @@ -203,11 +203,11 @@ public void FilterByExtensionSupportsAllVerbosityLevels() Assert.AreEqual(2, result1.Count()); Assert.IsTrue(result1.All(m => m.Verbosity == 1)); - // Test verbosity:3 - should return level 1 + level 3 metrics - var filter3 = new List { "verbosity:3" }; - var result3 = metrics.FilterBy(filter3); - Assert.AreEqual(4, result3.Count()); - Assert.IsTrue(result3.All(m => m.Verbosity <= 3)); + // Test verbosity:2 - should return level 1 + level 2 metrics + var filter2 = new List { "verbosity:2" }; + var result2 = metrics.FilterBy(filter2); + Assert.AreEqual(4, result2.Count()); + Assert.IsTrue(result2.All(m => m.Verbosity <= 2)); // Test verbosity:5 - should return all metrics var filter5 = new List { "verbosity:5" }; @@ -233,12 +233,33 @@ public void FilterByExtensionHandlesInvalidVerbosityValues() var result2 = metrics.FilterBy(invalidFilter2); Assert.AreEqual(0, result2.Count()); - // Test verbosity less than 1 - var invalidFilter3 = new List { "verbosity:0" }; + // Test verbosity less than 0 (negative) + var invalidFilter3 = new List { "verbosity:-1" }; var result3 = metrics.FilterBy(invalidFilter3); Assert.AreEqual(0, result3.Count()); } + [Test] + public void FilterByExtensionHandlesBackwardCompatibilityForVerbosityZero() + { + var metrics = new List + { + new Metric("critical_metric", 1, "unit", MetricRelativity.HigherIsBetter, verbosity: 0), + new Metric("standard_metric", 2, "unit", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("detailed_metric", 3, "unit", MetricRelativity.HigherIsBetter, verbosity: 2) + }; + + // verbosity:0 should be mapped to verbosity:1 for backward compatibility + var filter = new List { "verbosity:0" }; + var result = metrics.FilterBy(filter); + + // Should include metrics with verbosity 0 and 1 (since 0 maps to 1) + Assert.AreEqual(2, result.Count()); + Assert.IsTrue(result.Any(m => m.Name == "critical_metric")); + Assert.IsTrue(result.Any(m => m.Name == "standard_metric")); + Assert.IsFalse(result.Any(m => m.Name == "detailed_metric")); + } + [Test] public void FilterByExtensionSupportsExclusionFilters() { @@ -268,7 +289,7 @@ public void FilterByExtensionComposesVerbosityAndNameFilters() new Metric("bandwidth_write", 2, "MB/s", MetricRelativity.HigherIsBetter, verbosity: 1), new Metric("iops_read", 3, "ops/s", MetricRelativity.HigherIsBetter, verbosity: 1), new Metric("iops_write", 4, "ops/s", MetricRelativity.HigherIsBetter, verbosity: 1), - new Metric("latency_p99", 5, "ms", MetricRelativity.LowerIsBetter, verbosity: 3) + new Metric("latency_p99", 5, "ms", MetricRelativity.LowerIsBetter, verbosity: 5) }; // Filter: verbosity <= 1 AND name contains "bandwidth" @@ -283,18 +304,18 @@ public void FilterByExtensionSupportsComplexFilterCombinations() { var metrics = new List { - new Metric("h000_latency", 1, "ms", MetricRelativity.LowerIsBetter, verbosity: 3), - new Metric("h001_latency", 2, "ms", MetricRelativity.LowerIsBetter, verbosity: 3), + new Metric("h000_latency", 1, "ms", MetricRelativity.LowerIsBetter, verbosity: 5), + new Metric("h001_latency", 2, "ms", MetricRelativity.LowerIsBetter, verbosity: 5), new Metric("bandwidth_read", 3, "MB/s", MetricRelativity.HigherIsBetter, verbosity: 1), new Metric("bandwidth_write", 4, "MB/s", MetricRelativity.HigherIsBetter, verbosity: 1), new Metric("iops", 5, "ops/s", MetricRelativity.HigherIsBetter, verbosity: 1) }; - // Complex filter: verbosity <= 3, exclude h00* metrics, include only bandwidth or iops - var complexFilter = new List { "verbosity:3", "-h00*", "bandwidth|iops" }; + // Complex filter: verbosity <= 5, exclude h00* metrics, include only bandwidth or iops + var complexFilter = new List { "verbosity:5", "-h00*", "bandwidth|iops" }; var result = metrics.FilterBy(complexFilter); - - Assert.IsTrue(result.All(m => m.Verbosity <= 3)); + + Assert.IsTrue(result.All(m => m.Verbosity <= 5)); Assert.IsFalse(result.Any(m => m.Name.StartsWith("h00"))); Assert.IsTrue(result.All(m => m.Name.Contains("bandwidth") || m.Name.Contains("iops"))); } diff --git a/src/VirtualClient/VirtualClient.Contracts/Extensions/MetricExtensions.cs b/src/VirtualClient/VirtualClient.Contracts/Extensions/MetricExtensions.cs index d7f60f2bc9..fe9a23cb67 100644 --- a/src/VirtualClient/VirtualClient.Contracts/Extensions/MetricExtensions.cs +++ b/src/VirtualClient/VirtualClient.Contracts/Extensions/MetricExtensions.cs @@ -47,13 +47,16 @@ public static void AddMetadata(this IEnumerable metrics, IDictionary /// - /// Verbosity levels (only 1, 3, and 5 are currently used; 2 and 4 are reserved): + /// Verbosity levels define a convention for organizing metrics by importance: /// - 1 (Standard/Critical): Most important metrics - bandwidth, throughput, IOPS, key latency percentiles (p50, p99) - /// - 2 (Reserved): Reserved for future expansion - /// - 3 (Detailed): Additional detailed metrics - supplementary percentiles (p70, p90, p95, p99.9) + /// - 2 (Detailed): Additional detailed metrics - supplementary percentiles (p70, p90, p95, p99.9) + /// - 3 (Reserved): Reserved for future expansion /// - 4 (Reserved): Reserved for future expansion /// - 5 (Verbose): All diagnostic/internal metrics - histogram buckets, standard deviations, byte counts, I/O counts /// + /// Currently, only levels 1, 2, and 5 are actively used. Levels 3 and 4 are reserved for future use. + /// + /// For backward compatibility, verbosity:0 is mapped to verbosity:1. /// Filters are composable - verbosity filtering is applied first, then name-based filtering. /// public static IEnumerable FilterBy(this IEnumerable metrics, IEnumerable filterTerms) @@ -69,9 +72,15 @@ public static IEnumerable FilterBy(this IEnumerable metrics, IEn if (!string.IsNullOrEmpty(verbosityFilter)) { // Extract the verbosity level from the filter (e.g., "verbosity:3" -> 3) - string[] parts = verbosityFilter.Split(':'); - if (parts.Length == 2 && int.TryParse(parts[1].Trim(), out int maxVerbosity) && maxVerbosity >= 1 && maxVerbosity <= 5) + string[] parts = verbosityFilter.Split(':'); + if (parts.Length == 2 && int.TryParse(parts[1].Trim(), out int maxVerbosity) && maxVerbosity >= 0 && maxVerbosity <= 5) { + // Backward compatibility: Map old level 0 to new level 1 + if (maxVerbosity == 0) + { + maxVerbosity = 1; + } + // Filter metrics to include only those with verbosity <= maxVerbosity filteredMetrics = filteredMetrics.Where(m => m.Verbosity <= maxVerbosity); } @@ -124,7 +133,8 @@ public static void LogConsole(this IEnumerable metrics, string scenario, table.Columns.Add("Value", typeof(double)); table.Columns.Add("Unit", typeof(string)); - IEnumerable metricsToPrint = criticalOnly ? metrics.Where(m => m.Verbosity == 1).ToList() : metrics; + // Support both old (0) and new (1) critical levels for backward compatibility + IEnumerable metricsToPrint = criticalOnly ? metrics.Where(m => m.Verbosity == 0 || m.Verbosity == 1).ToList() : metrics; foreach (Metric metric in metricsToPrint) { diff --git a/src/VirtualClient/VirtualClient.Contracts/Metric.cs b/src/VirtualClient/VirtualClient.Contracts/Metric.cs index aebdd9ce14..2bd7bee971 100644 --- a/src/VirtualClient/VirtualClient.Contracts/Metric.cs +++ b/src/VirtualClient/VirtualClient.Contracts/Metric.cs @@ -26,7 +26,7 @@ public Metric(string name, double value) this.Name = name; this.Value = value; this.Relativity = MetricRelativity.Undefined; - this.Metadata = new Dictionary(); + this.Metadata = new Dictionary(); this.Tags = new List(); } @@ -142,14 +142,14 @@ public Metric(string name, double value, string unit, MetricRelativity relativit /// /// Metric verbosity to describe importance/priority of the metric. /// - /// Verbosity Levels (only 1, 3, and 5 are currently used): + /// Verbosity levels define a convention for organizing metrics by importance: /// - 1 (Standard/Critical): Most important metrics for decision making - bandwidth, throughput, IOPS, key latency percentiles (p50, p99) - /// - 2 (Reserved): Reserved for future expansion - /// - 3 (Detailed): Additional detailed metrics - supplementary percentiles (p70, p90, p95, p99.9) + /// - 2 (Detailed): Additional detailed metrics - supplementary percentiles (p70, p90, p95, p99.9) + /// - 3 (Reserved): Reserved for future expansion /// - 4 (Reserved): Reserved for future expansion /// - 5 (Verbose): All diagnostic/internal metrics - histogram buckets, standard deviations, byte counts, I/O counts /// - /// Currently, only levels 1, 3, and 5 are actively used. Levels 2 and 4 are reserved for future use. + /// Currently, only levels 1, 2, and 5 are actively used. Levels 3 and 4 are reserved for future use. /// /// Default = 1 (Standard). /// From 6f3e0f2a882fad6ee58edad86a6149f9e0cbf823 Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Thu, 19 Feb 2026 20:07:08 -0800 Subject: [PATCH 5/8] Reverted changes --- .../3DMark/ThreeDMarkMetricsParserTests.cs | 97 ++++++++----------- 1 file changed, 42 insertions(+), 55 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/3DMark/ThreeDMarkMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/3DMark/ThreeDMarkMetricsParserTests.cs index e428e6bdcc..c1df1a2e73 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/3DMark/ThreeDMarkMetricsParserTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/3DMark/ThreeDMarkMetricsParserTests.cs @@ -3,75 +3,62 @@ namespace VirtualClient.Actions { - using System; using System.Collections.Generic; - using System.ComponentModel; + using System.Diagnostics; using System.IO; using System.Linq; - using System.Reflection.Metadata; + using System.Reflection; using System.Text; - using System.Text.RegularExpressions; - using System.Xml.Linq; - using System.Xml.Serialization; + using System.Threading.Tasks; + using NUnit.Framework; + using VirtualClient.Common; using VirtualClient.Contracts; - /// - /// Parser for the NTttcp workload. - /// - public class ThreeDMarkMetricsParser : MetricsParser + [TestFixture] + [Category("Unit")] + public class ThreeDMarkMetricsParserTests { - /// - /// Parser for the 3DMark workload - /// - /// The raw text from the 3DMark export process. - /// The 3dmark definition. - public ThreeDMarkMetricsParser(string rawText, string definition) - : base(rawText) + [Test] + public void ThreeDMarkMetricsParserTestsCorrectly_ScenarioTSGT1() { - this.Defintion = definition; - } + string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + string outputPath = Path.Combine(workingDirectory, "Examples", "3DMark", "result_tsgt1.xml"); + string rawText = File.ReadAllText(outputPath); + + ThreeDMarkMetricsParser testParser = new ThreeDMarkMetricsParser(rawText, "custom_TSGT1.3dmdef"); + IList metrics = testParser.Parse(); - private string Defintion { get; set; } + Assert.AreEqual(1, metrics.Count); + MetricAssert.Exists(metrics, "timespy.graphics.1 [fps]", 59.65, "fps"); + } - /// - public override IList Parse() + [Test] + public void ThreeDMarkMetricsParserTestsCorrectly_ScenarioTSGT2() { - IList metrics = new List(); - try - { - if (this.Defintion == "custom_TSGT1.3dmdef") - { - metrics.Add(new Metric("timespy.graphics.1 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest1"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); - } - else if (this.Defintion == "custom_TSGT2.3dmdef") - { - metrics.Add(new Metric("timespy.graphics.2 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); - } - else if (this.Defintion == "custom_TSCT.3dmdef") - { - metrics.Add(new Metric("timespy.cpu [fps]", this.ParseXMLTag("TimeSpyPerformanceCpuSection2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); - } - } - catch (Exception exc) - { - throw new WorkloadException($"Results not found. The workload '3DMark' did not produce any valid results.", exc, ErrorReason.WorkloadFailed); - } + string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + string outputPath = Path.Combine(workingDirectory, "Examples", "3DMark", "result_tsgt2.xml"); + string rawText = File.ReadAllText(outputPath); + + ThreeDMarkMetricsParser testParser = new ThreeDMarkMetricsParser(rawText, "custom_TSGT2.3dmdef"); + IList metrics = testParser.Parse(); - return metrics; + Assert.AreEqual(1, metrics.Count); + MetricAssert.Exists(metrics, "timespy.graphics.2 [fps]", 58.10, "fps"); } - /// - /// Gets the value sandwiched between the given XML tags - /// - /// - /// - private double ParseXMLTag(string tagName) + [Test] + public void ThreeDMarkMetricsParserTestsCorrectly_ScenarioTSCT() { - string pattern = $"<{tagName}.*>((.|\\n)*?)<\\/{tagName}>"; - Match m = Regex.Match(this.RawText, pattern); - XElement tag = XElement.Parse(m.Value); - double val = double.Parse(tag.Value); - return val; + string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + string outputPath = Path.Combine(workingDirectory, "Examples", "3DMark", "result_tsct.xml"); + string rawText = File.ReadAllText(outputPath); + + ThreeDMarkMetricsParser testParser = new ThreeDMarkMetricsParser(rawText, "custom_TSCT.3dmdef"); + IList metrics = testParser.Parse(); + + Assert.AreEqual(1, metrics.Count); + MetricAssert.Exists(metrics, "timespy.cpu [fps]", 28.50, "fps"); } + } -} +} \ No newline at end of file From 6912ea6495fc534b642b7c4489fb0d94f425690c Mon Sep 17 00:00:00 2001 From: Rakesh <153008248+RakeshwarK@users.noreply.github.com> Date: Thu, 19 Feb 2026 20:08:34 -0800 Subject: [PATCH 6/8] Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Rakesh <153008248+RakeshwarK@users.noreply.github.com> --- website/docs/guides/0011-profiles.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/guides/0011-profiles.md b/website/docs/guides/0011-profiles.md index 11c7b3aa5f..6dca9fbb34 100644 --- a/website/docs/guides/0011-profiles.md +++ b/website/docs/guides/0011-profiles.md @@ -393,8 +393,8 @@ Metrics in the Virtual Client are assigned a verbosity level that indicates thei | Verbosity Level | Description | Example Metrics | |-----------------|-------------|-----------------| | 1 (Standard/Critical) | Most important metrics for decision making | bandwidth, throughput, IOPS, p50, p99 | -| 2 (Reserved) | Reserved for future expansion | N/A | -| 3 (Detailed) | Additional detailed metrics | p70, p90, p95, p99.9 | +| 2 (Detailed) | Additional detailed metrics | p70, p90, p95, p99.9 | +| 3 (Reserved) | Reserved for future expansion | N/A | | 4 (Reserved) | Reserved for future expansion | N/A | | 5 (Verbose) | All diagnostic/internal metrics | histogram buckets, standard deviations, byte counts, I/O counts | From 241c0505415e8b6f65c0bf500c277c94c6f0b081 Mon Sep 17 00:00:00 2001 From: Rakesh <153008248+RakeshwarK@users.noreply.github.com> Date: Thu, 19 Feb 2026 20:09:15 -0800 Subject: [PATCH 7/8] Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Rakesh <153008248+RakeshwarK@users.noreply.github.com> --- .../VirtualClient.Contracts/Extensibility/MetricDataPoint.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/VirtualClient/VirtualClient.Contracts/Extensibility/MetricDataPoint.cs b/src/VirtualClient/VirtualClient.Contracts/Extensibility/MetricDataPoint.cs index 2ef9547e7d..be3eba791c 100644 --- a/src/VirtualClient/VirtualClient.Contracts/Extensibility/MetricDataPoint.cs +++ b/src/VirtualClient/VirtualClient.Contracts/Extensibility/MetricDataPoint.cs @@ -79,8 +79,9 @@ public MetricDataPoint(TelemetryDataPoint dataPoint) /// /// The priority/verbosity of the metric. Recommended Values: /// - 1 (Standard/Critical): Most important metrics - /// - 3 (Detailed): Additional detailed metrics + /// - 2 (Detailed): Additional detailed metrics /// - 5 (Verbose): All diagnostic/internal metrics + /// Verbosity level 3 is reserved for future expansion and should not be used. /// /// /// Allows the user to ascribe different levels of priority/verbosity to a set of metrics that can From 8d74e83d667a105bf1d90452f10e06fc876706da Mon Sep 17 00:00:00 2001 From: Rakesh <153008248+RakeshwarK@users.noreply.github.com> Date: Thu, 19 Feb 2026 20:11:00 -0800 Subject: [PATCH 8/8] Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Rakesh <153008248+RakeshwarK@users.noreply.github.com> --- website/docs/guides/0011-profiles.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/guides/0011-profiles.md b/website/docs/guides/0011-profiles.md index 6dca9fbb34..7110d0d0b8 100644 --- a/website/docs/guides/0011-profiles.md +++ b/website/docs/guides/0011-profiles.md @@ -472,7 +472,7 @@ Multiple filter types can be combined. Verbosity filtering is applied first, fol ``` The example above will: -1. Include only metrics with verbosity ? 3 +1. Include only metrics with verbosity ≤ 3 2. Exclude metrics matching the pattern "histogram*" 3. From the remaining metrics, include only those matching "bandwidth", "iops", or "latency"