Improve failure handling and finish time estimation (#1921)

* don't use global chronometer for providing the estimated finish time, as it includes build time which skews the estimation

* don't use global chronometer for providing the total time in joined summary, use runsChronometer which does not include build time

* the time logged for sequential build should not include failed parallel build

* move all the logic responsible for handling execution result status to ExecuteResult

* benchmark can fail after few Workload Actual iterations, that is why we search for Workload Results as they are produced at the end

Author: adamsitnik

src/BenchmarkDotNet/Analysers/RuntimeErrorAnalyser.cs

@@ -1,7 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
 using BenchmarkDotNet.Reports;
-using BenchmarkDotNet.Validators;
 
 namespace BenchmarkDotNet.Analysers
 {
@@ -16,12 +15,7 @@ private RuntimeErrorAnalyser()
 
         protected override IEnumerable<Conclusion> AnalyseReport(BenchmarkReport report, Summary summary)
         {
-            var errors = report.ExecuteResults.SelectMany(r => r.Data)
-                .Union(report.ExecuteResults.SelectMany(r => r.ExtraOutput))
-                .Where(line => line.Contains(ValidationErrorReporter.ConsoleErrorPrefix))
-                .Select(line => line.Substring(ValidationErrorReporter.ConsoleErrorPrefix.Length).Trim());
-
-            foreach (string error in errors)
+            foreach (string error in report.ExecuteResults.SelectMany(r => r.Errors))
                 yield return CreateError(error, report);
         }
     }

src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs

@@ -1,18 +1,19 @@
 using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Toolchains.Results;
+using System.Linq;
 
 namespace BenchmarkDotNet.Diagnosers
 {
     public class DiagnoserResults
     {
-        public DiagnoserResults(BenchmarkCase benchmarkCase, long totalOperations, GcStats gcStats,
-            ThreadingStats threadingStats, BuildResult buildResult)
+        public DiagnoserResults(BenchmarkCase benchmarkCase, ExecuteResult executeResult, BuildResult buildResult)
         {
             BenchmarkCase = benchmarkCase;
-            TotalOperations = totalOperations;
-            GcStats = gcStats;
-            ThreadingStats = threadingStats;
+            TotalOperations = executeResult.Measurements.Where(measurement => measurement.IsWorkload()).Sum(m => m.Operations);
+            GcStats = executeResult.GcStats;
+            ThreadingStats = executeResult.ThreadingStats;
             BuildResult = buildResult;
         }
 

src/BenchmarkDotNet/Reports/BenchmarkReport.cs

@@ -36,17 +36,15 @@ public BenchmarkReport(
             GenerateResult generateResult,
             BuildResult buildResult,
             IReadOnlyList<ExecuteResult> executeResults,
-            IReadOnlyList<Measurement> allMeasurements,
-            GcStats gcStats,
             IReadOnlyList<Metric> metrics)
         {
             Success = success;
             BenchmarkCase = benchmarkCase;
             GenerateResult = generateResult;
             BuildResult = buildResult;
             ExecuteResults = executeResults ?? Array.Empty<ExecuteResult>();
-            AllMeasurements = allMeasurements ?? Array.Empty<Measurement>();
-            GcStats = gcStats;
+            AllMeasurements = ExecuteResults.SelectMany((results, index) => results.Measurements).ToArray();
+            GcStats = ExecuteResults.Count > 0 ? executeResults[executeResults.Count -1].GcStats : default;
             Metrics = metrics?.ToDictionary(metric => metric.Descriptor.Id)
                 ?? (IReadOnlyDictionary<string, Metric>)ImmutableDictionary<string, Metric>.Empty;
         }

src/BenchmarkDotNet/Reports/Measurement.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Diagnostics;
 using System.Globalization;
 using System.Linq;
 using System.Text;
@@ -111,12 +112,11 @@ public override string ToString()
         /// Will extract the number of <see cref="Operations"/> performed and the
         /// total number of <see cref="Nanoseconds"/> it took to perform them.
         /// </summary>
-        /// <param name="logger">The logger to write any diagnostic messages to.</param>
         /// <param name="line">The line to parse.</param>
-        /// <param name="processIndex"></param>
+        /// <param name="processIndex">Process launch index, indexed from one.</param>
         /// <returns>An instance of <see cref="Measurement"/> if parsed successfully. <c>Null</c> in case of any trouble.</returns>
         // ReSharper disable once UnusedParameter.Global
-        public static Measurement Parse(ILogger logger, string line, int processIndex)
+        public static Measurement Parse(string line, int processIndex)
         {
             if (string.IsNullOrWhiteSpace(line) || line.StartsWith(GcStats.ResultsLinePrefix))
                 return Error();
@@ -166,8 +166,8 @@ public static Measurement Parse(ILogger logger, string line, int processIndex)
             catch (Exception)
             {
 #if DEBUG // some benchmarks need to write to console and when we display this error it's confusing
-                logger.WriteLineError("Parse error in the following line:");
-                logger.WriteLineError(line);
+                Debug.WriteLine("Parse error in the following line:");
+                Debug.WriteLine(line);
 #endif
                 return Error();
             }