Skip to content

Commit 091bb56

Browse files
timcasselltimcassell
committed
Refactor to use IEngineStageEvaluator for constant instruction location and simpler engine code.
1 parent 284644f commit 091bb56

File tree

5 files changed

+225
-201
lines changed

5 files changed

+225
-201
lines changed

‎src/BenchmarkDotNet/Engines/Engine.cs‎

Lines changed: 44 additions & 185 deletions
Original file line numberDiff line numberDiff line change
@@ -105,14 +105,41 @@ public void Dispose()
105105
}
106106
}
107107

108+
[MethodImpl(MethodImplOptions.NoInlining)]
109+
private IEnumerable<(IterationStage stage, IterationMode mode, IEngineStageEvaluator evaluator)> EnumerateStages()
110+
{
111+
if (Strategy != RunStrategy.ColdStart)
112+
{
113+
if (Strategy != RunStrategy.Monitoring)
114+
{
115+
var pilotEvaluator = pilotStage.GetEvaluator();
116+
if (pilotEvaluator != null)
117+
{
118+
yield return (IterationStage.Pilot, IterationMode.Workload, pilotEvaluator);
119+
}
120+
121+
if (EvaluateOverhead)
122+
{
123+
yield return (IterationStage.Warmup, IterationMode.Overhead, warmupStage.GetOverheadEvaluator());
124+
yield return (IterationStage.Actual, IterationMode.Overhead, actualStage.GetOverheadEvaluator());
125+
}
126+
}
127+
128+
yield return (IterationStage.Warmup, IterationMode.Workload, warmupStage.GetWorkloadEvaluator(Strategy));
129+
}
130+
131+
Host.BeforeMainRun();
132+
133+
yield return (IterationStage.Actual, IterationMode.Workload, actualStage.GetWorkloadEvaluator(Strategy == RunStrategy.Monitoring));
134+
135+
Host.AfterMainRun();
136+
}
137+
108138
// AggressiveOptimization forces the method to go straight to tier1 JIT, and will never be re-jitted,
109139
// eliminating tiered JIT as a potential variable in measurements.
110140
[MethodImpl(CodeGenHelper.AggressiveOptimizationOption)]
111141
public RunResults Run()
112142
{
113-
// This method is huge, because all stages are inlined. This ensures the stack size
114-
// remains constant for each benchmark invocation, eliminating stack sizes as a potential variable in measurements.
115-
// #1120
116143
var measurements = new List<Measurement>();
117144
measurements.AddRange(jittingMeasurements);
118145

@@ -121,191 +148,23 @@ public RunResults Run()
121148
if (EngineEventSource.Log.IsEnabled())
122149
EngineEventSource.Log.BenchmarkStart(BenchmarkName);
123150

124-
if (Strategy != RunStrategy.ColdStart)
125-
{
126-
if (Strategy != RunStrategy.Monitoring)
127-
{
128-
// Pilot Stage
129-
{
130-
// If InvocationCount is specified, pilot stage should be skipped
131-
if (TargetJob.HasValue(RunMode.InvocationCountCharacteristic))
132-
{
133-
}
134-
// Here we want to guess "perfect" amount of invocation
135-
else if (TargetJob.HasValue(RunMode.IterationTimeCharacteristic))
136-
{
137-
// Perfect invocation count
138-
invokeCount = pilotStage.Autocorrect(MinInvokeCount);
139-
140-
int iterationCounter = 0;
141-
142-
int downCount = 0; // Amount of iterations where newInvokeCount < invokeCount
143-
while (true)
144-
{
145-
iterationCounter++;
146-
var measurement = RunIteration(new IterationData(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, UnrollFactor));
147-
measurements.Add(measurement);
148-
double actualIterationTime = measurement.Nanoseconds;
149-
long newInvokeCount = pilotStage.Autocorrect(Math.Max(pilotStage.minInvokeCount, (long) Math.Round(invokeCount * pilotStage.targetIterationTime / actualIterationTime)));
150-
151-
if (newInvokeCount < invokeCount)
152-
downCount++;
153-
154-
if (Math.Abs(newInvokeCount - invokeCount) <= 1 || downCount >= 3)
155-
break;
156-
157-
invokeCount = newInvokeCount;
158-
}
159-
WriteLine();
160-
}
161-
else
162-
{
163-
// A case where we don't have specific iteration time.
164-
invokeCount = pilotStage.Autocorrect(pilotStage.minInvokeCount);
165-
166-
int iterationCounter = 0;
167-
while (true)
168-
{
169-
iterationCounter++;
170-
var measurement = RunIteration(new IterationData(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, UnrollFactor));
171-
measurements.Add(measurement);
172-
double iterationTime = measurement.Nanoseconds;
173-
double operationError = 2.0 * pilotStage.resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision
174-
175-
// Max acceptable operation error
176-
double operationMaxError1 = iterationTime / invokeCount * pilotStage.maxRelativeError;
177-
double operationMaxError2 = pilotStage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue;
178-
double operationMaxError = Math.Min(operationMaxError1, operationMaxError2);
179-
180-
bool isFinished = operationError < operationMaxError && iterationTime >= pilotStage.minIterationTime.Nanoseconds;
181-
if (isFinished)
182-
break;
183-
if (invokeCount >= EnginePilotStage.MaxInvokeCount)
184-
break;
185-
186-
if (UnrollFactor == 1 && invokeCount < EnvironmentResolver.DefaultUnrollFactorForThroughput)
187-
invokeCount += 1;
188-
else
189-
invokeCount *= 2;
190-
}
191-
WriteLine();
192-
}
193-
}
194-
// End Pilot Stage
195-
196-
if (EvaluateOverhead)
197-
{
198-
// Warmup Overhead
199-
{
200-
var warmupMeasurements = new List<Measurement>();
201-
202-
var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(TargetJob, Resolver, IterationMode.Overhead, RunStrategy.Throughput);
203-
int iterationCounter = 0;
204-
while (!criteria.Evaluate(warmupMeasurements).IsFinished)
205-
{
206-
iterationCounter++;
207-
warmupMeasurements.Add(RunIteration(new IterationData(IterationMode.Overhead, IterationStage.Warmup, iterationCounter, invokeCount, UnrollFactor)));
208-
}
209-
WriteLine();
210-
211-
measurements.AddRange(warmupMeasurements);
212-
}
213-
// End Warmup Overhead
214-
215-
// Actual Overhead
216-
{
217-
var measurementsForStatistics = new List<Measurement>(actualStage.maxIterationCount);
218-
219-
int iterationCounter = 0;
220-
double effectiveMaxRelativeError = EngineActualStage.MaxOverheadRelativeError;
221-
while (true)
222-
{
223-
iterationCounter++;
224-
var measurement = RunIteration(new IterationData(IterationMode.Overhead, IterationStage.Actual, iterationCounter, invokeCount, UnrollFactor));
225-
measurements.Add(measurement);
226-
measurementsForStatistics.Add(measurement);
227-
228-
var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, actualStage.outlierMode);
229-
double actualError = statistics.LegacyConfidenceInterval.Margin;
230-
231-
double maxError1 = effectiveMaxRelativeError * statistics.Mean;
232-
double maxError2 = actualStage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue;
233-
double maxError = Math.Min(maxError1, maxError2);
234-
235-
if (iterationCounter >= actualStage.minIterationCount && actualError < maxError)
236-
break;
237-
238-
if (iterationCounter >= actualStage.maxIterationCount || iterationCounter >= EngineActualStage.MaxOverheadIterationCount)
239-
break;
240-
}
241-
WriteLine();
242-
}
243-
// End Actual Overhead
244-
}
245-
}
246-
247-
// Warmup Workload
248-
{
249-
var workloadMeasurements = new List<Measurement>();
250-
251-
var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(TargetJob, Resolver, IterationMode.Workload, Strategy);
252-
int iterationCounter = 0;
253-
while (!criteria.Evaluate(workloadMeasurements).IsFinished)
254-
{
255-
iterationCounter++;
256-
workloadMeasurements.Add(RunIteration(new IterationData(IterationMode.Workload, IterationStage.Warmup, iterationCounter, invokeCount, UnrollFactor)));
257-
}
258-
WriteLine();
259-
260-
measurements.AddRange(workloadMeasurements);
261-
}
262-
// End Warmup Workload
263-
}
264-
265-
Host.BeforeMainRun();
151+
// Enumerate the stages and run iterations in a loop to ensure each benchmark invocation is called with a constant stack size.
152+
// #1120
153+
foreach (var (stage, mode, evaluator) in EnumerateStages())
154+
{
155+
var stageMeasurements = new List<Measurement>(evaluator.MaxIterationCount);
156+
int iterationCounter = 0;
157+
while (!evaluator.EvaluateShouldStop(stageMeasurements, ref invokeCount))
158+
{
159+
// TODO: Not sure why index is 1-based? 0-based is standard.
160+
++iterationCounter;
161+
var measurement = RunIteration(new IterationData(mode, stage, iterationCounter, invokeCount, UnrollFactor));
162+
stageMeasurements.Add(measurement);
163+
}
164+
measurements.AddRange(stageMeasurements);
266165

267-
// Actual Workload
268-
{
269-
if (actualStage.iterationCount == null && Strategy != RunStrategy.Monitoring)
270-
{
271-
// RunAuto
272-
var measurementsForStatistics = new List<Measurement>(actualStage.maxIterationCount);
273-
274-
int iterationCounter = 0;
275-
double effectiveMaxRelativeError = actualStage.maxRelativeError;
276-
while (true)
277-
{
278-
iterationCounter++;
279-
var measurement = RunIteration(new IterationData(IterationMode.Workload, IterationStage.Actual, iterationCounter, invokeCount, UnrollFactor));
280-
measurements.Add(measurement);
281-
measurementsForStatistics.Add(measurement);
282-
283-
var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, actualStage.outlierMode);
284-
double actualError = statistics.LegacyConfidenceInterval.Margin;
285-
286-
double maxError1 = effectiveMaxRelativeError * statistics.Mean;
287-
double maxError2 = actualStage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue;
288-
double maxError = Math.Min(maxError1, maxError2);
289-
290-
if (iterationCounter >= actualStage.minIterationCount && actualError < maxError)
291-
break;
292-
293-
if (iterationCounter >= actualStage.maxIterationCount)
294-
break;
295-
}
296-
}
297-
else
298-
{
299-
// RunSpecific
300-
var iterationCount = actualStage.iterationCount ?? EngineActualStage.DefaultWorkloadCount;
301-
for (int i = 0; i < iterationCount; i++)
302-
measurements.Add(RunIteration(new IterationData(IterationMode.Workload, IterationStage.Actual, i + 1, invokeCount, UnrollFactor)));
303-
}
304166
WriteLine();
305167
}
306-
// End Actual Workload
307-
308-
Host.AfterMainRun();
309168

310169
(GcStats workGcHasDone, ThreadingStats threadingStats, double exceptionFrequency) = includeExtraStats
311170
? GetExtraStats(new IterationData(IterationMode.Workload, IterationStage.Actual, 0, invokeCount, UnrollFactor))

‎src/BenchmarkDotNet/Engines/EngineGeneralStage.cs‎ renamed to ‎src/BenchmarkDotNet/Engines/EngineActualStage.cs‎

Lines changed: 66 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,15 @@ namespace BenchmarkDotNet.Engines
1111
public class EngineActualStage : EngineStage
1212
{
1313
internal const int MaxOverheadIterationCount = 20;
14-
internal const double MaxOverheadRelativeError = 0.05;
15-
internal const int DefaultWorkloadCount = 10;
14+
private const double MaxOverheadRelativeError = 0.05;
15+
private const int DefaultWorkloadCount = 10;
1616

17-
internal readonly int? iterationCount;
18-
internal readonly double maxRelativeError;
19-
internal readonly TimeInterval? maxAbsoluteError;
20-
internal readonly OutlierMode outlierMode;
21-
internal readonly int minIterationCount;
22-
internal readonly int maxIterationCount;
17+
private readonly int? iterationCount;
18+
private readonly double maxRelativeError;
19+
private readonly TimeInterval? maxAbsoluteError;
20+
private readonly OutlierMode outlierMode;
21+
private readonly int minIterationCount;
22+
private readonly int maxIterationCount;
2323

2424
public EngineActualStage(IEngine engine) : base(engine)
2525
{
@@ -86,5 +86,63 @@ private List<Measurement> RunSpecific(long invokeCount, IterationMode iterationM
8686

8787
return measurements;
8888
}
89+
90+
internal IEngineStageEvaluator GetOverheadEvaluator()
91+
=> new AutoEvaluator(this, true);
92+
93+
internal IEngineStageEvaluator GetWorkloadEvaluator(bool forceSpecific)
94+
=> iterationCount == null && !forceSpecific
95+
? new AutoEvaluator(this, false)
96+
: new SpecificEvaluator(this);
97+
98+
private sealed class AutoEvaluator(EngineActualStage stage, bool isOverhead) : IEngineStageEvaluator
99+
{
100+
public int MaxIterationCount => stage.maxIterationCount;
101+
102+
private readonly List<Measurement> _measurementsForStatistics = new (stage.maxIterationCount);
103+
private int _iterationCounter = 0;
104+
105+
public bool EvaluateShouldStop(List<Measurement> measurements, ref long invokeCount)
106+
{
107+
if (measurements.Count == 0)
108+
{
109+
return false;
110+
}
111+
112+
double effectiveMaxRelativeError = isOverhead ? MaxOverheadRelativeError : stage.maxRelativeError;
113+
_iterationCounter++;
114+
var measurement = measurements[measurements.Count - 1];
115+
_measurementsForStatistics.Add(measurement);
116+
117+
var statistics = MeasurementsStatistics.Calculate(_measurementsForStatistics, stage.outlierMode);
118+
double actualError = statistics.LegacyConfidenceInterval.Margin;
119+
120+
double maxError1 = effectiveMaxRelativeError * statistics.Mean;
121+
double maxError2 = stage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue;
122+
double maxError = Math.Min(maxError1, maxError2);
123+
124+
if (_iterationCounter >= stage.minIterationCount && actualError < maxError)
125+
{
126+
return true;
127+
}
128+
129+
if (_iterationCounter >= stage.maxIterationCount || isOverhead && _iterationCounter >= MaxOverheadIterationCount)
130+
{
131+
return true;
132+
}
133+
134+
return false;
135+
}
136+
}
137+
138+
private sealed class SpecificEvaluator(EngineActualStage stage) : IEngineStageEvaluator
139+
{
140+
public int MaxIterationCount => stage.iterationCount ?? DefaultWorkloadCount;
141+
142+
private int _iterationCount = 0;
143+
144+
public bool EvaluateShouldStop(List<Measurement> measurements, ref long invokeCount)
145+
=> ++_iterationCount > MaxIterationCount;
146+
}
89147
}
90148
}

0 commit comments

Comments
 (0)