diff --git a/README.md b/README.md index ff23990..aa0b10e 100644 --- a/README.md +++ b/README.md @@ -373,7 +373,8 @@ perfkit checks the steps in order: 1. **Is the measurement valid?** A step is invalid when the load generator, not the server, was the bottleneck: SIPp used more than 80% of one core, placed less than 95% of the offered rate, - or the load generator dropped UDP packets. Then the verdict is **inconclusive**, the reason + placed more than 105% of it (a burst of calls it owed from earlier steps), or the load + generator dropped UDP packets. Then the verdict is **inconclusive**, the reason names the step and the cause, and no limit is published. 2. **Does it pass?** A step passes when p99 is **below** `p99LatencyMs` and failed % is **below** `failedCallsPct`. Both limits are strict: a p99 of exactly 50 ms breaks a 50 ms limit. The diff --git a/package-lock.json b/package-lock.json index 6fdee7b..9712281 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "perfkit", - "version": "0.1.6", + "version": "0.1.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "perfkit", - "version": "0.1.6", + "version": "0.1.7", "license": "MIT", "dependencies": { "commander": "^14.0.3", diff --git a/package.json b/package.json index aab21aa..8c6adc9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "perfkit", - "version": "0.1.6", + "version": "0.1.7", "description": "Benchmark harness for Fonoster projects: DigitalOcean runs, one report.json schema, one branded report page", "license": "MIT", "author": "Pedro Sanders ", diff --git a/src/analyze/measureCapacityStep.ts b/src/analyze/measureCapacityStep.ts index 0d3ec64..e3865f7 100644 --- a/src/analyze/measureCapacityStep.ts +++ b/src/analyze/measureCapacityStep.ts @@ -19,6 +19,11 @@ import { udpDropsOver } from "./udpDropsOver.js"; export const MAX_SIPP_CPU_PCT = 80; /** Placing less than this share of the offered rate means SIPp fell behind. */ export const MIN_ACHIEVED_RATIO = 0.95; +/** + * Placing more than this share means SIPp caught up on calls it owed from + * earlier steps in a burst, so the step measures the burst. + */ +export const MAX_ACHIEVED_RATIO = 1.05; /** The load generator's SIPp process, as the sampler names it. */ export const SIPP_UAC_PROCESS = "sipp-uac"; @@ -43,7 +48,7 @@ export interface MeasuredStep { * Measures one capacity step. Latency counts the calls whose request was * sent in the step; attempts, failures and retransmissions come from the * stat rows in it. A step is invalid when SIPp used more than 80% of one - * core, placed less than 95% of the offered rate, or the load generator's + * core, placed less than 95% or more than 105% of the offered rate, or the load generator's * UDP drops rose. */ export function measureCapacityStep(input: StepInput): MeasuredStep { @@ -78,9 +83,11 @@ export function measureCapacityStep(input: StepInput): MeasuredStep { ? `load generator saturated at ${rate} (SIPp at ${Math.round(sippCpuPct)}% of one core)` : achievedCps < offeredCps * MIN_ACHIEVED_RATIO ? `load generator fell behind at ${rate} (placed ${formatNumber(achievedCps)} ${input.unit}/s, ${Math.round((achievedCps / offeredCps) * 100)}% of offered)` - : loadgenUdpDrops > 0 - ? `load generator dropped UDP packets at ${rate} (${formatNumber(loadgenUdpDrops)} buffer errors)` - : null; + : achievedCps > offeredCps * MAX_ACHIEVED_RATIO + ? `load generator sent a burst at ${rate} (placed ${formatNumber(achievedCps)} ${input.unit}/s, ${Math.round((achievedCps / offeredCps) * 100)}% of offered)` + : loadgenUdpDrops > 0 + ? `load generator dropped UDP packets at ${rate} (${formatNumber(loadgenUdpDrops)} buffer errors)` + : null; return { failedPct, invalidReason, diff --git a/test/analyze/createAnalyzeCapacity.unit.test.ts b/test/analyze/createAnalyzeCapacity.unit.test.ts index 1fb58d3..c270b54 100644 --- a/test/analyze/createAnalyzeCapacity.unit.test.ts +++ b/test/analyze/createAnalyzeCapacity.unit.test.ts @@ -241,6 +241,23 @@ describe("createAnalyzeCapacity", () => { expect(result).not.to.have.property("rules"); }); + it("should be inconclusive when SIPp placed 110% of the offered rate", async () => { + // SIPp that falls behind its schedule later sends the calls it owes in a + // burst; that step measures the burst, not the server under test + // Act + const result = await analyze([ + { offeredCps: 50 }, + { offeredCps: 100, attempted: 6600 } + ]); + + // Assert + expect(result).to.include({ + verdict: "inconclusive", + reason: + "load generator sent a burst at 100 messages/s (placed 110 messages/s, 110% of offered)" + }); + }); + it("should be inconclusive when SIPp placed 90% of the offered rate", async () => { // Act const result = await analyze([