diff --git a/.github/workflows/tests_and_coverage.yml b/.github/workflows/tests_and_coverage.yml index 31d7838..002964c 100644 --- a/.github/workflows/tests_and_coverage.yml +++ b/.github/workflows/tests_and_coverage.yml @@ -77,6 +77,9 @@ jobs: - name: Build native library for source tests run: python scripts/build_native.py + - name: Verify the checked Rust metadata snapshot + run: git diff --exit-code -- benchset/_metadata.json + - name: Test Rust core with the generated package version run: cargo test --locked -- --test-threads=1 diff --git a/.gitignore b/.gitignore index a428854..e307200 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ coverage.xml target/ .venv/ benchset/_native/ +benchset/.benchset.publish.lock Artifacts/ .build/ release-artifacts/ diff --git a/Cargo.lock b/Cargo.lock index 28ca4af..46f7181 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,4 +4,4 @@ version = 3 [[package]] name = "benchset" -version = "0.0.6" +version = "0.0.7" diff --git a/Cargo.toml b/Cargo.toml index 13d3da3..fc9be10 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,10 @@ [package] name = "benchset" # Generated from [project].version in pyproject.toml by scripts/version.py. -version = "0.0.6" +version = "0.0.7" edition = "2021" rust-version = "1.74" +build = "native/build.rs" description = "Embeddable sustained scalar CPU benchmarks with a C ABI" license = "MIT" repository = "https://github.com/mutating/benchset" diff --git a/MANIFEST.in b/MANIFEST.in index 91f426f..f39c35d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ include setup.py include Cargo.toml Cargo.lock rust-toolchain.toml LICENSE README.md recursive-include native *.rs include native/reference-calibration.json +include benchset/_metadata.json recursive-include include *.h *.modulemap recursive-include scripts *.py recursive-include examples *.c *.rs *.swift diff --git a/README.md b/README.md index ea08df0..b722360 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,18 @@ Sustained scalar CPU throughput, with an embeddable Rust core, a public C ABI, a Python library, a CLI, and a macOS Swift package. No Python interpreter is needed by applications embedding the native library. +Rust is the single source of benchmark behavior: workloads, timing, validation, +suite order and early stopping, scores and aggregation all live in the native +core. Python and Swift are thin bindings to its C ABI. They handle language +types, callbacks, signals and native-handle lifetimes without reimplementing +benchmark rules. The C ABI supports both static and dynamic linking. + +Benchmark identities, default durations and calibration are also defined in +Rust. Native builds generate the Python package's `_metadata.json` from those +definitions; it is a generated data snapshot, not a second implementation. +This keeps imports, metadata inspection and CLI help available without loading +the native library. Update Rust sources rather than editing that snapshot. + The initial suite contains **`single_core`** (one computational worker) and **`multi_core`** (all available computational workers). Both execute the same fixed mixed workload and return a score in **completed iterations per second**. @@ -62,6 +74,9 @@ Completed result fields, including all measurement samples, are printed to stdout. Failed/cancelled results go to stderr. Exit codes: 0 completed, 1 execution failure, 2 invalid CLI arguments, 130 interrupted/cancelled. The suite stops after a failed or cancelled benchmark. +Duration validation loads the Rust library. If loading fails, that error takes +precedence over invalid duration values and the CLI exits with code 1. +Argument syntax, `--help` and `--version` do not require loading the library. Each result is flushed immediately, including when stdout is redirected. Progress fits the terminal width and is cleared before the result. Output uses `× pi` and `—` when the stream supports them, with `x pi` and `-` as @@ -102,7 +117,7 @@ With the current two benchmarks this is mathematically independently of its worker count or measurement duration. The resulting multiplier is dimensionless; individual `score` fields remain iterations/s. -The implementations use natural logarithms and `exp` on the original, +The Rust core uses natural logarithms and `exp` on the original, unrounded double-precision ratios. Log space avoids intermediate overflow or underflow from multiplying ratios. Any consistent logarithm base and its inverse give the same mathematical result; ordinary floating-point rounding @@ -118,6 +133,10 @@ A full score requires exactly `single_core` then `multi_core`, both with unstable results participate. After validating the complete set, any zero ratio makes the full score `0.0`. Zero is a valid score: check `is not None` in Python, `Some` in Rust, `if let` in Swift, or the presence flag in C. +Native numeric validation rejects NaN and infinities without adding floating-point +exception flags; flags already set by the caller are preserved. Native validation +classifies the stored numeric value even when the FPU flushes subnormal numbers +to zero; this also applies to duration validation in every language. An incomplete, failed or cancelled suite has no aggregate: `None` in Python/Rust, `nil` in Swift, or a zero presence flag in C. When a benchmark @@ -127,6 +146,7 @@ complete order, without duplicates or extra entries. Use `SuiteResult.from_results(tuple_of_results)` in Python, `SuiteResult::new(results)` in Rust, or `SuiteResult(results: results)` in Swift to calculate the aggregate from their stored individual Pi equivalents. +The Python and Swift factories call the same Rust calculator through the C ABI. Python's direct `SuiteResult(results, aggregated_scores)` constructor stores the supplied fields; it does not calculate an aggregate. @@ -141,6 +161,24 @@ restart for the next benchmark. Python 3.8–3.15, including free-threaded CPython, is supported. The wrapper uses standard-library `ctypes` and loads the version-matched bundled native library. +`run`, `run_all` and `SuiteResult.from_results` require that native library, +including when inputs are invalid or an aggregate is unavailable. A library +loading or compatibility error takes precedence over benchmark argument +validation. Argument checks and their diagnostics are supplied by Rust; +Python still performs its ordinary numeric type conversions at the boundary. +Integer subclass ratios are calculated from their `float(...)` conversions; +Python aggregation retains Python comparisons, including their behavior under +the caller's FPU mode and subclass overrides. Each requested `result.success` +value is converted to `bool` once. When aggregating a tuple subclass, +the factory uses its actual stored elements and length, retaining the original +tuple in `suite.results`. + +Benchmark names must be strings. A name subclass may customize an unknown-name +diagnostic through Python's string operators; its result must also be a string, +otherwise `TypeError` is raised before any benchmark starts. Compatibility +covers the documented public API, not incidental dependency names imported by +internal modules or the exact time temporary Python objects are finalized. +The CLI does not guarantee context-variable inheritance by its internal threads. ```python from benchset import run, run_all @@ -373,6 +411,8 @@ Swift uses camelCase fields, such as `shortName`, `startedAtUnixSeconds` and `started_at` datetime; Swift offers `startedAt: Date` for the same instant. The frozen Python records use a tuple for results; Swift returns immutable value snapshots. Reading the Python globals does not load the native library. +Their values come from the Rust-generated snapshot bundled with the package; +builds refresh it from the same sources used to compile the native library. [Calibration values and provenance](native/reference-calibration.json) include the original kernel commit, binary SHA-256, raw pooled totals and series @@ -402,7 +442,7 @@ pinned release toolchain described under [Publishing](#publishing). ```toml [dependencies] -benchset = "0.0.6" +benchset = "0.0.7" # For changes from a local source checkout, use: # benchset = { path = "../benchset" } ``` @@ -495,6 +535,12 @@ Check `suite.aggregated_scores.has_pi_equivalent` before reading its `benchset_suite_reason`. Use `benchset_suite_cancel` and `benchset_suite_free` for cancellation and cleanup. The same single-use and lifetime rules apply. +`benchset_run_all_observed` additionally accepts a result callback for streaming +each result before the next benchmark starts. Its child handle is borrowed only +during that callback; copy diagnostics through the single-benchmark getters. +Returning any nonzero value stops further benchmarks. Python and Swift bindings +use this entry point; suite ordering and stopping remain in Rust. + For example, using the initialized `config` above and no progress callback: ```c @@ -562,7 +608,7 @@ the same repository. Rust is not required by the consumer. For a Swift manifest, add: ```swift -.package(url: "https://github.com/mutating/benchset.git", .exact("0.0.6")) +.package(url: "https://github.com/mutating/benchset.git", .exact("0.0.7")) ``` Add `.product(name: "Benchset", package: "benchset")` to your target's @@ -601,7 +647,9 @@ if let fullPi = suite.aggregatedScores.piEquivalent { ``` `runAll` keeps the runner reserved across both benchmarks; cancellation also -applies between its native handles. Native failure/cancellation returns the +applies between benchmarks through the native suite's cancellation token. +Both Swift `runAll` and Python `run_all` call the Rust suite runner through C. +Native failure/cancellation returns the results obtained so far with a `nil` aggregate. Wrapper errors such as allocation failure or an unavailable native result throw `BenchsetError`; that call returns no `SuiteResult`. @@ -677,6 +725,8 @@ controlled initialization failures instead of exhausting machine resources. Full-suite tests also cover complete ordering, geometric means of unrounded ratios, zero/invalid/extreme values, early stopping, serialization and the aggregate's presence in each public interface. +The [core centralization regression matrix](tests/core-centralization-plan.md) +maps the shared-core contracts to characterization tests and directed mutations. Python coverage includes statements and branches, including CLI subprocesses; CI installs a coverage startup hook for those subprocesses. Rust and Swift each diff --git a/Sources/Benchset/Benchset.swift b/Sources/Benchset/Benchset.swift index cf4abd7..676b6d6 100644 --- a/Sources/Benchset/Benchset.swift +++ b/Sources/Benchset/Benchset.swift @@ -42,6 +42,10 @@ private let piReference = try! loadReference(using: .live) public let piReferenceMachine: ReferenceMachine = piReference.machine public let piReferenceRun: ReferenceRun = piReference.run +private let nativeDefaults = loadDefaults(using: .live) +public let defaultWarmupSeconds = nativeDefaults.warmup_seconds +public let defaultMeasurementSeconds = nativeDefaults.measurement_seconds + public struct Progress { public let phase: UInt32 public let workers: UInt32 @@ -98,15 +102,16 @@ public struct SuiteResult { public init(results: [BenchmarkResult]) { self.results = results - let complete = results.map { $0.benchmark } == [.singleCore, .multiCore] - && results.allSatisfy { - $0.success && $0.piEquivalent.map { $0.isFinite && $0 >= 0 } == true - } - let equivalent: Double? - if !complete { equivalent = nil } - else if results.contains(where: { $0.piEquivalent == 0 }) { equivalent = 0 } - else { equivalent = exp(results.reduce(0) { $0 + log($1.piEquivalent!) } / Double(results.count)) } - self.aggregatedScores = AggregatedScores(piEquivalent: equivalent) + let inputs = results.map { + BenchsetAggregateInput(benchmark: $0.benchmark.rawValue, status: $0.status, + pi_equivalent: $0.piEquivalent ?? 0, + has_pi_equivalent: $0.piEquivalent == nil ? 0 : 1) + } + var scores = BenchsetAggregatedScores() + _ = inputs.withUnsafeBufferPointer { benchset_aggregate($0.baseAddress, UInt64($0.count), &scores) } + self.aggregatedScores = AggregatedScores( + piEquivalent: scores.has_pi_equivalent == 0 ? nil : scores.pi_equivalent + ) } } @@ -164,6 +169,21 @@ internal final class CallbackContext { // A private function table keeps ownership/error handling testable without // changing the public Swift API or replacing the benchmark algorithm. internal typealias NativeCallback = @convention(c) (UnsafePointer?, UnsafeMutableRawPointer?) -> Void +internal typealias NativeSuiteCallback = @convention(c) (UnsafePointer?, UnsafeMutableRawPointer?) -> Void +internal typealias NativeResultCallback = @convention(c) (UInt32, OpaquePointer?, UnsafeMutableRawPointer?) -> UInt32 +internal struct SuiteAPI { + var create: () -> OpaquePointer? + var run: (OpaquePointer, UnsafePointer, NativeSuiteCallback, NativeResultCallback, UnsafeMutableRawPointer) -> UInt32 + var cancel: (OpaquePointer) -> Void + var free: (OpaquePointer) -> Void + + static let live = SuiteAPI( + create: { benchset_suite_create() }, + run: { benchset_run_all_observed($0, $1, $2, $3, $4) }, + cancel: { benchset_suite_cancel($0) }, free: { benchset_suite_free($0) } + ) +} + internal struct NativeAPI { var abiVersion: () -> UInt32 var create: () -> OpaquePointer? @@ -175,6 +195,7 @@ internal struct NativeAPI { var reason: (OpaquePointer) -> UnsafePointer? var version: () -> UnsafePointer var free: (OpaquePointer) -> Void + var suite: SuiteAPI = .live static let live = NativeAPI( abiVersion: { benchset_abi_version() }, @@ -186,14 +207,60 @@ internal struct NativeAPI { ) } +internal func loadDefaults(using native: NativeAPI) -> BenchsetConfig { + var config = BenchsetConfig() + guard native.abiVersion() == BENCHSET_ABI_VERSION, native.defaultConfig(&config) == 1 else { + return BenchsetConfig() + } + return config +} + internal func forwardProgress(_ pointer: UnsafePointer?, _ rawContext: UnsafeMutableRawPointer?) { guard let value = pointer?.pointee, let rawContext = rawContext else { return } let context = Unmanaged.fromOpaque(rawContext).takeUnretainedValue() - context.callback(Progress( + context.callback(copyProgress(value)) +} + +private func copyProgress(_ value: BenchsetProgress) -> Progress { + Progress( phase: value.phase, workers: value.workers, fraction: value.fraction, phaseElapsedSeconds: value.phase_elapsed_seconds, phaseDurationSeconds: value.phase_duration_seconds, iterations: value.iterations - )) + ) +} + +internal final class SuiteCallbackContext { + let native: NativeAPI + let callback: (Benchmark, Progress) -> Void + var results: [BenchmarkResult] = [] + var error: Error? + + init(native: NativeAPI, callback: @escaping (Benchmark, Progress) -> Void) { + self.native = native + self.callback = callback + } +} + +internal func forwardSuiteProgress(_ pointer: UnsafePointer?, _ rawContext: UnsafeMutableRawPointer?) { + guard let value = pointer?.pointee, let rawContext = rawContext else { return } + let context = Unmanaged.fromOpaque(rawContext).takeUnretainedValue() + guard let benchmark = Benchmark(rawValue: value.benchmark) else { + context.error = BenchsetError.resultUnavailable + return + } + context.callback(benchmark, copyProgress(value.progress)) +} + +internal func forwardSuiteResult(_ id: UInt32, _ handle: OpaquePointer?, _ rawContext: UnsafeMutableRawPointer?) -> UInt32 { + guard let rawContext = rawContext else { return 1 } + let context = Unmanaged.fromOpaque(rawContext).takeUnretainedValue() + guard let benchmark = Benchmark(rawValue: id), let handle = handle else { + context.error = BenchsetError.resultUnavailable + return 1 + } + do { context.results.append(try readResult(using: context.native, handle: handle, benchmark: benchmark)) } + catch { context.error = error } + return context.error == nil ? 0 : 1 } /// Run from a background queue. Callbacks execute on that queue, not the UI queue. @@ -201,9 +268,7 @@ internal func forwardProgress(_ pointer: UnsafePointer?, _ raw public final class BenchmarkRunner { private let lock = NSLock() private let native: NativeAPI - private var handle: OpaquePointer? - // nil outside runAll; otherwise keep cancellation across its native handles. - private var suiteCancelled: Bool? + private var active: (handle: OpaquePointer, cancel: (OpaquePointer) -> Void, free: (OpaquePointer) -> Void)? public init() { self.native = .live } internal init(native: NativeAPI) { self.native = native } @@ -211,98 +276,98 @@ public final class BenchmarkRunner { public func cancel() { lock.lock() defer { lock.unlock() } - if suiteCancelled != nil { suiteCancelled = true } - if let handle = handle { native.cancel(handle) } + if let active = active { active.cancel(active.handle) } } - /// Run singleCore then multiCore, stopping after failure or cancellation. + /// Run the native suite. Rust owns its order, early stopping and cancellation. public func runAll( - warmupSeconds: Double = 120, - measurementSeconds: Double = 60, + warmupSeconds: Double = defaultWarmupSeconds, + measurementSeconds: Double = defaultMeasurementSeconds, onProgress: @escaping (Benchmark, Progress) -> Void = { _, _ in } ) throws -> SuiteResult { - lock.lock() - guard handle == nil && suiteCancelled == nil else { lock.unlock(); throw BenchsetError.alreadyRunning } - suiteCancelled = false - lock.unlock() - defer { - lock.lock() - suiteCancelled = nil - lock.unlock() - } - var results: [BenchmarkResult] = [] - for benchmark in [Benchmark.singleCore, .multiCore] { - let result = try runBenchmark( - benchmark, warmupSeconds: warmupSeconds, measurementSeconds: measurementSeconds, - inSuite: true, - onProgress: { onProgress(benchmark, $0) } - ) - results.append(result) - if !result.success { break } + let current = try acquire(create: native.suite.create, cancel: native.suite.cancel, free: native.suite.free) + defer { release() } + var config = try configuration(warmupSeconds: warmupSeconds, measurementSeconds: measurementSeconds) + let context = SuiteCallbackContext(native: native, callback: onProgress) + _ = withExtendedLifetime(context) { + native.suite.run(current, &config, forwardSuiteProgress, forwardSuiteResult, + Unmanaged.passUnretained(context).toOpaque()) } - return SuiteResult(results: results) + if let error = context.error { throw error } + return SuiteResult(results: context.results) } public func run( _ benchmark: Benchmark, - warmupSeconds: Double = 120, - measurementSeconds: Double = 60, + warmupSeconds: Double = defaultWarmupSeconds, + measurementSeconds: Double = defaultMeasurementSeconds, onProgress: @escaping (Progress) -> Void = { _ in } ) throws -> BenchmarkResult { - try runBenchmark(benchmark, warmupSeconds: warmupSeconds, measurementSeconds: measurementSeconds, - inSuite: false, onProgress: onProgress) + let current = try acquire(create: native.create, cancel: native.cancel, free: native.free) + defer { release() } + var config = try configuration(warmupSeconds: warmupSeconds, measurementSeconds: measurementSeconds) + config.benchmark = benchmark.rawValue + let context = CallbackContext(onProgress) + _ = withExtendedLifetime(context) { + native.run(current, &config, forwardProgress, Unmanaged.passUnretained(context).toOpaque()) + } + return try readResult(using: native, handle: current, benchmark: benchmark) } - private func runBenchmark( - _ benchmark: Benchmark, warmupSeconds: Double, measurementSeconds: Double, - inSuite: Bool, onProgress: @escaping (Progress) -> Void - ) throws -> BenchmarkResult { + private func acquire( + create: () -> OpaquePointer?, cancel: @escaping (OpaquePointer) -> Void, + free: @escaping (OpaquePointer) -> Void + ) throws -> OpaquePointer { lock.lock() - guard handle == nil && (suiteCancelled == nil || inSuite) else { lock.unlock(); throw BenchsetError.alreadyRunning } - guard native.abiVersion() == BENCHSET_ABI_VERSION else { lock.unlock(); throw BenchsetError.resultUnavailable } - guard let current = native.create() else { lock.unlock(); throw BenchsetError.allocationFailed } - handle = current - if suiteCancelled == true { native.cancel(current) } - lock.unlock() - defer { - lock.lock() - handle = nil - native.free(current) - lock.unlock() - } + defer { lock.unlock() } + guard active == nil else { throw BenchsetError.alreadyRunning } + guard native.abiVersion() == BENCHSET_ABI_VERSION else { throw BenchsetError.resultUnavailable } + guard let current = create() else { throw BenchsetError.allocationFailed } + active = (current, cancel, free) + return current + } + + private func release() { + lock.lock() + defer { lock.unlock() } + let current = active! + active = nil + current.free(current.handle) + } + + private func configuration(warmupSeconds: Double, measurementSeconds: Double) throws -> BenchsetConfig { var config = BenchsetConfig() guard native.defaultConfig(&config) == 1 else { throw BenchsetError.resultUnavailable } - config.benchmark = benchmark.rawValue config.warmup_seconds = warmupSeconds config.measurement_seconds = measurementSeconds - let context = CallbackContext(onProgress) - _ = withExtendedLifetime(context) { - native.run(current, &config, forwardProgress, Unmanaged.passUnretained(context).toOpaque()) - } - var extended = BenchsetResultV2() - guard native.result(current, &extended, UInt32(MemoryLayout.size)) == 1 else { - throw BenchsetError.resultUnavailable - } - let result = extended.base - var samples: [Sample] = [] - for index in 0.. BenchmarkResult { + var extended = BenchsetResultV2() + guard native.result(handle, &extended, UInt32(MemoryLayout.size)) == 1 else { + throw BenchsetError.resultUnavailable + } + let result = extended.base + var samples: [Sample] = [] + for index in 0.. Optional[float]: + if isinstance(results, tuple): + # Snapshot actual tuple storage without invoking subclass protocols. + results = tuple.__getitem__(results, slice(None)) + benchmarks = tuple(result.benchmark for result in results) + # Keep model imports independent of native-library availability. + from benchset._native import Native, _raise_original # noqa: PLC0415 + + native = Native() + # Host mode reads names through the callback; only a non-null pointer is needed. + names = _Names() + errors: List[BaseException] = [] + + def read(index: int, field: int, output: ctypes.POINTER(_Input), _user: object) -> int: # type: ignore[valid-type] + try: + result = results[index] + if field == 0: + output.contents.status = 0 if result.success else 2 + elif field == 1: + output.contents.has_pi_equivalent = result.pi_equivalent is not None + elif field == 2: + output.contents.has_pi_equivalent = bool(cast(float, result.pi_equivalent) >= output.contents.status) + elif field == 3: + output.contents.has_pi_equivalent = bool(result.pi_equivalent == output.contents.status) + elif field == 4: + # Rust requests this primitive only to restore Python's log-domain exception. + output.contents.pi_equivalent = math.log(output.contents.pi_equivalent) + elif field == 6: + native.double(result.pi_equivalent, ctypes.byref(output.contents, _Input.pi_equivalent.offset)) + elif field == 7: + # Preserve Python's exception only when Rust detected finite exp overflow. + output.contents.pi_equivalent = math.exp(output.contents.pi_equivalent) + else: + expected = canonical_name(ctypes.string_at(output.contents.name, output.contents.name_size)) + output.contents.has_pi_equivalent = bool(benchmarks[index] == expected) + return 1 + except BaseException as error: # noqa: BLE001 — Host exceptions cannot unwind through C. + errors.append(error) + return 0 + + function = native.lib.benchset_aggregate_read + function.argtypes = [ctypes.POINTER(ctypes.c_uint32), ctypes.c_uint64, _Reader, + ctypes.c_void_p, ctypes.POINTER(_Scores), ctypes.c_uint32] + function.restype = ctypes.c_int32 + scores = _Scores() + try: + available = function(names, len(results), _Reader(read), None, ctypes.byref(scores), 1) + if errors: + _raise_original(errors[0]) + if not available: + raise RuntimeError('native aggregate is unavailable') + return scores.pi_equivalent if scores.has_pi_equivalent else None + finally: + errors.clear() diff --git a/benchset/_metadata.json b/benchset/_metadata.json new file mode 100644 index 0000000..d1c3873 --- /dev/null +++ b/benchset/_metadata.json @@ -0,0 +1 @@ +{"benchmarks":[{"id":1,"name":"single_core"},{"id":2,"name":"multi_core"}],"default_warmup_seconds":120.0,"default_measurement_seconds":60.0,"reference_machine":{"name":"Raspberry Pi 5 Model B Rev 1.0","short_name":"pi","os_name":"Ubuntu Server","os_version":"26.04 LTS","cpu_architecture":"aarch64","memory_bytes":8589934592,"cpu_cores":4},"reference_run":{"started_at_unix_seconds":1792767590.517391,"start_temperature_c":41.35,"end_temperature_c":66.65,"warmup_seconds":1248.0,"results":[{"benchmark":"single_core","score":47231.227029007096},{"benchmark":"multi_core","score":181953.17419139884}]}} diff --git a/benchset/_metadata.py b/benchset/_metadata.py new file mode 100644 index 0000000..f6424a4 --- /dev/null +++ b/benchset/_metadata.py @@ -0,0 +1,52 @@ +"""Read the immutable data snapshot generated by Rust during native builds.""" + +import json +from pkgutil import get_data +from typing import List, TypedDict, cast + + +class BenchmarkData(TypedDict): + id: int + name: str + + +class ReferenceMachineData(TypedDict): + name: str + short_name: str + os_name: str + os_version: str + cpu_architecture: str + memory_bytes: int + cpu_cores: int + + +class ReferenceResultData(TypedDict): + benchmark: str + score: float + + +class ReferenceRunData(TypedDict): + started_at_unix_seconds: float + start_temperature_c: float + end_temperature_c: float + warmup_seconds: float + results: List[ReferenceResultData] + + +class Metadata(TypedDict): + benchmarks: List[BenchmarkData] + default_warmup_seconds: float + default_measurement_seconds: float + reference_machine: ReferenceMachineData + reference_run: ReferenceRunData + + +_data = cast(bytes, get_data('benchset', '_metadata.json')) +DATA = cast(Metadata, json.loads(_data.decode('utf-8'))) +_NAMES = {item['name']: item['name'] for item in DATA['benchmarks']} + + +def canonical_name(value: bytes) -> str: + """Reuse public catalog strings when decoding native comparison operands.""" + name = value.decode('utf-8') + return _NAMES.get(name, name) diff --git a/benchset/_native.py b/benchset/_native.py index f2ff322..20fb0fe 100644 --- a/benchset/_native.py +++ b/benchset/_native.py @@ -1,15 +1,95 @@ """Private ctypes boundary. All measurement logic lives in Rust.""" +import _ctypes import ctypes import sys +from _ctypes import CFuncPtr +from functools import lru_cache +from os.path import abspath from pathlib import Path -from typing import Callable, List, Optional +from typing import TYPE_CHECKING, Any, Callable, List, NoReturn, Optional, Tuple, cast -from benchset.models import BenchmarkResult, Progress, Sample, __version__ +from benchset._metadata import DATA, canonical_name +from benchset.models import ( + AggregatedScores, + BenchmarkResult, + Progress, + Sample, + SuiteResult, + __version__, +) ABI_VERSION = 1 +if TYPE_CHECKING: # pragma: no cover — Only the type checker uses this alias. + _CDLL = ctypes.CDLL +else: + _CDLL = ctypes.cdll._dlltype + + +class _Library(_CDLL): + """Adopt the opened handle when modern ctypes delegates to its platform hook.""" + + def _load_library(self, name: str, mode: int, handle: int, winmode: Optional[int]) -> int: # noqa: ARG002 — ctypes hook signature. + self._name = name + return handle + + +@lru_cache(maxsize=1) +def _load_library(path: str) -> _CDLL: + """Open before constructing ctypes types, rolling back handles that cannot be adopted.""" + if _AsDouble is None: + return cast(_CDLL, cast(Any, ctypes.cdll)._dlltype(path)) + mode = ctypes.DEFAULT_MODE + if sys.platform == 'win32': + path = abspath(path) # noqa: PTH100 — Match CDLL without resolving symlinks. + mode = 0x1100 # CDLL's LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR. + opener = getattr(ctypes, '_dlopen', None) + if opener is None: + opener = getattr(_ctypes, 'LoadLibrary' if sys.platform == 'win32' else 'dlopen') + handle = opener(path, mode) + try: + factory = cast(Any, ctypes.cdll)._dlltype + return cast(_CDLL, (_Library if factory is _CDLL else factory)(path, handle=handle)) + except BaseException: + if sys.platform == 'win32': + _ctypes.FreeLibrary(handle) + else: + _ctypes.dlclose(handle) + raise + + +class _Functions: + """Own independent address-bound functions without per-call ctypes cycles.""" + + def __init__(self, library: Any) -> None: + self._library = library + + def __getattr__(self, name: str) -> Any: + function = getattr(self._library, name) + if isinstance(function, CFuncPtr) and ( + (function._objects and function._objects.get('0') is self._library) + or getattr(function, 'dll', None) is self._library + ): + function = type(function)(cast(int, ctypes.cast(function, ctypes.c_void_p).value)) + cast(Any, function).__name__ = name + cast(Any, function)._library = self._library + setattr(self, name, function) + return function + + +def _raise_original(error: BaseException) -> NoReturn: + """Preserve host context without retaining the exception in its own traceback.""" + descriptor = BaseException.__dict__['__context__'] + context = descriptor.__get__(error) + try: + raise error + finally: + descriptor.__set__(error, context) + del error + + class Config(ctypes.Structure): _fields_ = [ # noqa: RUF012 — ctypes requires this mutable class attribute. ('abi_version', ctypes.c_uint32), @@ -100,6 +180,42 @@ class CReferenceBenchmarkResult(ctypes.Structure): Callback = ctypes.CFUNCTYPE(None, ctypes.POINTER(CProgress), ctypes.c_void_p) +class CSuiteProgress(ctypes.Structure): + _fields_ = [('benchmark', ctypes.c_uint32), ('progress', CProgress)] # noqa: RUF012 + + +class CAggregatedScores(ctypes.Structure): + _fields_ = [('pi_equivalent', ctypes.c_double), ('has_pi_equivalent', ctypes.c_uint32)] # noqa: RUF012 + + +class CSuiteResult(ctypes.Structure): + _fields_ = [ # noqa: RUF012 + ('results', ctypes.c_void_p), ('result_count', ctypes.c_uint64), ('aggregated_scores', CAggregatedScores), + ] + + +SuiteCallback = ctypes.CFUNCTYPE(None, ctypes.POINTER(CSuiteProgress), ctypes.c_void_p) +ResultCallback = ctypes.CFUNCTYPE(ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p, ctypes.c_void_p) +ConfigCallback = ctypes.CFUNCTYPE(ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(Config), ctypes.c_void_p) +NameMatcher = ctypes.CFUNCTYPE(ctypes.c_int32, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_void_p) +_DoubleCopy = ctypes.PYFUNCTYPE(ctypes.c_int32, ctypes.c_void_p, ctypes.py_object, ctypes.c_void_p) +_api = getattr(ctypes, 'pythonapi', None) +_AsDouble = ctypes.cast(_api.PyFloat_AsDouble, ctypes.c_void_p) if _api is not None else None + + +class _HostComparisonError(BaseException): + def __init__(self, error: BaseException) -> None: + self.error = error + + +class _SuiteConfigurationError(BaseException): + """Carry a caller conversion exception past backend failure handling.""" + + def __init__(self, error: BaseException) -> None: + super().__init__() + self.error = error + + class Native: def __init__(self) -> None: filename = ( @@ -108,7 +224,10 @@ def __init__(self) -> None: else ('libbenchset.dylib' if sys.platform == 'darwin' else 'libbenchset.so') ) path = Path(__file__).parent / '_native' / filename - self.lib = ctypes.CDLL(str(path)) + # Share the loaded image, retaining fresh signatures and injected-loader behavior. + factory = ctypes.CDLL + loader = _load_library if factory is cast(Any, ctypes.cdll)._dlltype else factory + self.lib = _Functions(loader(str(path))) # Check the fixed ABI/version entry points before resolving V2 symbols. self.lib.benchset_abi_version.argtypes = [] self.lib.benchset_abi_version.restype = ctypes.c_uint32 @@ -149,6 +268,27 @@ def __init__(self) -> None: ), 'benchset_reason': ([ctypes.c_void_p], ctypes.c_char_p), 'benchset_free': ([ctypes.c_void_p], None), + 'benchset_benchmark_id': ([ctypes.c_char_p, ctypes.c_uint64], ctypes.c_uint32), + 'benchset_benchmark_match': ([NameMatcher, ctypes.c_void_p, ctypes.c_uint32], ctypes.c_uint32), + 'benchset_duration_error': ([ctypes.c_double, ctypes.c_double], ctypes.c_uint32), + 'benchset_validation_error': ([ctypes.c_uint32, ctypes.c_double, ctypes.c_double], ctypes.c_uint32), + 'benchset_validation_message': ([ctypes.c_uint32, ctypes.c_uint32], ctypes.c_char_p), + 'benchset_aggregate': ([ctypes.c_void_p, ctypes.c_uint64, ctypes.c_void_p], ctypes.c_int32), + 'benchset_aggregate_read': ([ctypes.c_void_p, ctypes.c_uint64, ctypes.c_void_p, + ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint32], ctypes.c_int32), + 'benchset_suite_create': ([], ctypes.c_void_p), + 'benchset_suite_cancel': ([ctypes.c_void_p], None), + 'benchset_suite_free': ([ctypes.c_void_p], None), + 'benchset_run_all_observed': ( + [ctypes.c_void_p, ctypes.POINTER(Config), SuiteCallback, ResultCallback, ctypes.c_void_p], + ctypes.c_uint32, + ), + 'benchset_run_all_configured': ( + [ctypes.c_void_p, ctypes.POINTER(Config), SuiteCallback, ResultCallback, ConfigCallback, ctypes.c_void_p], + ctypes.c_uint32, + ), + 'benchset_suite_result': ([ctypes.c_void_p, ctypes.POINTER(CSuiteResult), ctypes.c_uint32], ctypes.c_int32), + 'benchset_copy_double': ([ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p], ctypes.c_int32), } for name, (arguments, result) in signatures.items(): try: @@ -158,109 +298,236 @@ def __init__(self) -> None: function.argtypes = arguments function.restype = result - def create(self) -> int: - handle = self.lib.benchset_create() + def double(self, value: Any, output: Any = None) -> ctypes.c_double: + """Copy with the interpreter's numeric primitive without signaling NaNs. + + PYFUNCTYPE keeps the Python thread state/GIL and propagates conversion + errors. The local function and self retain the library during the call. + """ + if _AsDouble is None: # Preserve math's conversions when ctypes reads subclass storage directly. + if issubclass(type(value), (float, int)): + value = float(value) + number = ctypes.c_double(value) + if output is not None: + ctypes.memmove(output, ctypes.byref(number), ctypes.sizeof(number)) + return number + number = ctypes.c_double() + function = self.lib.benchset_copy_double + copy = _DoubleCopy(cast(int, ctypes.c_void_p.from_buffer(function).value)) + if not copy(_AsDouble, value, ctypes.byref(number) if output is None else output): + raise RuntimeError('native numeric conversion is unavailable') + return number + + def config(self, benchmark: int, warmup: float, measurement: float) -> Config: + config = Config(ABI_VERSION, ctypes.sizeof(Config), benchmark) + self.double(warmup, ctypes.byref(config, Config.warmup_seconds.offset)) + self.double(measurement, ctypes.byref(config, Config.measurement_seconds.offset)) + return config + + def benchmark_id(self, name: str, *, execution: bool = False, wrap_host: bool = False) -> int: + if type(name) is str: + encoded = str.encode(name, 'utf-8', errors='surrogatepass') + return int(self.lib.benchset_benchmark_id(encoded, len(encoded))) + errors: List[BaseException] = [] + + def matches(pointer: int, size: int, _user: object) -> int: + try: + expected = canonical_name(ctypes.string_at(pointer, size)) + return int(bool(name == expected if execution else expected == name)) + except BaseException as error: # noqa: BLE001 — Host operators cannot unwind through C. + errors.append(error) + return -1 + + try: + value = self.lib.benchset_benchmark_match(NameMatcher(matches), None, execution) + if errors: + if wrap_host: + raise _HostComparisonError(errors[0]) + _raise_original(errors[0]) + return int(value) + finally: + errors.clear() + + def validation_error( + self, benchmark: str, warmup: float, measurement: float, *, wrap_host: bool = False, + ) -> Optional[str]: + code = self.lib.benchset_validation_error(self.benchmark_id(benchmark, wrap_host=wrap_host), + self.double(warmup), self.double(measurement)) + if not code: + return None + reason = str(self.lib.benchset_validation_message(code, 0).decode('utf-8')) + if code != 1: + return reason + try: + reason = reason + ': ' + benchmark + if not issubclass(type(reason), str): + raise TypeError('benchmark diagnostic must be a string') + return reason + except BaseException as error: + if wrap_host: + raise _HostComparisonError(error) from None + raise + + def duration_error(self, warmup: float, measurement: float) -> Optional[str]: + code = self.lib.benchset_duration_error(self.double(warmup), self.double(measurement)) + return str(self.lib.benchset_validation_message(code, 1).decode('utf-8')) if code else None + + def create(self, suite: bool = False) -> int: + handle = self.lib.benchset_suite_create() if suite else self.lib.benchset_create() if not handle: raise RuntimeError('native library could not allocate a benchmark handle') return int(handle) - def cancel(self, handle: int) -> None: - self.lib.benchset_cancel(handle) + def cancel(self, handle: int, suite: bool = False) -> None: + if suite: + self.lib.benchset_suite_cancel(handle) + else: + self.lib.benchset_cancel(handle) + + def free(self, handle: int, suite: bool = False) -> None: + if suite: + self.lib.benchset_suite_free(handle) + else: + self.lib.benchset_free(handle) - def free(self, handle: int) -> None: - self.lib.benchset_free(handle) + @staticmethod + def progress(benchmark: str, value: CProgress) -> Progress: + return Progress( + benchmark, {1: 'warmup', 2: 'measurement', 3: 'completed', 4: 'cancelled', 5: 'failed'}[value.phase], + value.workers, value.fraction, value.phase_elapsed_seconds, value.phase_duration_seconds, value.iterations, + ) + + def result(self, handle: int, benchmark: str, errors: List[BaseException]) -> BenchmarkResult: + value = CResultV2() + if not self.lib.benchset_result_v2(handle, ctypes.byref(value), ctypes.sizeof(value)): + raise RuntimeError('native library did not return a result') + samples = [] + for index in range(value.sample_count): + sample = CSample() + if not self.lib.benchset_sample(handle, index, ctypes.byref(sample)): + raise RuntimeError('native sample is unavailable') + samples.append(Sample(sample.elapsed_seconds, sample.duration_seconds, + sample.iterations, sample.iterations_per_second)) + reason = self.lib.benchset_reason(handle) + status = {0: 'completed', 1: 'cancelled', 2: 'failed'}[value.status] + message = reason.decode('utf-8') if reason else None + if errors: + status = 'failed' + try: + message = f'progress callback failed: {type(errors[0]).__name__}: {errors[0]}' + except BaseException: # noqa: BLE001 — User exception formatters can fail too. + message = f'progress callback failed: {type(errors[0]).__name__}' + return BenchmarkResult( + benchmark=benchmark, status=status, reason=message, + score=value.score if status == 'completed' else None, workers=value.workers, + warmup_seconds=value.warmup_seconds, measurement_seconds=value.measurement_seconds, + elapsed_seconds=value.elapsed_seconds, iterations=value.iterations, checksum=value.checksum, + samples=tuple(samples), variation_percent=value.variation_percent if value.has_variation else None, + unstable=bool(value.unstable), + pi_equivalent=value.pi_equivalent if status == 'completed' and value.has_pi_equivalent else None, + ) def execute( - self, - handle: int, - benchmark: str, - warmup_seconds: float, - measurement_seconds: float, + self, handle: int, benchmark: str, warmup_seconds: float, measurement_seconds: float, on_progress: Optional[Callable[[Progress], None]], ) -> BenchmarkResult: - config = Config( - ABI_VERSION, - ctypes.sizeof(Config), - 1 if benchmark == 'single_core' else 2, - 0, - warmup_seconds, - measurement_seconds, - ) - callback_errors: List[BaseException] = [] + config = self.config(self.benchmark_id(benchmark, execution=True), warmup_seconds, measurement_seconds) + errors: List[BaseException] = [] def callback(pointer: ctypes.POINTER(CProgress), _user: object) -> None: # type: ignore[valid-type] - if on_progress is None or callback_errors: - return - value = pointer.contents - progress = Progress( - benchmark, - { - 1: 'warmup', - 2: 'measurement', - 3: 'completed', - 4: 'cancelled', - 5: 'failed', - }[value.phase], - value.workers, - value.fraction, - value.phase_elapsed_seconds, - value.phase_duration_seconds, - value.iterations, - ) - try: - on_progress(progress) - except BaseException as error: # noqa: BLE001 — Never unwind a Python exception through a C callback. - callback_errors.append(error) - self.cancel(handle) + if on_progress is not None and not errors: + try: + on_progress(self.progress(benchmark, pointer.contents)) + except BaseException as error: # noqa: BLE001 — Never unwind through a C callback. + errors.append(error) + self.cancel(handle) c_callback = Callback(callback) try: self.lib.benchset_run(handle, ctypes.byref(config), c_callback, None) - value = CResultV2() - if not self.lib.benchset_result_v2(handle, ctypes.byref(value), ctypes.sizeof(value)): - raise RuntimeError('native library did not return a result') - samples = [] - for index in range(value.sample_count): - sample = CSample() - if not self.lib.benchset_sample(handle, index, ctypes.byref(sample)): - raise RuntimeError('native sample is unavailable') - samples.append( - Sample( - sample.elapsed_seconds, - sample.duration_seconds, - sample.iterations, - sample.iterations_per_second, - ), - ) - reason = self.lib.benchset_reason(handle) - status = {0: 'completed', 1: 'cancelled', 2: 'failed'}[value.status] - message = reason.decode('utf-8') if reason else None - if callback_errors: - status = 'failed' + return self.result(handle, benchmark, errors) + finally: + # User tracebacks retain the callback frame and its captured data. + errors.clear() + + def execute_suite( # noqa: PLR0913, PLR0915 — Callbacks share one native lifetime and error boundary. + self, handle: int, warmup_seconds: float, measurement_seconds: float, + on_progress: Optional[Callable[[Progress], None]], + on_result: Optional[Callable[[BenchmarkResult], bool]] = None, + convert_windows: Optional[Callable[[], Tuple[float, float]]] = None, + ) -> SuiteResult: + config = self.config(0, warmup_seconds, measurement_seconds) + names = {item['id']: item['name'] for item in DATA['benchmarks']} + results: List[BenchmarkResult] = [] + errors: List[BaseException] = [] + failures: List[BaseException] = [] + configuration_errors: List[BaseException] = [] + unavailable = False + configured = False + current_warmup, current_measurement = warmup_seconds, measurement_seconds + + def configure(_benchmark: int, pointer: ctypes.POINTER(Config), _user: object) -> int: # type: ignore[valid-type] + nonlocal configured, current_warmup, current_measurement + try: + # The public runner already converted the first pair before loading. + if configured and convert_windows is not None: + current_warmup, current_measurement = convert_windows() + self.double(current_warmup, ctypes.byref(pointer.contents, Config.warmup_seconds.offset)) + self.double(current_measurement, ctypes.byref(pointer.contents, Config.measurement_seconds.offset)) + configured = True + return 1 + except BaseException as error: # noqa: BLE001 — Never unwind caller conversions through C. + configuration_errors.append(error) + self.cancel(handle, True) + return 0 + + def progress(pointer: ctypes.POINTER(CSuiteProgress), _user: object) -> None: # type: ignore[valid-type] + if on_progress is not None and not errors and not failures: try: - message = f'progress callback failed: {type(callback_errors[0]).__name__}: {callback_errors[0]}' - except BaseException: # noqa: BLE001 — User exception formatters can fail too. - message = f'progress callback failed: {type(callback_errors[0]).__name__}' - return BenchmarkResult( - benchmark=benchmark, - status=status, - reason=message, - score=value.score if status == 'completed' else None, - workers=value.workers, - warmup_seconds=value.warmup_seconds, - measurement_seconds=value.measurement_seconds, - elapsed_seconds=value.elapsed_seconds, - iterations=value.iterations, - checksum=value.checksum, - samples=tuple(samples), - variation_percent=value.variation_percent if value.has_variation else None, - unstable=bool(value.unstable), - pi_equivalent=( - value.pi_equivalent - if status == 'completed' and value.has_pi_equivalent else None - ), - ) + value = pointer.contents + on_progress(self.progress(names[value.benchmark], value.progress)) + except BaseException as error: # noqa: BLE001 — Python exceptions cannot cross C. + errors.append(error) + self.cancel(handle, True) + + def completed(benchmark: int, child: int, _user: object) -> int: + nonlocal unavailable + try: + try: + reason = self.validation_error(names[benchmark], current_warmup, current_measurement) + if reason is not None and not errors: + result = BenchmarkResult(names[benchmark], reason=reason, warmup_seconds=current_warmup, + measurement_seconds=current_measurement) + else: + result = self.result(child, names[benchmark], errors) + except (OSError, RuntimeError, OverflowError) as error: + unavailable = True + result = BenchmarkResult(names[benchmark], reason=str(error), warmup_seconds=current_warmup, + measurement_seconds=current_measurement) + results.append(result) + stop = on_result(result) if on_result is not None else False + return int(stop or bool(errors) or unavailable) + except BaseException as error: # noqa: BLE001 — Rethrow only after native cleanup. + failures.append(error) + self.cancel(handle, True) + return 1 + + c_progress, c_result, c_config = SuiteCallback(progress), ResultCallback(completed), ConfigCallback(configure) + try: + self.lib.benchset_run_all_configured(handle, ctypes.byref(config), c_progress, c_result, c_config, None) + if configuration_errors: + raise _SuiteConfigurationError(configuration_errors[0]) + if failures: + raise failures[0] + value = CSuiteResult() + if not self.lib.benchset_suite_result(handle, ctypes.byref(value), ctypes.sizeof(value)): + raise RuntimeError('native library did not return a suite result') + if value.result_count != len(results): + raise RuntimeError('native suite results are unavailable') + scores = value.aggregated_scores + equivalent = scores.pi_equivalent if scores.has_pi_equivalent and not errors and not unavailable else None + return SuiteResult(tuple(results), AggregatedScores(equivalent)) finally: - # Tracebacks retain the callback frame and its captured user data. - # Break that cycle even if running or decoding the result fails. - callback_errors.clear() + errors.clear() + failures.clear() + configuration_errors.clear() diff --git a/benchset/api.py b/benchset/api.py index be8132d..7d1962d 100644 --- a/benchset/api.py +++ b/benchset/api.py @@ -1,22 +1,36 @@ """Python API with cancellation and a synchronized native-handle lifetime.""" -import math import signal -from dataclasses import replace +from dataclasses import dataclass as _dataclass, replace from threading import RLock, current_thread, main_thread from types import FrameType -from typing import Callable, Optional +from typing import Callable, Optional, Tuple, TypeVar, Union, cast -from benchset._native import Native +from benchset._native import ( + Native, + _HostComparisonError, + _raise_original, + _SuiteConfigurationError, +) from benchset.models import ( BENCHMARKS, DEFAULT_MEASUREMENT_SECONDS, DEFAULT_WARMUP_SECONDS, + AggregatedScores, BenchmarkResult, Progress, SuiteResult, ) +_HostValue = TypeVar('_HostValue') + + +@_dataclass(eq=False) +class _RunState: + """Keep cancellation with its call while host validation yields the runner.""" + + cancelled: bool = False + class BenchmarkRunner: """Reusable runner. Run in a background thread and cancel from any thread. @@ -28,26 +42,27 @@ class BenchmarkRunner: def __init__(self) -> None: # User signal handlers can reenter cancel() on the calling thread. self._lock = RLock() - self._active: Optional[object] = None - self._cancelled = False + self._active: Optional[_RunState] = None self._native: Optional[Native] = None self._handle: Optional[int] = None + self._suite = False + self._on_native: Optional[Callable[[Native], None]] = None def cancel(self) -> None: """Cancel the active call, including native initialization or warmup.""" with self._lock: if self._active: - self._cancelled = True + self._active.cancelled = True if self._native is not None and self._handle is not None: - self._native.cancel(self._handle) + self._native.cancel(self._handle, True) if self._suite else self._native.cancel(self._handle) - def _release(self, owner: object, native: Optional[Native], handle: Optional[int]) -> None: + def _release(self, owner: _RunState, native: Optional[Native], handle: Optional[int]) -> None: with self._lock: if self._active is owner: self._handle = None self._native = None if native is not None and handle is not None: - native.free(handle) + native.free(handle, True) if self._suite else native.free(handle) self._active = None def run( @@ -58,27 +73,54 @@ def run( measurement_seconds: float = DEFAULT_MEASUREMENT_SECONDS, on_progress: Optional[Callable[[Progress], None]] = None, ) -> BenchmarkResult: + try: + return cast(BenchmarkResult, self._run(benchmark, warmup_seconds, measurement_seconds, on_progress)) + except _HostComparisonError as error: + _raise_original(error.error) + + def _run_all( + self, warmup_seconds: float, measurement_seconds: float, + on_progress: Optional[Callable[[Progress], None]], + on_result: Optional[Callable[[BenchmarkResult], bool]] = None, + ) -> SuiteResult: + try: + return cast(SuiteResult, self._run(BENCHMARKS[0], warmup_seconds, measurement_seconds, + on_progress, on_result, suite=True)) + except (_SuiteConfigurationError, _HostComparisonError) as error: + # _run has released its native handle and restored signal handling. + _raise_original(error.error) + + def _run( # noqa: PLR0913, PLR0915 — One lifetime owner for both execution modes and signal cleanup. + self, benchmark: str, warmup_seconds: float, measurement_seconds: float, + on_progress: Optional[Callable[[Progress], None]], + on_result: Optional[Callable[[BenchmarkResult], bool]] = None, + *, suite: bool = False, + ) -> Union[BenchmarkResult, SuiteResult]: + raw_windows = (warmup_seconds, measurement_seconds) warmup_seconds = float(warmup_seconds) measurement_seconds = float(measurement_seconds) + if not isinstance(benchmark, str): + raise TypeError('benchmark must be a string') + name = benchmark failure = BenchmarkResult( - benchmark=benchmark, + benchmark=name, warmup_seconds=warmup_seconds, measurement_seconds=measurement_seconds, ) - if benchmark not in BENCHMARKS: - return replace(failure, reason='unknown benchmark: ' + benchmark) - if not math.isfinite(warmup_seconds) or warmup_seconds < 0: - return replace( - failure, reason='warmup_seconds must be finite and nonnegative', - ) - if not math.isfinite(measurement_seconds) or measurement_seconds <= 0: - return replace( - failure, reason='measurement_seconds must be finite and positive', - ) native: Optional[Native] = None handle: Optional[int] = None interrupted = False - owner = object() + host_failed = False + owner = _RunState() + + def failed(reason: str) -> Union[BenchmarkResult, SuiteResult]: + result = replace(failure, reason=reason) + if suite and on_result is not None: + on_result(result) + return SuiteResult((result,), AggregatedScores()) if suite else result + + def completed(result: BenchmarkResult) -> bool: + return interrupted or (on_result(result) if on_result is not None else False) def interrupt(_signum: int, _frame: Optional[FrameType]) -> None: nonlocal interrupted @@ -86,44 +128,95 @@ def interrupt(_signum: int, _frame: Optional[FrameType]) -> None: # The runner lock may already be held. Native cancellation is an # atomic operation; the local handle is valid until cleanup starts. if native is not None and handle is not None: - native.cancel(handle) + native.cancel(handle, True) if suite else native.cancel(handle) - handles_signals = ( - current_thread() is main_thread() - and signal.getsignal(signal.SIGINT) is signal.default_int_handler - ) - try: - if handles_signals: + on_main_thread = current_thread() is main_thread() + + def acquire_signals() -> None: + # Caller operators may select a new policy before each execution. + if on_main_thread and signal.getsignal(signal.SIGINT) is signal.default_int_handler: signal.signal(signal.SIGINT, interrupt) + + def caller_operation(operation: Callable[[], _HostValue]) -> _HostValue: + nonlocal host_failed + owns_signals = on_main_thread and signal.getsignal(signal.SIGINT) is interrupt + try: + if owns_signals: + signal.signal(signal.SIGINT, signal.default_int_handler) + return operation() + except BaseException: + # Retain the caller's exception through the native cleanup boundary. + host_failed = True + raise + finally: + if owns_signals and signal.getsignal(signal.SIGINT) is signal.default_int_handler: + signal.signal(signal.SIGINT, interrupt) + + def convert_windows() -> Tuple[float, float]: + try: + # Both conversions precede the next child's signal ownership. + return caller_operation(lambda: (float(raw_windows[0]), float(raw_windows[1]))) + finally: + # Also protect shared-handle cleanup after a caller error. + acquire_signals() + + try: + acquire_signals() try: with self._lock: - if self._active: - return replace(failure, reason='this runner is already running') - self._cancelled = False - # One assignment publishes both activity and ownership, - # even if a user signal handler raises immediately after. - self._active = owner - native = Native() - handle = native.create() + busy = self._active is not None + active_native = self._native + if not busy: + # Publish activity before loading so cancellation is sticky. + self._active = owner + # Reentrant signals before the claim may finish another mode. + self._suite = suite + native = active_native if busy and active_native is not None else Native() + if self._on_native is not None: + self._on_native(native) + with self._lock: + if self._active is owner: + # Caller operators preceded runner ownership before native validation. + self._active = None + reason = caller_operation( + lambda: (native.validation_error(name, warmup_seconds, measurement_seconds) + if type(name) is str else + native.validation_error(name, warmup_seconds, measurement_seconds, wrap_host=True)), + ) + if reason is not None: + return failed(reason) + acquire_signals() + with self._lock: + busy = self._active is not None + if not busy: + self._active = owner + self._suite = suite + if busy: + return failed('this runner is already running') + handle = native.create(True) if suite else native.create() with self._lock: self._native = native self._handle = handle - if self._cancelled or interrupted: - native.cancel(handle) + if owner.cancelled or interrupted: + native.cancel(handle, True) if suite else native.cancel(handle) + if suite: + return native.execute_suite(handle, warmup_seconds, measurement_seconds, + on_progress, completed, convert_windows) return native.execute( - handle, benchmark, warmup_seconds, measurement_seconds, on_progress, + handle, name, warmup_seconds, measurement_seconds, on_progress, ) except (OSError, RuntimeError, OverflowError) as error: - return replace(failure, reason=str(error)) + host_failed = False + return failed(str(error)) finally: # Stop signal-driven cancellation before freeing this call's # handle. _release preserves a concurrent call's ownership. current, handle = handle, None self._release(owner, native, current) finally: - if handles_signals and signal.getsignal(signal.SIGINT) is interrupt: + if on_main_thread and signal.getsignal(signal.SIGINT) is interrupt: signal.signal(signal.SIGINT, signal.default_int_handler) - if interrupted: + if interrupted and not host_failed: # Raising inside a ctypes callback would lose the exception. # Wait until native workers are joined and the handle is freed. raise KeyboardInterrupt @@ -151,15 +244,4 @@ def run_all( on_progress: Optional[Callable[[Progress], None]] = None, ) -> SuiteResult: """Run single_core then multi_core; stop after failure or cancellation.""" - results = [] - for benchmark in BENCHMARKS: - result = run( - benchmark, - warmup_seconds=warmup_seconds, - measurement_seconds=measurement_seconds, - on_progress=on_progress, - ) - results.append(result) - if not result.success: - break - return SuiteResult.from_results(tuple(results)) + return BenchmarkRunner()._run_all(warmup_seconds, measurement_seconds, on_progress) diff --git a/benchset/cli.py b/benchset/cli.py index 1b7af7d..b8cfc95 100644 --- a/benchset/cli.py +++ b/benchset/cli.py @@ -1,18 +1,19 @@ """CLI settings are read by skelet, without environment/config sources.""" import argparse -import math import os import re import shutil import signal import sys -from threading import Event, Thread, current_thread, main_thread +from queue import Empty, Queue +from threading import Event, Event as ResultEvent, Thread, current_thread, main_thread from types import FrameType -from typing import Dict, List, NoReturn, Optional, TextIO +from typing import Dict, List, NoReturn, Optional, TextIO, Tuple from skelet import Field, FixedCLISource, Storage +from benchset._native import Native, _raise_original from benchset.api import BenchmarkRunner from benchset.models import ( BENCHMARKS, @@ -54,7 +55,7 @@ def error(self, message: str) -> NoReturn: super().error(message) -def settings(argv: List[str]) -> Settings: +def _settings(argv: List[str]) -> Tuple[Settings, argparse.ArgumentParser]: source = _Arguments(named_arguments=[]) source.parser = _Parser( prog='benchset', argument_default=argparse.SUPPRESS, @@ -84,11 +85,22 @@ def settings(argv: List[str]) -> Settings: # Bind parsed arguments locally: FixedCLISource's normal getter reparses # global sys.argv and temporarily redirects the process's stderr. source._arguments = vars(source.parser.parse_args(argv)) - result = Settings(_sources=[source]) - if not math.isfinite(result.warmup) or result.warmup < 0: - source.parser.error('--warmup must be finite and nonnegative') - if not math.isfinite(result.duration) or result.duration <= 0: - source.parser.error('--duration must be finite and positive') + return Settings(_sources=[source]), source.parser + + +def _duration_error(config: Settings, native: Optional[Native] = None) -> Optional[str]: + try: + return (Native() if native is None else native).duration_error(config.warmup, config.duration) + except Exception: # noqa: BLE001 — Let the worker preserve the CLI failure boundary. + # Execution reports loader errors as ordinary failed results. + return None + + +def settings(argv: List[str]) -> Settings: + result, parser = _settings(argv) + reason = _duration_error(result) + if reason is not None: + parser.error(reason) return result @@ -154,6 +166,10 @@ def _write(stream: TextIO, text: str) -> None: class _Progress: def __init__(self) -> None: + self.reset() + + def reset(self) -> None: + """Capture the current child's output policy without retaining prior frames.""" self.width = 0 # A disconnected PTY can stop reporting itself as a terminal. Keep # writing progress so the I/O error cancels the native run promptly. @@ -207,77 +223,195 @@ def main(argv: Optional[List[str]] = None) -> int: return 130 -def _execute_benchmark( # noqa: PLR0913 — explicit state passed to the worker thread. - runner: BenchmarkRunner, name: str, config: Settings, progress: _Progress, - results: List[BenchmarkResult], done: Event, -) -> None: - try: - results.append(runner.run( - name, warmup_seconds=config.warmup, measurement_seconds=config.duration, - on_progress=progress, - )) - except Exception as error: # noqa: BLE001 — return background failures to the console. - results.append(BenchmarkResult(benchmark=name, reason=str(error))) - finally: - done.set() - - -def _main(argv: Optional[List[str]]) -> int: +def _main(argv: Optional[List[str]]) -> int: # noqa: PLR0915 — Keep signal/worker ownership and cleanup together. try: - config = settings(list(sys.argv[1:] if argv is None else argv)) + config, parser = _settings(list(sys.argv[1:] if argv is None else argv)) except SystemExit as exit_status: return int(exit_status.code or 0) - names = BENCHMARKS if config.bench == 'all' else (config.bench,) - suite_results: List[BenchmarkResult] = [] - for name in names: - runner = BenchmarkRunner() - results: List[BenchmarkResult] = [] - done = Event() + runner = BenchmarkRunner() + done = Event() + try: progress = _Progress() + except BaseException: + # Argument errors preceded terminal inspection before native validation. + reason = _duration_error(config) + if reason is None: + raise + try: + parser.error(reason) + except SystemExit as exit_status: + return int(exit_status.code or 0) + messages: Queue[BenchmarkResult] = Queue() + resume = ResultEvent() + suites: List[SuiteResult] = [] + interrupted = False + aborted = False + pending: Optional[BenchmarkResult] = None + argument_errors: List[str] = [] + + def validate(native: Native) -> None: + # Load under runner ownership so startup Ctrl+C remains cancellable. + reason = _duration_error(config, native) + if reason is not None: + argument_errors.append(reason) + + runner._on_native = validate + + def publish(result: BenchmarkResult) -> bool: + if aborted or (argument_errors and not interrupted): + return True + # A parked observer must not consume the signal interrupting main's I/O. + mask = getattr(signal, 'pthread_sigmask', None) + previous_mask = mask(signal.SIG_BLOCK, {signal.SIGINT}) if mask is not None else None + try: + messages.put(result) + resume.wait() + resume.clear() + finally: + if mask is not None: + mask(signal.SIG_SETMASK, previous_mask) + return interrupted or aborted - worker = Thread(target=_execute_benchmark, args=(runner, name, config, progress, results, done), - name='benchset-cli') - interrupted = False - - def interrupt(_signum: int, _frame: Optional[FrameType]) -> None: - nonlocal interrupted - # Raising during Thread.start()/join() can leave the worker running. - # Set a flag only; cancellation happens outside the signal handler. - interrupted = True - - handles_signals = current_thread() is main_thread() - previous_handler = signal.getsignal(signal.SIGINT) - if handles_signals: - signal.signal(signal.SIGINT, interrupt) + def execute() -> None: try: - worker.start() - while not done.is_set(): - try: - done.wait(0.05) - except KeyboardInterrupt: - interrupted = True - if interrupted: - runner.cancel() - worker.join() + if config.bench == 'all': + suite = runner._run_all(config.warmup, config.duration, progress, publish) + suites.append(suite) + else: + result = runner.run(config.bench, warmup_seconds=config.warmup, + measurement_seconds=config.duration, on_progress=progress) + publish(result) + except Exception as error: # noqa: BLE001 — Bring background errors to the console. + publish(BenchmarkResult(benchmark=config.bench if config.bench != 'all' else BENCHMARKS[0], reason=str(error))) finally: - if handles_signals: - signal.signal(signal.SIGINT, previous_handler) + done.set() + + def interrupt(_signum: int, _frame: Optional[FrameType]) -> None: + nonlocal interrupted + # Defer cancellation and exception handling until outside signal hooks. + interrupted = True + + def interrupt_output(signum: int, frame: Optional[FrameType]) -> None: + # Break blocked output once, then defer further signals through cleanup. + if not interrupted: + interrupt(signum, frame) + raise KeyboardInterrupt + + def show(result: BenchmarkResult) -> None: progress.clear() - result = results[0] - suite_results.append(result) stream = sys.stdout if result.success and not interrupted else sys.stderr if stream is not None: render(result, stream) stream.flush() - exit_code = 130 if interrupted or result.status == 'cancelled' else (0 if result.success else 1) - if exit_code: - if config.bench == 'all' and sys.stderr is not None: - # SIGINT can arrive after the final native result completed. - render_suite(SuiteResult(tuple(suite_results), AggregatedScores()), sys.stderr) - return exit_code - if name != names[-1]: - sys.stdout.write('\n') + + worker = Thread(target=execute, name='benchset-cli') + handles_signals = current_thread() is main_thread() + previous_handler = signal.getsignal(signal.SIGINT) + output_handler = previous_handler + + def restore_interrupt(unwinding: bool, output: bool = False) -> None: + nonlocal previous_handler + if handles_signals: + while True: + try: + if output: + selected = signal.getsignal(signal.SIGINT) + if selected is not interrupt and selected is not output_handler: + previous_handler = selected + signal.signal(signal.SIGINT, interrupt) + return + except BaseException: # Caller signals cannot replace an error during cleanup. + if not unwinding: + raise + + if handles_signals: + signal.signal(signal.SIGINT, interrupt) + joined = False + guard_errors: List[BaseException] = [] + try: + worker.start() + while not done.is_set(): + try: + done.wait(0.05) + except KeyboardInterrupt: + interrupted = True + if interrupted: + runner.cancel() + try: + result = messages.get_nowait() + except Empty: + continue + # Intermediate results must be flushed before native may continue; + # final results wait for join so late interrupts retain precedence. + if config.bench == 'all' and result.success and result.benchmark != BENCHMARKS[-1] and not interrupted: + # Output may block; restore the caller's interrupt behavior until acknowledgement. + output_handler = interrupt_output if previous_handler is signal.default_int_handler else previous_handler + try: + if handles_signals: + signal.signal(signal.SIGINT, output_handler) + if interrupted: + pending = result + else: + show(result) + sys.stdout.write('\n') + progress.reset() + except BaseException: + restore_interrupt(True, True) + raise + else: + restore_interrupt(False, True) + else: + pending = result + resume.set() + worker.join() + joined = True + finally: + try: + restore_interrupt(not joined) + except BaseException as error: # noqa: BLE001 — Retain a new guard error until native cleanup finishes. + guard_errors.append(error) + restore_interrupt(True) + try: + if worker.ident is not None and worker.is_alive(): + aborted = True + runner.cancel() + resume.set() + worker.join() + if handles_signals: + retry = False + while True: + try: + if retry: + selected = signal.getsignal(signal.SIGINT) + if selected is not interrupt: + previous_handler = selected + retry = True + signal.signal(signal.SIGINT, previous_handler) + break + except BaseException as error: # noqa: BLE001 — Caller signals cannot abandon handler restoration. + if joined and not guard_errors: + guard_errors.append(error) + if previous_handler is None and isinstance(error, TypeError): + break + finally: + if guard_errors: + _raise_original(guard_errors.pop()) # pragma: no branch — This NoReturn call has no normal successor (Python 3.8). + if argument_errors and not interrupted: + try: + parser.error(argument_errors[0]) + except SystemExit as exit_status: + return int(exit_status.code or 0) + if pending is None: + if interrupted: + return 130 + raise RuntimeError('native execution did not return a result') + show(pending) + exit_code = 130 if interrupted or pending.status == 'cancelled' else (0 if pending.success else 1) if config.bench == 'all': - sys.stdout.write('\n') - render_suite(SuiteResult.from_results(tuple(suite_results)), sys.stdout) - return 0 + suite = suites[0] if suites and not exit_code else SuiteResult((), AggregatedScores()) + stream = sys.stdout if not exit_code else sys.stderr + if stream is not None: + if not exit_code: + stream.write('\n') + render_suite(suite, stream) + return exit_code diff --git a/benchset/models.py b/benchset/models.py index b413ffa..4abaedc 100644 --- a/benchset/models.py +++ b/benchset/models.py @@ -1,10 +1,10 @@ """Public, immutable measurement objects.""" -import math from dataclasses import dataclass from importlib.metadata import version -from typing import Optional, Tuple +from typing import Optional, Tuple, cast +from benchset._metadata import DATA from benchset.reference import ( PI_REFERENCE_MACHINE, PI_REFERENCE_RUN, @@ -13,9 +13,10 @@ ) __version__ = version('benchset') -BENCHMARKS = ('single_core', 'multi_core') -DEFAULT_WARMUP_SECONDS = 120.0 -DEFAULT_MEASUREMENT_SECONDS = 60.0 +_benchmarks = tuple(benchmark['name'] for benchmark in DATA['benchmarks']) +BENCHMARKS = cast(Tuple[str, str], _benchmarks) +DEFAULT_WARMUP_SECONDS = DATA['default_warmup_seconds'] +DEFAULT_MEASUREMENT_SECONDS = DATA['default_measurement_seconds'] @dataclass(frozen=True) @@ -85,19 +86,7 @@ class SuiteResult: @classmethod def from_results(cls, results: Tuple[BenchmarkResult, ...]) -> 'SuiteResult': - complete = ( - tuple(result.benchmark for result in results) == BENCHMARKS - and all( - result.success and result.pi_equivalent is not None - and math.isfinite(result.pi_equivalent) and result.pi_equivalent >= 0 - for result in results - ) - ) - equivalent: Optional[float] = None - if complete: - equivalent = ( - 0.0 if any(result.pi_equivalent == 0 for result in results) - else math.exp(sum(math.log(result.pi_equivalent) for result in results - if result.pi_equivalent is not None) / len(results)) - ) - return cls(results, AggregatedScores(pi_equivalent=equivalent)) + # Load Rust only when computing, preserving the native-free constructor. + from benchset._aggregate import aggregate # noqa: PLC0415 + + return cls(results, AggregatedScores(pi_equivalent=aggregate(results))) diff --git a/benchset/reference.py b/benchset/reference.py index cd76d86..3dbde21 100644 --- a/benchset/reference.py +++ b/benchset/reference.py @@ -1,4 +1,4 @@ -"""Immutable Pi calibration, mirrored from Rust for native-free inspection. +"""Immutable Pi calibration, generated by Rust for native-free inspection. Integration tests compare every value with the Rust C ABI and calibration fixture. The reference covers the six final runs, excluding preparatory @@ -9,6 +9,8 @@ from datetime import datetime, timezone from typing import Tuple +from benchset._metadata import DATA + @dataclass(frozen=True) class ReferenceMachine: @@ -52,23 +54,9 @@ def started_at(self) -> datetime: return datetime.fromtimestamp(self.started_at_unix_seconds, timezone.utc) -PI_REFERENCE_MACHINE = ReferenceMachine( - name='Raspberry Pi 5 Model B Rev 1.0', - short_name='pi', - os_name='Ubuntu Server', - os_version='26.04 LTS', - cpu_architecture='aarch64', - memory_bytes=8 * 1024 * 1024 * 1024, - cpu_cores=4, -) - +PI_REFERENCE_MACHINE = ReferenceMachine(**DATA['reference_machine']) +_run = DATA['reference_run'] PI_REFERENCE_RUN = ReferenceRun( - started_at_unix_seconds=1792767590.517391, - start_temperature_c=41.35, - end_temperature_c=66.65, - warmup_seconds=1248.0, - results=( - ReferenceBenchmarkResult('single_core', 47231.227029007096), - ReferenceBenchmarkResult('multi_core', 181953.17419139884), - ), + _run['started_at_unix_seconds'], _run['start_temperature_c'], _run['end_temperature_c'], + _run['warmup_seconds'], tuple(ReferenceBenchmarkResult(**result) for result in _run['results']), ) diff --git a/include/benchset.h b/include/benchset.h index 9bfdbdf..099c9c8 100644 --- a/include/benchset.h +++ b/include/benchset.h @@ -69,7 +69,27 @@ typedef void (*BenchsetProgressCallback)(const BenchsetProgress *, void *); uint32_t benchset_abi_version(void); const char *benchset_version(void); +/* Read once on the calling thread and copy its double without arithmetic. + * reader/user are borrowed only during the call; user may be NULL if supported + * by reader. reader must not unwind. NULL reader/out returns 0 without calling + * reader or modifying output; otherwise aligned writable out receives the value + * and the function returns 1. out may alias caller data used by reader. */ +typedef double (*BenchsetDoubleReader)(void *); +int32_t benchset_copy_double(BenchsetDoubleReader, void *user, double *out); int32_t benchset_default_config(BenchsetConfig *out); +/* UTF-8 byte length, including any embedded NUL. Unknown names return 0. */ +uint32_t benchset_benchmark_id(const uint8_t *name, uint64_t size); +/* Compare caller-owned host names with borrowed catalog UTF-8 names. A positive + * return matches, zero continues, negative stops. Execution mode compares every + * entry except the final fallback; validation mode checks the full catalog. */ +typedef int32_t (*BenchsetNameMatcher)(const uint8_t *, uint64_t, void *); +uint32_t benchset_benchmark_match(BenchsetNameMatcher, void *user, uint32_t execution); +/* Validation codes: 0 valid, 1 unknown benchmark, 2 warmup, 3 measurement. + * Duration validation is independent of representability as a Rust Duration. */ +uint32_t benchset_duration_error(double warmup, double measurement); +uint32_t benchset_validation_error(uint32_t benchmark, double warmup, double measurement); +/* Immutable library-lifetime message; nonzero cli selects CLI option names. */ +const char *benchset_validation_message(uint32_t error, uint32_t cli); BenchsetHandle *benchset_create(void); /* A handle is single-use. Only one run per loaded library may be active. * Callbacks run on the calling thread. They must return promptly and must not @@ -110,6 +130,40 @@ typedef struct { double pi_equivalent; /* Geometric mean: exp(mean(log(Pi equivalent))); zero if any value is zero. */ uint32_t has_pi_equivalent; /* Absent unless the complete suite succeeded. */ } BenchsetAggregatedScores; +typedef struct { + uint32_t benchmark, status; + double pi_equivalent; + uint32_t has_pi_equivalent; +} BenchsetAggregateInput; +/* Aggregate caller-owned snapshots. Invalid/incomplete suites have no score. + * Any nonzero presence flag means present. NULL inputs is valid only at count 0. + * Non-NULL pointers must be aligned and valid; out must be writable, and a + * complete-suite input must contain count readable entries. Only NULL is checked. + * Returns 0 for invalid NULL pointers, leaving out unchanged; otherwise returns 1. */ +int32_t benchset_aggregate(const BenchsetAggregateInput *, uint64_t count, BenchsetAggregatedScores *out); +typedef struct { + BenchsetAggregateInput value; + const uint8_t *name; + uint64_t name_size; +} BenchsetAggregateRequest; +/* Lazy host primitives: 0 reads status; 1 reads ratio/presence; 2 requests >=; + * 3 requests ==; 4 requests log(value.pi_equivalent) for a nonpositive argument; + * 5 compares the caller's name with the borrowed UTF-8 name/name_size operand; + * 6 reads a fresh ratio in host-name mode, where 1 reads only its presence; + * 7 requests exp(value.pi_equivalent) only for finite host-mode overflow. + * Operators 2/3 receive an integer operand in value.status. Comparisons return + * their truth value in value.has_pi_equivalent; fields 4/7 return a double. + * Every nonzero presence/comparison flag and reader return is accepted. + * The callback-borrowed buffer must retain its size; only requested fields are read. + * Host-name mode compares the shared prefix before checking count, then the core + * requests status/ratio/>= for each entry, followed by short-circuit == and logs. + * Each host-mode log separately requests presence, then a fresh ratio if present. + * With host_names=0, ordered raw IDs and count are checked before host reads. + * A zero reader result makes the aggregate unavailable. Reader is required; + * other NULL/output rules match benchset_aggregate. */ +typedef int32_t (*BenchsetAggregateReader)(uint64_t index, uint32_t field, BenchsetAggregateRequest *, void *); +int32_t benchset_aggregate_read(const uint32_t *benchmarks, uint64_t count, + BenchsetAggregateReader, void *user, BenchsetAggregatedScores *out, uint32_t host_names); typedef struct { const BenchsetResultV3 *results; /* Immutable array, borrowed until suite_free. */ uint64_t result_count; @@ -125,6 +179,19 @@ BenchsetSuiteHandle *benchset_suite_create(void); * multi_core, stopping after failure/cancellation. Same callback and lifetime * rules as benchset_run. Returns the final individual result's status. */ uint32_t benchset_run_all(BenchsetSuiteHandle *, const BenchsetConfig *, BenchsetSuiteProgressCallback, void *); +/* Called synchronously after each result and before starting the next benchmark. + * The child handle is borrowed only during this callback; use single-run getters + * to copy its result, samples and reason. Never free it. Nonzero stops the suite. */ +typedef uint32_t (*BenchsetResultCallback)(uint32_t benchmark, const BenchsetHandle *, void *); +uint32_t benchset_run_all_observed(BenchsetSuiteHandle *, const BenchsetConfig *, BenchsetSuiteProgressCallback, BenchsetResultCallback, void *); +/* Called on the calling thread before each child, after checking and copying the + * config prefix. The writable copy is borrowed only during this callback; its + * benchmark is ignored. Nonzero proceeds to normal validation/execution; zero + * aborts with BENCHSET_FAILED and retains the completed prefix without adding a + * result. NULL/incompatible configs still publish their normal failed result + * without calling this callback. The caller's original config is unchanged. */ +typedef uint32_t (*BenchsetConfigCallback)(uint32_t benchmark, BenchsetConfig *, void *); +uint32_t benchset_run_all_configured(BenchsetSuiteHandle *, const BenchsetConfig *, BenchsetSuiteProgressCallback, BenchsetResultCallback, BenchsetConfigCallback, void *); void benchset_suite_cancel(BenchsetSuiteHandle *); /* Getters return 1 on success, 0 on NULL/wrong size/unavailable index, leaving * output unchanged on failure. All indices are zero-based. */ diff --git a/native/build.rs b/native/build.rs new file mode 100644 index 0000000..4da3745 --- /dev/null +++ b/native/build.rs @@ -0,0 +1,69 @@ +//! Export immutable package data from the same Rust definitions as the runtime. +#![allow(dead_code)] + +#[path = "src/catalog.rs"] +mod catalog; +#[path = "src/reference.rs"] +mod reference; +pub use catalog::Benchmark; + +use std::{env, fmt::Write, fs, path::PathBuf}; + +fn quoted(value: &str) -> String { + let mut output = String::from("\""); + for character in value.chars() { + match character { + '"' => output.push_str("\\\""), + '\\' => output.push_str("\\\\"), + '\u{0}'..='\u{1f}' => write!(output, "\\u{:04x}", character as u32).unwrap(), + _ => output.push(character), + } + } + output.push('"'); + output +} + +fn main() { + let machine = &reference::PI_REFERENCE_MACHINE; + let run = &reference::PI_REFERENCE_RUN; + let benchmarks = catalog::BENCHMARKS + .map(|benchmark| { + format!( + "{{\"id\":{},\"name\":{}}}", + benchmark as u32, + quoted(benchmark.name()) + ) + }) + .join(","); + let results = run + .results + .iter() + .map(|result| { + format!( + "{{\"benchmark\":{},\"score\":{:?}}}", + quoted(result.benchmark), + result.score + ) + }) + .collect::>() + .join(","); + let snapshot = format!(concat!( + "{{\"benchmarks\":[{}],\"default_warmup_seconds\":{:?},\"default_measurement_seconds\":{:?},", + "\"reference_machine\":{{\"name\":{},\"short_name\":{},\"os_name\":{},\"os_version\":{},", + "\"cpu_architecture\":{},\"memory_bytes\":{},\"cpu_cores\":{}}},", + "\"reference_run\":{{\"started_at_unix_seconds\":{:?},\"start_temperature_c\":{:?},", + "\"end_temperature_c\":{:?},\"warmup_seconds\":{:?},\"results\":[{}]}}}}\n" + ), benchmarks, catalog::DEFAULT_WARMUP_SECONDS, catalog::DEFAULT_MEASUREMENT_SECONDS, + quoted(machine.name), quoted(machine.short_name), quoted(machine.os_name), quoted(machine.os_version), + quoted(machine.cpu_architecture), machine.memory_bytes, machine.cpu_cores, + run.started_at_unix_seconds, run.start_temperature_c, run.end_temperature_c, run.warmup_seconds, results); + let path = PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("benchset-metadata.json"); + fs::write(&path, snapshot).unwrap(); + let mut encoded_path = String::new(); + for byte in path.as_os_str().as_encoded_bytes() { + write!(encoded_path, "{byte:02x}").unwrap(); + } + println!("cargo:rustc-env=BENCHSET_METADATA_PATH_HEX={encoded_path}"); + println!("cargo:rerun-if-changed=native/src/catalog.rs"); + println!("cargo:rerun-if-changed=native/src/reference.rs"); +} diff --git a/native/src/aggregate_protocol_tests.rs b/native/src/aggregate_protocol_tests.rs new file mode 100644 index 0000000..084c304 --- /dev/null +++ b/native/src/aggregate_protocol_tests.rs @@ -0,0 +1,545 @@ +use super::*; +use std::slice; + +#[derive(Debug, PartialEq)] +struct Call { + index: u64, + field: u32, + benchmark: u32, + operand_status: u32, + operand_bits: u64, + presence: u32, + name: Vec, +} + +struct ReaderState { + ratios: [f64; 2], + log_ratios: Option<[f64; 2]>, + presence: [u32; 2], + log_presence: [u32; 2], + presence_reads: [u32; 2], + numeric_reads: [u32; 2], + host_names: bool, + name_matches: [u32; 2], + nonnegative: [u32; 2], + zero: [u32; 2], + logarithms: [f64; 2], + exponential: f64, + reject: Option<(u64, u32)>, + reject_at: Option, + calls: Vec, + invalid_request: bool, +} + +impl Default for ReaderState { + fn default() -> Self { + Self { + ratios: [0.5, 2.0], + log_ratios: None, + presence: [u32::MAX; 2], + log_presence: [7; 2], + presence_reads: [0; 2], + numeric_reads: [0; 2], + host_names: false, + name_matches: [7, u32::MAX], + nonnegative: [7, u32::MAX], + zero: [0, 0], + logarithms: [1.0, 3.0], + exponential: 42.5, + reject: None, + reject_at: None, + calls: Vec::new(), + invalid_request: false, + } + } +} + +unsafe extern "C" fn read( + index: u64, + field: u32, + request: *mut CAggregateRequest, + user: *mut c_void, +) -> i32 { + let state = &mut *user.cast::(); + let Some(request) = request.as_mut() else { + state.invalid_request = true; + return 0; + }; + if index >= 2 || field > 7 || request.name.is_null() || request.name_size > 32 { + state.invalid_request = true; + return 0; + } + // Copy borrowed UTF-8 during the callback; no request pointer escapes it. + let name = slice::from_raw_parts(request.name, request.name_size as usize).to_vec(); + state.calls.push(Call { + index, + field, + benchmark: request.value.benchmark, + operand_status: request.value.status, + operand_bits: request.value.pi_equivalent.to_bits(), + presence: request.value.has_pi_equivalent, + name, + }); + if state.reject == Some((index, field)) || state.reject_at == Some(state.calls.len() - 1) { + return 0; + } + let index = index as usize; + match field { + 0 => request.value.status = 0, + 1 => { + if state.host_names { + request.value.has_pi_equivalent = if state.presence_reads[index] == 0 { + state.presence[index] + } else { + state.log_presence[index] + }; + state.presence_reads[index] += 1; + } else { + request.value.pi_equivalent = state.ratios[index]; + request.value.has_pi_equivalent = state.presence[index]; + } + } + 2 => request.value.has_pi_equivalent = state.nonnegative[index], + 3 => request.value.has_pi_equivalent = state.zero[index], + 4 => request.value.pi_equivalent = state.logarithms[index], + 5 => request.value.has_pi_equivalent = state.name_matches[index], + 6 => { + request.value.pi_equivalent = if state.numeric_reads[index] == 0 { + state.ratios[index] + } else { + state.log_ratios.unwrap_or(state.ratios)[index] + }; + state.numeric_reads[index] += 1; + } + 7 => request.value.pi_equivalent = state.exponential, + _ => return 0, + } + -7 +} + +fn execute(state: &mut ReaderState, count: u64, host_names: u32) -> CAggregatedScores { + execute_ids(state, count, host_names, [0, u32::MAX, 0]) +} + +fn execute_ids( + state: &mut ReaderState, + count: u64, + host_names: u32, + ids: [u32; 3], +) -> CAggregatedScores { + #[repr(C)] + struct Guarded { + before: u64, + scores: CAggregatedScores, + after: u64, + } + let mut guarded = Guarded { + before: 0x123456789abcdef0, + scores: CAggregatedScores { + pi_equivalent: 123.0, + has_pi_equivalent: 7, + }, + after: 0xfedcba9876543210, + }; + state.host_names = host_names != 0; + // Host-name mode must not reconstruct identity from these unrelated IDs. + let result = unsafe { + benchset_aggregate_read( + ids.as_ptr(), + count, + Some(read), + (state as *mut ReaderState).cast(), + &mut guarded.scores, + host_names, + ) + }; + assert_eq!(result, 1); + assert_eq!(guarded.before, 0x123456789abcdef0); + assert_eq!(guarded.after, 0xfedcba9876543210); + assert!(!state.invalid_request); + guarded.scores +} + +fn trace(fields: &[(u64, u32)], ratios: [f64; 2]) -> Vec { + fields + .iter() + .map(|&(index, field)| { + let operand = if field == 4 { + ratios[index as usize] + } else { + 0.0 + }; + Call { + index, + field, + benchmark: 0, + operand_status: 0, + operand_bits: operand.to_bits(), + presence: 0, + name: if field == 5 { + [b"single_core".as_slice(), b"multi_core".as_slice()][index as usize].to_vec() + } else { + Vec::new() + }, + } + }) + .collect() +} + +const COMPLETE: [(u64, u32); 16] = [ + (0, 5), + (1, 5), + (0, 0), + (0, 1), + (0, 6), + (0, 2), + (1, 0), + (1, 1), + (1, 6), + (1, 2), + (0, 3), + (1, 3), + (0, 1), + (0, 6), + (1, 1), + (1, 6), +]; + +/// Host names use exact borrowed UTF-8 and canonical operands, ignore raw IDs, +/// accept noncanonical true flags, and compare the common prefix before length. +#[test] +fn host_names_preserve_operands_prefix_and_nonzero_flags() { + for flag in [1, 7, u32::MAX] { + let mut state = ReaderState { + name_matches: [flag, flag], + nonnegative: [flag, flag], + ..ReaderState::default() + }; + let scores = execute(&mut state, 2, u32::MAX); + assert_eq!(scores.has_pi_equivalent, 1); + assert_eq!(scores.pi_equivalent, 1.0); + assert_eq!(state.calls, trace(&COMPLETE, state.ratios)); + } + for count in [0, 1, 3, u64::MAX] { + let mut state = ReaderState::default(); + let scores = execute(&mut state, count, 1); + assert_eq!(scores.has_pi_equivalent, 0); + assert_eq!(scores.pi_equivalent, 0.0); + assert_eq!( + state.calls, + trace(&COMPLETE[..count.min(2) as usize], state.ratios) + ); + } + let mut state = ReaderState::default(); + let scores = execute(&mut state, 2, 0); + assert_eq!(scores.has_pi_equivalent, 0); + assert!(state.calls.is_empty()); +} + +/// A false comparison and an unavailable host comparison each stop at that name, +/// without inspecting later identities, status, ratios or numeric operators. +#[test] +fn host_name_mismatch_and_reader_failure_stop_before_values() { + for index in 0..2 { + for failed_reader in [false, true] { + let mut state = ReaderState::default(); + if failed_reader { + state.reject = Some((index, 5)); + } else { + state.name_matches[index as usize] = 0; + } + let scores = execute(&mut state, 2, 1); + assert_eq!(scores.has_pi_equivalent, 0); + assert_eq!(scores.pi_equivalent, 0.0); + assert_eq!( + state.calls, + trace(&COMPLETE[..index as usize + 1], state.ratios) + ); + } + } +} + +/// Nonfinite primitives stop before host >=; a false host >= is authoritative +/// even for positive primitives. Either position must suppress every later read. +#[test] +fn host_numeric_validation_preserves_short_circuiting() { + for index in 0..2 { + for invalid in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let mut state = ReaderState::default(); + state.ratios[index as usize] = invalid; + let scores = execute(&mut state, 2, 1); + let stop = COMPLETE + .iter() + .position(|entry| *entry == (index, 6)) + .unwrap() + + 1; + assert_eq!(scores.has_pi_equivalent, 0); + assert_eq!(scores.pi_equivalent, 0.0); + assert_eq!(state.calls, trace(&COMPLETE[..stop], state.ratios)); + } + let mut state = ReaderState::default(); + state.nonnegative[index as usize] = 0; + let scores = execute(&mut state, 2, 1); + let stop = COMPLETE + .iter() + .position(|entry| *entry == (index, 2)) + .unwrap() + + 1; + assert_eq!(scores.has_pi_equivalent, 0); + assert_eq!(scores.pi_equivalent, 0.0); + assert_eq!(state.calls, trace(&COMPLETE[..stop], state.ratios)); + } +} + +/// Host operators can admit nonpositive primitives. Forward successful host logs +/// in order, propagate either reader failure, and skip all logs after host zero. +#[test] +fn host_logs_forward_values_failures_and_short_circuit_order() { + for failure in [None, Some(0), Some(1)] { + let mut state = ReaderState { + ratios: [0.0, -2.0], + reject: failure.map(|index| (index, 4)), + ..ReaderState::default() + }; + let scores = execute(&mut state, 2, 1); + let mut expected = COMPLETE[..14].to_vec(); + expected.push((0, 4)); + if failure != Some(0) { + expected.extend_from_slice(&COMPLETE[14..]); + expected.push((1, 4)); + } + assert_eq!(state.calls, trace(&expected, state.ratios)); + assert_eq!(scores.has_pi_equivalent, u32::from(failure.is_none())); + assert_eq!( + scores.pi_equivalent, + if failure.is_none() { + 2.0_f64.exp() + } else { + 0.0 + } + ); + } + let mut state = ReaderState { + ratios: [0.0, -2.0], + zero: [u32::MAX, 0], + ..ReaderState::default() + }; + let scores = execute(&mut state, 2, 1); + assert_eq!(scores.has_pi_equivalent, 1); + assert_eq!(scores.pi_equivalent.to_bits(), 0.0_f64.to_bits()); + assert_eq!(state.calls, trace(&COMPLETE[..11], state.ratios)); +} + +/// Every requested host primitive is fallible, including the separate numeric +/// reads and repeated presence guards used only while computing logarithms. +#[test] +fn host_reader_failures_stop_at_each_requested_primitive() { + for rejected in 0..=COMPLETE.len() { + let mut state = ReaderState { + reject_at: (rejected < COMPLETE.len()).then_some(rejected), + ..ReaderState::default() + }; + let scores = execute(&mut state, 2, 1); + assert_eq!( + state.calls, + trace( + &COMPLETE[..(rejected + 1).min(COMPLETE.len())], + state.ratios + ) + ); + assert_eq!( + scores.has_pi_equivalent, + u32::from(rejected == COMPLETE.len()) + ); + assert_eq!( + scores.pi_equivalent, + if rejected == COMPLETE.len() { 1.0 } else { 0.0 } + ); + } +} + +/// Presence checks do not convert numbers. A missing validation value stops +/// before its numeric getter; a missing log guard skips just that logarithm. +#[test] +fn host_presence_guards_skip_only_their_unused_numeric_reads() { + for index in 0..2 { + let mut state = ReaderState::default(); + state.presence[index] = 0; + let scores = execute(&mut state, 2, 1); + let stop = COMPLETE + .iter() + .position(|entry| *entry == (index as u64, 1)) + .unwrap() + + 1; + assert_eq!(scores.has_pi_equivalent, 0); + assert_eq!(scores.pi_equivalent, 0.0); + assert_eq!(state.calls, trace(&COMPLETE[..stop], state.ratios)); + + let mut state = ReaderState::default(); + state.log_presence[index] = 0; + let scores = execute(&mut state, 2, 1); + let omitted = (index as u64, 6); + let mut expected = COMPLETE[..12].to_vec(); + expected.extend( + COMPLETE[12..] + .iter() + .copied() + .filter(|entry| *entry != omitted), + ); + assert_eq!(state.calls, trace(&expected, state.ratios)); + assert_eq!(scores.has_pi_equivalent, 1); + assert_eq!( + scores.pi_equivalent, + (state.ratios[1 - index].ln() / 2.0).exp() + ); + } +} + +/// The logarithm consumes a fresh getter value after its guard, rather than the +/// finite primitive captured during validation, including fresh domain errors. +#[test] +fn host_logs_use_fresh_values_after_their_presence_guards() { + let mut state = ReaderState { + log_ratios: Some([2.0, 8.0]), + ..ReaderState::default() + }; + let scores = execute(&mut state, 2, 1); + assert_eq!(state.calls, trace(&COMPLETE, state.ratios)); + assert_eq!(state.numeric_reads, [2, 2]); + assert_eq!(state.presence_reads, [2, 2]); + assert_eq!(scores.has_pi_equivalent, 1); + assert_eq!(scores.pi_equivalent, 4.0); + + for index in 0..2 { + let mut state = ReaderState { + log_ratios: Some([0.0, -2.0]), + reject: Some((index, 4)), + ..ReaderState::default() + }; + let scores = execute(&mut state, 2, 1); + let mut expected = COMPLETE[..14].to_vec(); + expected.push((0, 4)); + if index == 1 { + expected.extend_from_slice(&COMPLETE[14..]); + expected.push((1, 4)); + } + let mut expected_calls = trace(&expected, state.ratios); + for call in &mut expected_calls { + if call.field == 4 { + let operand = state.log_ratios.unwrap()[call.index as usize]; + call.operand_status = 0; + call.operand_bits = operand.to_bits(); + } + } + assert_eq!(state.calls, expected_calls); + assert_eq!(scores.has_pi_equivalent, 0); + assert_eq!(scores.pi_equivalent, 0.0); + } +} + +fn overflowing_log_trace(exponent: f64) -> Vec { + let mut fields = COMPLETE[..14].to_vec(); + fields.push((0, 4)); + fields.extend_from_slice(&COMPLETE[14..]); + fields.push((1, 4)); + let mut calls = trace(&fields, [0.0, -2.0]); + calls.push(Call { + index: 0, + field: 7, + benchmark: 0, + operand_status: 0, + operand_bits: exponent.to_bits(), + presence: 0, + name: Vec::new(), + }); + calls +} + +/// A finite overflowing exponent requests one host primitive after every log, +/// carrying the actual mean and preserving the host's returned double exactly. +#[test] +fn host_exponential_overflow_requests_its_operand_and_forwards_the_primitive() { + for returned in [0.0, -0.0, 42.5, f64::INFINITY, f64::NAN] { + let mut state = ReaderState { + ratios: [0.0, -2.0], + logarithms: [1200.0, 1800.0], + exponential: returned, + ..ReaderState::default() + }; + let scores = execute(&mut state, 2, 1); + assert_eq!(state.calls, overflowing_log_trace(1500.0)); + assert_eq!(scores.has_pi_equivalent, 1); + assert_eq!(scores.pi_equivalent.to_bits(), returned.to_bits()); + } +} + +/// An unavailable overflow primitive must suppress the score without extra host +/// calls, so a Python exception survives until its own callback bridge rethrows it. +#[test] +fn host_exponential_reader_failure_keeps_the_aggregate_unavailable() { + let mut state = ReaderState { + ratios: [0.0, -2.0], + logarithms: [1200.0, 1800.0], + reject: Some((0, 7)), + ..ReaderState::default() + }; + let scores = execute(&mut state, 2, 1); + assert_eq!(state.calls, overflowing_log_trace(1500.0)); + assert_eq!(scores.has_pi_equivalent, 0); + assert_eq!(scores.pi_equivalent, 0.0); +} + +/// Nonfinite exponents, finite underflow and raw-reader mode retain Rust exp +/// semantics; none may ask for Python's finite-overflow exception primitive. +#[test] +fn exponential_translation_is_only_for_finite_host_overflow() { + for fresh in [f64::INFINITY, f64::NAN] { + let mut state = ReaderState { + log_ratios: Some([fresh, 1.0]), + ..ReaderState::default() + }; + let scores = execute(&mut state, 2, 1); + assert_eq!(state.calls, trace(&COMPLETE, state.ratios)); + assert_eq!(scores.has_pi_equivalent, 1); + if fresh.is_nan() { + assert!(scores.pi_equivalent.is_nan()); + } else { + assert_eq!(scores.pi_equivalent, f64::INFINITY); + } + } + let raw_fields = [ + (0, 0), + (0, 1), + (0, 2), + (1, 0), + (1, 1), + (1, 2), + (0, 3), + (1, 3), + (0, 4), + (1, 4), + ]; + for host in [0, 1] { + for logarithm in [-1500.0, 1500.0] { + if host != 0 && logarithm > 0.0 { + continue; + } + let mut state = ReaderState { + ratios: [0.0, -2.0], + logarithms: [logarithm; 2], + ..ReaderState::default() + }; + let scores = execute_ids(&mut state, 2, host, [1, 2, 0]); + let expected_calls = if host == 0 { + trace(&raw_fields, state.ratios) + } else { + let mut calls = overflowing_log_trace(logarithm); + calls.pop(); + calls + }; + assert_eq!(state.calls, expected_calls); + assert_eq!(scores.has_pi_equivalent, 1); + assert_eq!(scores.pi_equivalent, logarithm.exp()); + } + } +} diff --git a/native/src/aggregation_ffi.rs b/native/src/aggregation_ffi.rs new file mode 100644 index 0000000..e495d21 --- /dev/null +++ b/native/src/aggregation_ffi.rs @@ -0,0 +1,173 @@ +use super::suite::CAggregatedScores; +use crate::{aggregate, AggregateOperator, AggregateValue, BENCHMARKS}; +use std::ffi::c_void; + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct CAggregateInput { + pub benchmark: u32, + pub status: u32, + pub pi_equivalent: f64, + pub has_pi_equivalent: u32, +} + +/// Aggregate caller-owned snapshots using the same rules as Rust SuiteResult. +#[no_mangle] +pub unsafe extern "C" fn benchset_aggregate( + inputs: *const CAggregateInput, + count: u64, + out: *mut CAggregatedScores, +) -> i32 { + if out.is_null() || (inputs.is_null() && count != 0) { + return 0; + } + // Only a complete suite needs reading; this also bounds the raw slice. + let equivalent = if count == BENCHMARKS.len() as u64 { + aggregate( + count as usize, + |index, expected| (*inputs.add(index)).benchmark == expected as u32, + |index| (*inputs.add(index)).status, + |index| { + let value = &*inputs.add(index); + (value.has_pi_equivalent != 0).then_some(value.pi_equivalent) + }, + ) + } else { + None + }; + write_scores(out, equivalent) +} + +unsafe fn write_scores(out: *mut CAggregatedScores, equivalent: Option) -> i32 { + out.write(CAggregatedScores { + pi_equivalent: equivalent.unwrap_or(0.0), + has_pi_equivalent: u32::from(equivalent.is_some()), + }); + 1 +} + +#[repr(C)] +pub struct CAggregateRequest { + pub value: CAggregateInput, + pub name: *const u8, + pub name_size: u64, +} + +pub type AggregateReader = + Option i32>; + +type ReadValue<'a> = dyn Fn(usize, u32, f64, &str) -> Option + 'a; + +struct ReadRatio<'a> { + index: usize, + number: f64, + host: bool, + read: &'a ReadValue<'a>, +} + +impl AggregateValue for ReadRatio<'_> { + fn number(&self) -> f64 { + self.number + } + + fn compare(&self, operator: AggregateOperator, operand: u32) -> Option { + (self.read)(self.index, operator as u32, f64::from(operand), "") + .map(|value| value.has_pi_equivalent != 0) + } + + fn logarithm(&self) -> Option { + let number = if self.host { + if (self.read)(self.index, 1, 0.0, "")?.has_pi_equivalent == 0 { + return Some(0.0); + } + (self.read)(self.index, 6, 0.0, "")?.pi_equivalent + } else { + self.number + }; + if number <= 0.0 { + (self.read)(self.index, 4, number, "").map(|value| value.pi_equivalent) + } else { + Some(number.ln()) + } + } + + fn exponential(&self, exponent: f64) -> Option { + let value = exponent.exp(); + if self.host && crate::finite(exponent) && value.is_infinite() { + (self.read)(self.index, 7, exponent, "").map(|value| value.pi_equivalent) + } else { + Some(value) + } + } +} + +/// Request host primitives only in the order selected by the shared calculator. +/// Host mode preserves positional tuple equality, including its common prefix. +#[no_mangle] +pub unsafe extern "C" fn benchset_aggregate_read( + benchmarks: *const u32, + count: u64, + reader: AggregateReader, + user: *mut c_void, + out: *mut CAggregatedScores, + host_names: u32, +) -> i32 { + if out.is_null() || (benchmarks.is_null() && count != 0) || reader.is_none() { + return 0; + } + if host_names == 0 && count != BENCHMARKS.len() as u64 { + return write_scores(out, None); + } + let read = |index: usize, field, operand, name: &str| { + let mut request = CAggregateRequest { + value: CAggregateInput { + benchmark: 0, + status: 0, + pi_equivalent: operand, + has_pi_equivalent: 0, + }, + name: name.as_ptr(), + name_size: name.len() as u64, + }; + (reader.unwrap()(index as u64, field, &mut request, user) != 0).then_some(request.value) + }; + write_scores( + out, + aggregate( + count as usize, + |index, expected| { + if host_names == 0 { + *benchmarks.add(index) == expected as u32 + } else { + read(index, 5, 0.0, expected.name()) + .is_some_and(|value| value.has_pi_equivalent != 0) + } + }, + |index| read(index, 0, 0.0, "").map_or(u32::MAX, |value| value.status), + |index| { + let value = read(index, 1, 0.0, "")?; + if value.has_pi_equivalent == 0 { + return None; + } + Some(ReadRatio { + index, + number: if host_names != 0 { + read(index, 6, 0.0, "")?.pi_equivalent + } else { + value.pi_equivalent + }, + host: host_names != 0, + read: &read, + }) + }, + ), + ) +} + +#[cfg(test)] +#[path = "aggregation_ffi_tests.rs"] +mod tests; + +#[cfg(test)] +#[path = "aggregate_protocol_tests.rs"] +mod protocol_tests; diff --git a/native/src/aggregation_ffi_tests.rs b/native/src/aggregation_ffi_tests.rs new file mode 100644 index 0000000..5b1712f --- /dev/null +++ b/native/src/aggregation_ffi_tests.rs @@ -0,0 +1,279 @@ +use super::*; +use crate::{Benchmark, Status}; +use std::ptr; + +fn input(benchmark: Benchmark, ratio: f64) -> CAggregateInput { + CAggregateInput { + benchmark: benchmark as u32, + status: Status::Completed as u32, + pi_equivalent: ratio, + has_pi_equivalent: 1, + } +} + +struct ReaderState { + values: [CAggregateInput; 2], + calls: Vec<(u64, u32)>, + reject: Option<(u64, u32)>, + accepted: i32, +} + +unsafe extern "C" fn read( + index: u64, + field: u32, + out: *mut CAggregateRequest, + user: *mut c_void, +) -> i32 { + let state = &mut *user.cast::(); + state.calls.push((index, field)); + if state.reject == Some((index, field)) || index >= 2 || field > 3 { + return 0; + } + let mut value = state.values[index as usize]; + if field >= 2 { + let operand = f64::from((*out).value.status); + let matches = if field == 2 { + value.pi_equivalent >= operand + } else { + value.pi_equivalent == operand + }; + value.has_pi_equivalent = if matches { value.has_pi_equivalent } else { 0 }; + } + (*out).value = value; + state.accepted +} + +/// The core requests status/ratio/operators in order, stops before unused host fields, +/// and treats failed readers like unavailable values without overriding identities. +/// All nonzero reader returns and presence flags, including negative returns, are accepted. +#[test] +fn aggregate_reader_preserves_lazy_order_and_failures() { + unsafe { + let names = [Benchmark::SingleCore as u32, Benchmark::MultiCore as u32]; + let expected = [ + (0, 0), + (0, 1), + (0, 2), + (1, 0), + (1, 1), + (1, 2), + (0, 3), + (1, 3), + ]; + for rejected in 0..=expected.len() { + let mut state = ReaderState { + values: [ + input(Benchmark::MultiCore, 0.5), + input(Benchmark::SingleCore, 2.0), + ], + calls: Vec::new(), + reject: expected.get(rejected).copied(), + accepted: -1, + }; + state.values[0].has_pi_equivalent = 7; + state.values[1].has_pi_equivalent = u32::MAX; + let mut out = CAggregatedScores { + pi_equivalent: 123.0, + has_pi_equivalent: 7, + }; + assert_eq!( + benchset_aggregate_read( + names.as_ptr(), + 2, + Some(read), + (&mut state as *mut ReaderState).cast(), + &mut out, + 0 + ), + 1 + ); + assert_eq!(state.calls, expected[..(rejected + 1).min(expected.len())]); + assert_eq!(out.has_pi_equivalent, u32::from(rejected == expected.len())); + assert_eq!( + out.pi_equivalent, + if rejected == expected.len() { 1.0 } else { 0.0 } + ); + } + for field in 0..=1 { + for index in 0..2 { + let mut state = ReaderState { + values: [ + input(Benchmark::SingleCore, 0.5), + input(Benchmark::MultiCore, 2.0), + ], + calls: Vec::new(), + reject: None, + accepted: 7, + }; + if field == 0 { + state.values[index].status = Status::Cancelled as u32; + } else { + state.values[index].has_pi_equivalent = 0; + } + let mut out = CAggregatedScores { + pi_equivalent: 123.0, + has_pi_equivalent: 7, + }; + assert_eq!( + benchset_aggregate_read( + names.as_ptr(), + 2, + Some(read), + (&mut state as *mut ReaderState).cast(), + &mut out, + 0 + ), + 1 + ); + assert_eq!(out.has_pi_equivalent, 0); + let stop = expected + .iter() + .position(|entry| *entry == (index as u64, field)) + .unwrap() + + 1; + assert_eq!(state.calls, expected[..stop]); + } + } + } +} + +/// Malformed pointers/readers leave output intact; incomplete or unordered IDs +/// never invoke host code, including maximum counts and a valid empty input. +#[test] +fn aggregate_reader_pointer_count_and_identity_contract() { + unsafe { + let names = [Benchmark::MultiCore as u32, Benchmark::SingleCore as u32]; + let mut out = CAggregatedScores { + pi_equivalent: 123.0, + has_pi_equivalent: 7, + }; + for count in [0, 1, 2, 3, u64::MAX] { + assert_eq!( + benchset_aggregate_read(names.as_ptr(), count, None, ptr::null_mut(), &mut out, 0), + 0 + ); + assert_eq!( + benchset_aggregate_read( + names.as_ptr(), + count, + Some(read), + ptr::null_mut(), + ptr::null_mut(), + 0 + ), + 0 + ); + assert_eq!(out.pi_equivalent, 123.0); + assert_eq!(out.has_pi_equivalent, 7); + } + for count in [1, 2, 3, u64::MAX] { + assert_eq!( + benchset_aggregate_read( + ptr::null(), + count, + Some(read), + ptr::null_mut(), + &mut out, + 0 + ), + 0 + ); + assert_eq!(out.pi_equivalent, 123.0); + } + for count in [0, 1, 2, 3, u64::MAX] { + assert_eq!( + benchset_aggregate_read( + names.as_ptr(), + count, + Some(read), + ptr::null_mut(), + &mut out, + 0 + ), + 1 + ); + assert_eq!(out.has_pi_equivalent, 0); + } + assert_eq!( + benchset_aggregate_read(ptr::null(), 0, Some(read), ptr::null_mut(), &mut out, 0), + 1 + ); + } +} + +/// Keeps empty/incomplete counts safe, bounds raw input reads and leaves output intact on invalid pointers. +#[test] +fn aggregate_pointer_and_count_contract() { + unsafe { + #[repr(C)] + struct Guarded { + before: u64, + out: CAggregatedScores, + after: u64, + } + let mut guarded = Guarded { + before: 0x123456789abcdef0, + out: CAggregatedScores { + pi_equivalent: 123.0, + has_pi_equivalent: 7, + }, + after: 0xfedcba9876543210, + }; + let out = &mut guarded.out; + for count in [1, 2, 3, u64::MAX] { + assert_eq!(benchset_aggregate(ptr::null(), count, out), 0); + assert_eq!(out.pi_equivalent, 123.0); + assert_eq!(out.has_pi_equivalent, 7); + } + let one = input(Benchmark::SingleCore, 0.5); + for count in [0, 1, 2, 3, u64::MAX] { + assert_eq!(benchset_aggregate(&one, count, ptr::null_mut()), 0); + } + for count in [0, 1, 3, u64::MAX] { + assert_eq!(benchset_aggregate(&one, count, out), 1); + assert_eq!(out.has_pi_equivalent, 0); + } + assert_eq!(benchset_aggregate(ptr::null(), 0, out), 1); + assert_eq!(out.has_pi_equivalent, 0); + assert_eq!(guarded.before, 0x123456789abcdef0); + assert_eq!(guarded.after, 0xfedcba9876543210); + } +} + +/// Preserves every nonzero presence flag and rejects raw unknown IDs/statuses or missing ratios at either position. +/// Shared core tests cover numerical boundaries; this matrix guards the C-to-Rust field interpretation. +#[test] +fn aggregate_ffi_preserves_raw_fields_and_presence() { + unsafe { + let original = [ + input(Benchmark::SingleCore, 0.5), + input(Benchmark::MultiCore, 8.0), + ]; + let mut out = CAggregatedScores { + pi_equivalent: 0.0, + has_pi_equivalent: 0, + }; + for flag in [1, 7, u32::MAX] { + let inputs = original.map(|mut value| { + value.has_pi_equivalent = flag; + value + }); + assert_eq!(benchset_aggregate(inputs.as_ptr(), 2, &mut out), 1); + assert_eq!(out.has_pi_equivalent, 1); + assert!((out.pi_equivalent - 2.0).abs() < 1e-15); + } + for position in 0..2 { + for invalid in 0..4 { + let mut inputs = original; + match invalid { + 0 => inputs[position].benchmark = 0, + 1 => inputs[position].benchmark = u32::MAX, + 2 => inputs[position].status = u32::MAX, + _ => inputs[position].has_pi_equivalent = 0, + } + assert_eq!(benchset_aggregate(inputs.as_ptr(), 2, &mut out), 1); + assert_eq!(out.has_pi_equivalent, 0); + } + } + } +} diff --git a/native/src/catalog.rs b/native/src/catalog.rs new file mode 100644 index 0000000..527f9d7 --- /dev/null +++ b/native/src/catalog.rs @@ -0,0 +1,33 @@ +// Shared public benchmark identities, suite order and measurement defaults. + +pub const DEFAULT_WARMUP_SECONDS: f64 = 120.0; +pub const DEFAULT_MEASUREMENT_SECONDS: f64 = 60.0; +pub const BENCHMARKS: [Benchmark; 2] = [Benchmark::SingleCore, Benchmark::MultiCore]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u32)] +pub enum Benchmark { + SingleCore = 1, + MultiCore = 2, +} + +impl Benchmark { + pub fn name(self) -> &'static str { + match self { + Self::SingleCore => "single_core", + Self::MultiCore => "multi_core", + } + } + + pub(crate) fn from_id(id: u32) -> Option { + BENCHMARKS + .into_iter() + .find(|benchmark| *benchmark as u32 == id) + } + + pub(crate) fn from_name(name: &str) -> Option { + BENCHMARKS + .into_iter() + .find(|benchmark| benchmark.name() == name) + } +} diff --git a/native/src/config_ffi.rs b/native/src/config_ffi.rs new file mode 100644 index 0000000..569e9cb --- /dev/null +++ b/native/src/config_ffi.rs @@ -0,0 +1,79 @@ +use super::*; + +pub(super) fn duration_error(warmup: f64, measurement: f64) -> u32 { + if !crate::nonnegative(warmup) { + 2 + } else if !crate::positive(measurement) { + 3 + } else { + 0 + } +} + +/// Length-delimited UTF-8 keeps unknown names, including embedded NUL, unknown. +#[no_mangle] +pub unsafe extern "C" fn benchset_benchmark_id(name: *const u8, size: u64) -> u32 { + if name.is_null() || size > isize::MAX as u64 { + return 0; + } + std::str::from_utf8(std::slice::from_raw_parts(name, size as usize)) + .ok() + .and_then(Benchmark::from_name) + .map_or(0, |benchmark| benchmark as u32) +} + +pub type NameMatcher = Option i32>; + +/// The catalog selects comparison order; execution retains the legacy final fallback. +#[no_mangle] +pub unsafe extern "C" fn benchset_benchmark_match( + matcher: NameMatcher, + user: *mut c_void, + execution: u32, +) -> u32 { + let Some(matcher) = matcher else { return 0 }; + for (index, benchmark) in crate::BENCHMARKS.into_iter().enumerate() { + if execution != 0 && index + 1 == crate::BENCHMARKS.len() { + return benchmark as u32; + } + let name = benchmark.name().as_bytes(); + match matcher(name.as_ptr(), name.len() as u64, user) { + value if value > 0 => return benchmark as u32, + 0 => {} + _ => return 0, + } + } + 0 +} + +#[no_mangle] +pub extern "C" fn benchset_duration_error(warmup: f64, measurement: f64) -> u32 { + duration_error(warmup, measurement) +} + +#[no_mangle] +pub extern "C" fn benchset_validation_error(benchmark: u32, warmup: f64, measurement: f64) -> u32 { + if Benchmark::from_id(benchmark).is_none() { + 1 + } else { + duration_error(warmup, measurement) + } +} + +/// Messages belong to the core; bindings only select their parameter spelling. +#[no_mangle] +pub extern "C" fn benchset_validation_message(error: u32, cli: u32) -> *const c_char { + let message: &[u8] = match (error, cli != 0) { + (1, _) => b"unknown benchmark\0", + (2, false) => b"warmup_seconds must be finite and nonnegative\0", + (3, false) => b"measurement_seconds must be finite and positive\0", + (2, true) => b"--warmup must be finite and nonnegative\0", + (3, true) => b"--duration must be finite and positive\0", + _ => b"\0", + }; + message.as_ptr().cast() +} + +#[cfg(test)] +#[path = "config_ffi_tests.rs"] +mod tests; diff --git a/native/src/config_ffi_tests.rs b/native/src/config_ffi_tests.rs new file mode 100644 index 0000000..6d3d2eb --- /dev/null +++ b/native/src/config_ffi_tests.rs @@ -0,0 +1,296 @@ +use super::*; +use std::ffi::CStr; + +/// Native validation classifies stored subnormal bits, even with FPU flushing. +/// Exercise both signs, subnormal boundaries and signed zeros through duration +/// and aggregate entry points. Restore the thread's FPU state even on failure; +/// black_box prevents constant folding from hiding comparisons affected by FTZ. +#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] +#[test] +fn stored_numeric_validation_ignores_fpu_flushing() { + use super::super::aggregation::{benchset_aggregate, CAggregateInput}; + use super::super::suite::CAggregatedScores; + use std::arch::asm; + use std::hint::black_box; + + unsafe fn control() -> u64 { + #[cfg(target_arch = "aarch64")] + { + let value; + asm!("mrs {}, fpcr", out(reg) value); + value + } + #[cfg(target_arch = "x86_64")] + { + let mut value = 0_u64; + asm!("stmxcsr [{}]", in(reg) &mut value); + value + } + } + unsafe fn set_control(value: u64) { + #[cfg(target_arch = "aarch64")] + asm!("msr fpcr, {}", in(reg) value); + #[cfg(target_arch = "x86_64")] + asm!("ldmxcsr [{}]", in(reg) &value); + } + struct Restore { + control: u64, + #[cfg(target_arch = "aarch64")] + flags: u64, + } + impl Drop for Restore { + fn drop(&mut self) { + unsafe { + set_control(self.control); + #[cfg(target_arch = "aarch64")] + asm!("msr fpsr, {}", in(reg) self.flags); + } + } + } + unsafe { + #[cfg(target_arch = "aarch64")] + let flags; + #[cfg(target_arch = "aarch64")] + asm!("mrs {}, fpsr", out(reg) flags); + let saved = Restore { + control: control(), + #[cfg(target_arch = "aarch64")] + flags, + }; + #[cfg(target_arch = "aarch64")] + let flush = 1 << 24; + #[cfg(target_arch = "x86_64")] + let flush = { + // DAZ is optional: query the supported MXCSR bits before setting it. + #[repr(align(16))] + struct State([u8; 512]); + let mut state = State([0; 512]); + asm!("fxsave [{}]", in(reg) state.0.as_mut_ptr()); + let mask = u32::from_le_bytes(state.0[28..32].try_into().unwrap()); + (1 << 15) | u64::from(mask & (1 << 6)) + }; + for enabled in [false, true] { + let mode = (saved.control & !flush) | if enabled { flush } else { 0 }; + set_control(mode); + assert_eq!(control() & flush, mode & flush); + for (bits, warmup_error, measurement_error, available, zero) in [ + (0x8000_0000_0000_0001, 2, 3, 0, false), + (0x800f_ffff_ffff_ffff, 2, 3, 0, false), + (0x0000_0000_0000_0001, 0, 0, 1, false), + (0x000f_ffff_ffff_ffff, 0, 0, 1, false), + (0x8000_0000_0000_0000, 0, 3, 1, true), + (0x0000_0000_0000_0000, 0, 3, 1, true), + ] { + let value = black_box(f64::from_bits(bits)); + assert_eq!(benchset_duration_error(value, 1.0), warmup_error); + assert_eq!(benchset_duration_error(0.0, value), measurement_error); + assert_eq!(crate::zero(value), zero); + for position in 0..2 { + let mut inputs = crate::BENCHMARKS.map(|benchmark| CAggregateInput { + benchmark: benchmark as u32, + status: 0, + pi_equivalent: 1.0, + has_pi_equivalent: 1, + }); + inputs[position].pi_equivalent = value; + let mut out = CAggregatedScores { + pi_equivalent: -1.0, + has_pi_equivalent: 7, + }; + assert_eq!(benchset_aggregate(inputs.as_ptr(), 2, &mut out), 1); + assert_eq!(out.has_pi_equivalent, available); + } + } + } + } +} + +/// Host equality follows catalog order, accepts positive matches, stops on errors, +/// and execution uses its final fallback without requesting another comparison. +#[test] +fn host_name_matching_preserves_comparison_order_and_fallback() { + struct State { + answers: [i32; 2], + names: Vec, + } + unsafe extern "C" fn matches(name: *const u8, size: u64, user: *mut c_void) -> i32 { + let state = &mut *user.cast::(); + let answer = state.answers[state.names.len()]; + state.names.push( + std::str::from_utf8(std::slice::from_raw_parts(name, size as usize)) + .unwrap() + .to_owned(), + ); + answer + } + unsafe { + assert_eq!(benchset_benchmark_match(None, ptr::null_mut(), 0), 0); + for (answers, execution, expected, calls) in [ + ([0, 7], 0, 2, 2), + ([0, -1], 1, 2, 1), + ([7, -1], 0, 1, 1), + ([-1, 7], 0, 0, 1), + ([0, 0], 0, 0, 2), + ] { + let mut state = State { + answers, + names: Vec::new(), + }; + assert_eq!( + benchset_benchmark_match( + Some(matches), + (&mut state as *mut State).cast(), + execution + ), + expected + ); + let expected_names = crate::BENCHMARKS.map(|benchmark| benchmark.name()); + assert_eq!(state.names, expected_names[..calls]); + } + } +} + +/// Name lookup uses every UTF-8 byte and never treats embedded NUL as terminator. +#[test] +fn names_are_length_delimited_and_catalog_driven() { + unsafe { + for benchmark in crate::BENCHMARKS { + let name = benchmark.name().as_bytes(); + assert_eq!( + benchset_benchmark_id(name.as_ptr(), name.len() as u64), + benchmark as u32 + ); + assert_eq!(benchset_benchmark_id(name.as_ptr(), 0), 0); + } + for name in [ + b"single_core\0suffix".as_slice(), + "неизвестный".as_bytes(), + &[0xff], + b"SINGLE_CORE", + b" single_core", + b"single_core\n", + b"single-core", + b"all", + b"single_core_suffix", + b"prefix_single_core", + ] { + assert_eq!(benchset_benchmark_id(name.as_ptr(), name.len() as u64), 0); + } + assert_eq!(benchset_benchmark_id(ptr::null(), 0), 0); + assert_eq!(benchset_benchmark_id(ptr::null(), 1), 0); + assert_eq!(benchset_benchmark_id(b"x".as_ptr(), u64::MAX), 0); + } +} + +/// Bindings share error precedence and exact diagnostics, including NaN and signed zero. +#[test] +fn validation_rules_and_messages_preserve_each_interface() { + assert_eq!(benchset_validation_error(999, -1.0, 0.0), 1); + for warmup in [-1.0, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + assert_eq!(benchset_validation_error(1, warmup, 0.0), 2); + assert_eq!(benchset_duration_error(warmup, 0.0), 2); + } + for measurement in [-1.0, -0.0, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + assert_eq!(benchset_validation_error(2, -0.0, measurement), 3); + } + for (warmup, measurement) in [(0.0, 1.0), (-0.0, 1e-12), (f64::MAX, f64::MAX)] { + assert_eq!(benchset_validation_error(1, warmup, measurement), 0); + } + for (code, api, cli) in [ + (0, "", ""), + (1, "unknown benchmark", "unknown benchmark"), + ( + 2, + "warmup_seconds must be finite and nonnegative", + "--warmup must be finite and nonnegative", + ), + ( + 3, + "measurement_seconds must be finite and positive", + "--duration must be finite and positive", + ), + (999, "", ""), + ] { + unsafe { + assert_eq!( + CStr::from_ptr(benchset_validation_message(code, 0)) + .to_str() + .unwrap(), + api + ); + for flag in [1, 7, u32::MAX] { + assert_eq!( + CStr::from_ptr(benchset_validation_message(code, flag)) + .to_str() + .unwrap(), + cli + ); + } + } + } +} + +/// Refactoring shared rules must not replace the original C diagnostic vocabulary +/// with Python/CLI parameter names, nor hide the requested invalid double values. +#[test] +fn existing_c_calls_keep_their_exact_validation_diagnostics() { + unsafe { + for (benchmark, reserved, warmup, measurement, expected) in [ + (999, 0, -1.0, 0.0, "unknown benchmark"), + (1, 1, -1.0, 0.0, "reserved config field must be zero"), + ( + 1, + 0, + -1.0, + 1.0, + "durations must be finite: warmup >= 0, measurement > 0", + ), + ( + 2, + 0, + f64::NAN, + f64::INFINITY, + "durations must be finite: warmup >= 0, measurement > 0", + ), + ( + 2, + 0, + 0.0, + -0.0, + "durations must be finite: warmup >= 0, measurement > 0", + ), + ( + 1, + 0, + f64::MAX, + 1.0, + "durations must be finite: warmup >= 0, measurement > 0", + ), + (1, 0, 0.0, 1e-12, "measurement duration must be positive"), + ] { + let handle = benchset_create(); + let config = Config { + abi_version: ABI_VERSION, + struct_size: std::mem::size_of::() as u32, + benchmark, + reserved, + warmup_seconds: warmup, + measurement_seconds: measurement, + }; + assert_eq!( + benchset_run(handle, &config, None, ptr::null_mut()), + Status::Failed as u32 + ); + assert_eq!( + CStr::from_ptr(benchset_reason(handle)).to_str().unwrap(), + expected + ); + let mut view = std::mem::MaybeUninit::::uninit(); + assert_eq!(benchset_result(handle, view.as_mut_ptr()), 1); + let view = view.assume_init(); + assert_eq!(view.warmup_seconds.to_bits(), warmup.to_bits()); + assert_eq!(view.measurement_seconds.to_bits(), measurement.to_bits()); + benchset_free(handle); + } + } +} diff --git a/native/src/configured_suite_tests.rs b/native/src/configured_suite_tests.rs new file mode 100644 index 0000000..c504d23 --- /dev/null +++ b/native/src/configured_suite_tests.rs @@ -0,0 +1,350 @@ +use super::*; +use std::mem::{size_of, MaybeUninit}; + +struct Suite(*mut SuiteHandle); +impl Drop for Suite { + fn drop(&mut self) { + unsafe { benchset_suite_free(self.0) } + } +} + +struct Events { + suite: *mut SuiteHandle, + calls: Vec<(char, u32)>, + results: Vec<(u32, u32, f64, f64)>, + abort_at: u32, + invalid_at: u32, + stop: u32, + cancel_after: u32, + caller: std::thread::ThreadId, +} + +impl Events { + fn new(suite: *mut SuiteHandle) -> Self { + Self { + suite, + calls: Vec::new(), + results: Vec::new(), + abort_at: 0, + invalid_at: 0, + stop: 0, + cancel_after: 0, + caller: std::thread::current().id(), + } + } + + fn user(&mut self) -> *mut c_void { + (self as *mut Self).cast() + } +} + +fn config() -> Config { + Config { + abi_version: ABI_VERSION, + struct_size: size_of::() as u32, + benchmark: 999, + reserved: 0, + warmup_seconds: 10.0, + measurement_seconds: 20.0, + } +} + +unsafe extern "C" fn configure(benchmark: u32, config: *mut Config, user: *mut c_void) -> u32 { + let events = &mut *user.cast::(); + assert_eq!(std::thread::current().id(), events.caller); + events.calls.push(('c', benchmark)); + if events.abort_at == benchmark { + return 0; + } + let config = &mut *config; + // Every callback receives a fresh copy; neither an earlier callback's + // changes nor the suite's benchmark override may mutate the caller's config. + assert_eq!(config.benchmark, 999); + assert_eq!( + (config.warmup_seconds, config.measurement_seconds), + (10.0, 20.0) + ); + config.benchmark = 1234; + config.warmup_seconds = f64::from(benchmark) / 4.0; + config.measurement_seconds = if events.invalid_at == benchmark { + 0.0 + } else { + f64::from(benchmark) / 2.0 + }; + u32::MAX +} + +unsafe extern "C" fn observed(benchmark: u32, handle: *const Handle, user: *mut c_void) -> u32 { + let events = &mut *user.cast::(); + assert_eq!(std::thread::current().id(), events.caller); + let mut result = MaybeUninit::::uninit(); + assert_eq!( + benchset_result_v2(handle, result.as_mut_ptr(), size_of::() as u32), + 1 + ); + let result = result.assume_init().base; + assert_eq!(result.benchmark, benchmark); + events.calls.push(('r', benchmark)); + events.results.push(( + benchmark, + result.status, + result.warmup_seconds, + result.measurement_seconds, + )); + if events.cancel_after == benchmark { + benchset_suite_cancel(events.suite); + } + events.stop +} + +fn completed( + benchmark: Benchmark, + options: Options, + cancellation: &Cancellation, +) -> BenchmarkResult { + let mut result = BenchmarkResult::failed(benchmark, options, String::new()); + result.status = if cancellation.is_cancelled() { + Status::Cancelled + } else { + Status::Completed + }; + result.pi_equivalent = (result.status == Status::Completed).then_some(1.0); + result +} + +unsafe fn summary(suite: *const SuiteHandle) -> CSuiteResult { + let mut result = MaybeUninit::uninit(); + assert_eq!( + benchset_suite_result(suite, result.as_mut_ptr(), size_of::() as u32), + 1 + ); + result.assume_init() +} + +/// Config callbacks run in catalog order between prior publication and later +/// execution, using independent copies and preserving the Rust benchmark ID. +#[test] +fn configured_windows_reach_each_child_without_mutating_the_original() { + unsafe { + let suite = Suite(benchset_suite_create()); + let mut events = Events::new(suite.0); + let config = config(); + let mut runs = Vec::new(); + assert_eq!( + execute_suite( + suite.0, + &config, + None, + Some(observed), + Some(configure), + events.user(), + &mut |benchmark, options, cancellation, _| { + runs.push(( + benchmark, + options.warmup.as_secs_f64(), + options.measurement.as_secs_f64(), + )); + completed(benchmark, options, cancellation) + } + ), + Status::Completed as u32 + ); + assert_eq!( + runs, + [ + (Benchmark::SingleCore, 0.25, 0.5), + (Benchmark::MultiCore, 0.5, 1.0) + ] + ); + assert_eq!(events.calls, [('c', 1), ('r', 1), ('c', 2), ('r', 2)]); + assert_eq!(events.results, [(1, 0, 0.25, 0.5), (2, 0, 0.5, 1.0)]); + assert_eq!( + ( + config.benchmark, + config.warmup_seconds, + config.measurement_seconds + ), + (999, 10.0, 20.0) + ); + let result = summary(suite.0); + assert_eq!(result.result_count, 2); + assert_eq!(result.aggregated_scores.has_pi_equivalent, 1); + } +} + +/// Aborting before any child publishes an available empty suite. Aborting +/// later retains the completed prefix and reports failure without inventing a child. +#[test] +fn configuration_abort_preserves_empty_or_completed_prefix() { + unsafe { + for abort_at in [1, 2] { + let suite = Suite(benchset_suite_create()); + let mut events = Events::new(suite.0); + events.abort_at = abort_at; + assert_eq!( + execute_suite( + suite.0, + &config(), + None, + Some(observed), + Some(configure), + events.user(), + &mut |b, o, c, _| completed(b, o, c) + ), + Status::Failed as u32 + ); + assert_eq!(events.results.len(), (abort_at - 1) as usize); + let result = summary(suite.0); + assert_eq!(result.result_count, u64::from(abort_at - 1)); + assert_eq!(result.aggregated_scores.has_pi_equivalent, 0); + assert!(benchset_suite_reason(suite.0, result.result_count).is_null()); + // A broken single-use guard must fail quickly rather than start + // the real benchmark during this regression check. + let mut repeated = config(); + repeated.reserved = 1; + assert_eq!( + benchset_run_all_configured(suite.0, &repeated, None, None, None, ptr::null_mut()), + Status::Failed as u32 + ); + let preserved = summary(suite.0); + assert_eq!(preserved.result_count, result.result_count); + if preserved.result_count != 0 { + assert_eq!( + (*preserved.results).base.base.status, + Status::Completed as u32 + ); + } + } + } +} + +/// A callback may provide invalid numeric windows, but shared execute validation +/// must publish the actual failed child and never call its benchmark runner. +#[test] +fn invalid_reconfigured_windows_publish_failure_and_stop() { + unsafe { + for invalid_at in [1, 2] { + let suite = Suite(benchset_suite_create()); + let mut events = Events::new(suite.0); + events.invalid_at = invalid_at; + let mut runs = 0; + assert_eq!( + execute_suite( + suite.0, + &config(), + None, + Some(observed), + Some(configure), + events.user(), + &mut |b, o, c, _| { + runs += 1; + completed(b, o, c) + } + ), + Status::Failed as u32 + ); + assert_eq!(runs, invalid_at - 1); + assert_eq!(events.results.len(), invalid_at as usize); + assert_eq!(events.results.last().unwrap().1, Status::Failed as u32); + assert_eq!(events.results.last().unwrap().3, 0.0); + assert_eq!(summary(suite.0).aggregated_scores.has_pi_equivalent, 0); + } + } +} + +/// Observer stops preserve their legacy final status, while cancellation remains +/// sticky and still requests the next child's configuration before execution. +#[test] +fn observer_stop_and_cancellation_retain_their_existing_semantics() { + unsafe { + for stop in [0, 7] { + let suite = Suite(benchset_suite_create()); + let mut events = Events::new(suite.0); + events.stop = stop; + events.cancel_after = 1; + let status = execute_suite( + suite.0, + &config(), + None, + Some(observed), + Some(configure), + events.user(), + &mut |b, o, c, _| completed(b, o, c), + ); + assert_eq!( + status, + if stop == 0 { + Status::Cancelled + } else { + Status::Completed + } as u32 + ); + assert_eq!(events.results.len(), if stop == 0 { 2 } else { 1 }); + assert_eq!(events.calls.len(), if stop == 0 { 4 } else { 2 }); + assert_eq!(summary(suite.0).aggregated_scores.has_pi_equivalent, 0); + } + } +} + +/// The additive entrypoint must retain NULL/bad-prefix failures without calling +/// config callbacks or reading a whole incompatible structure. +#[test] +fn invalid_prefixes_skip_configuration_and_keep_normal_failed_results() { + unsafe { + assert_eq!( + benchset_run_all_configured( + ptr::null_mut(), + ptr::null(), + None, + None, + None, + ptr::null_mut() + ), + Status::Failed as u32 + ); + let bad_prefix = [ABI_VERSION, 8]; + for config in [ptr::null(), bad_prefix.as_ptr().cast::()] { + let suite = Suite(benchset_suite_create()); + let mut events = Events::new(suite.0); + assert_eq!( + benchset_run_all_configured( + suite.0, + config, + None, + Some(observed), + Some(configure), + events.user() + ), + Status::Failed as u32 + ); + assert_eq!(events.calls, [('r', 1)]); + assert_eq!(events.results.len(), 1); + assert_eq!(events.results[0].1, Status::Failed as u32); + } + } +} + +/// The exported wrapper forwards configuration and observer callbacks before a +/// pre-cancelled run, without requiring timed work or worker initialization. +#[test] +fn public_entrypoint_forwards_configuration_and_observation() { + let _serial = crate::tests::TEST_LOCK.lock().unwrap(); + unsafe { + let suite = Suite(benchset_suite_create()); + let mut events = Events::new(suite.0); + benchset_suite_cancel(suite.0); + assert_eq!( + benchset_run_all_configured( + suite.0, + &config(), + None, + Some(observed), + Some(configure), + events.user() + ), + Status::Cancelled as u32 + ); + assert_eq!(events.calls, [('c', 1), ('r', 1)]); + assert_eq!(events.results, [(1, Status::Cancelled as u32, 0.25, 0.5)]); + } +} diff --git a/native/src/ffi.rs b/native/src/ffi.rs index 7d3efea..b5b0362 100644 --- a/native/src/ffi.rs +++ b/native/src/ffi.rs @@ -14,6 +14,12 @@ const ABI_VERSION: u32 = 1; #[path = "suite_ffi.rs"] mod suite; +#[path = "aggregation_ffi.rs"] +mod aggregation; + +#[path = "config_ffi.rs"] +mod config; + #[repr(C)] #[derive(Clone, Copy)] pub struct Config { @@ -226,6 +232,22 @@ pub extern "C" fn benchset_version() -> *const c_char { concat!(env!("CARGO_PKG_VERSION"), "\0").as_ptr().cast() } +pub type DoubleReader = Option f64>; + +/// Copy one caller-converted double without floating arithmetic or classification. +#[no_mangle] +pub unsafe extern "C" fn benchset_copy_double( + reader: DoubleReader, + user: *mut c_void, + out: *mut f64, +) -> i32 { + let Some(reader) = reader.filter(|_| !out.is_null()) else { + return 0; + }; + out.write(reader(user)); + 1 +} + #[no_mangle] pub unsafe extern "C" fn benchset_default_config(out: *mut Config) -> i32 { if out.is_null() { @@ -304,9 +326,8 @@ unsafe fn execute( } let config = config.read(); requested = Some((config.warmup_seconds, config.measurement_seconds)); - benchmark = match config.benchmark { - 1 => Benchmark::SingleCore, - 2 => Benchmark::MultiCore, + benchmark = match Benchmark::from_id(config.benchmark) { + Some(value) => value, _ => return BenchmarkResult::failed(benchmark, options, "unknown benchmark".into()), }; if config.reserved != 0 { @@ -316,10 +337,16 @@ unsafe fn execute( "reserved config field must be zero".into(), ); } - let warmup = Duration::try_from_secs_f64(config.warmup_seconds); - let measurement = Duration::try_from_secs_f64(config.measurement_seconds); - options = match (warmup, measurement) { - (Ok(warmup), Ok(measurement)) if config.measurement_seconds > 0.0 => Options { + let durations = + (self::config::duration_error(config.warmup_seconds, config.measurement_seconds) == 0) + .then(|| { + ( + Duration::try_from_secs_f64(config.warmup_seconds), + Duration::try_from_secs_f64(config.measurement_seconds), + ) + }); + options = match durations { + Some((Ok(warmup), Ok(measurement))) => Options { warmup, measurement, }, diff --git a/native/src/ffi_tests.rs b/native/src/ffi_tests.rs index 25de0ae..5ecefb3 100644 --- a/native/src/ffi_tests.rs +++ b/native/src/ffi_tests.rs @@ -1,4 +1,96 @@ use super::*; + +/// A converted double is copied once, preserving bits, borrowed input and output bounds. +/// Null readers/outputs must not invoke the callback or change the destination; +/// an aliased input/output verifies that reading finishes before writing starts. +#[test] +fn double_copy_preserves_bits_and_borrowed_ownership() { + #[repr(C)] + struct Input { + bits: u64, + calls: usize, + } + unsafe extern "C" fn read(user: *mut c_void) -> f64 { + let input = &mut *user.cast::(); + input.calls += 1; + let bits = input.bits; + input.bits = bits.wrapping_add(1); + f64::from_bits(bits) + } + static NULL_CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + unsafe extern "C" fn read_null(user: *mut c_void) -> f64 { + assert!(user.is_null()); + NULL_CALLS.fetch_add(1, Ordering::Relaxed); + -0.0 + } + unsafe { + let mut input = Input { bits: 1, calls: 0 }; + let mut untouched = 42.0; + assert_eq!( + benchset_copy_double(None, ptr::null_mut(), &mut untouched), + 0 + ); + assert_eq!(untouched, 42.0); + assert_eq!( + benchset_copy_double(None, ptr::null_mut(), ptr::null_mut()), + 0 + ); + assert_eq!( + benchset_copy_double( + Some(read), + (&mut input as *mut Input).cast(), + ptr::null_mut() + ), + 0 + ); + assert_eq!(input.calls, 0); + assert_eq!(input.bits, 1); + for bits in [ + 0, + 1, + 0x8000_0000_0000_0000, + 0x7fef_ffff_ffff_ffff, + 0x7ff0_0000_0000_0001, + 0xfff0_0000_0000_0001, + 0x7ff8_0000_0000_0001, + 0x7ff0_0000_0000_0000, + ] { + let mut input = Input { bits, calls: 0 }; + let mut output = [0x1234_u64, 0x5678, 0x9abc]; + assert_eq!( + benchset_copy_double( + Some(read), + (&mut input as *mut Input).cast(), + output.as_mut_ptr().add(1).cast() + ), + 1 + ); + assert_eq!(output, [0x1234, bits, 0x9abc]); + assert_eq!(input.calls, 1); + assert_eq!(input.bits, bits.wrapping_add(1)); + input.calls = 0; + input.bits = bits; + let pointer = &mut input as *mut Input; + assert_eq!( + benchset_copy_double( + Some(read), + pointer.cast(), + ptr::addr_of_mut!((*pointer).bits).cast() + ), + 1 + ); + assert_eq!(input.calls, 1); + assert_eq!(input.bits, bits); + } + NULL_CALLS.store(0, Ordering::Relaxed); + assert_eq!( + benchset_copy_double(Some(read_null), ptr::null_mut(), &mut untouched), + 1 + ); + assert_eq!(untouched.to_bits(), (-0.0_f64).to_bits()); + assert_eq!(NULL_CALLS.load(Ordering::Relaxed), 1); + } +} use crate::{Phase, Sample}; use std::ffi::CStr; use std::mem::{size_of, MaybeUninit}; diff --git a/native/src/lib.rs b/native/src/lib.rs index e6d6200..c85d710 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -1,4 +1,5 @@ //! Sustained scalar CPU throughput, available through Rust and a C ABI. +include!("catalog.rs"); mod ffi; mod kernel; mod reference; @@ -15,27 +16,8 @@ use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); -pub const DEFAULT_WARMUP_SECONDS: f64 = 120.0; -pub const DEFAULT_MEASUREMENT_SECONDS: f64 = 60.0; -pub const BENCHMARKS: [Benchmark; 2] = [Benchmark::SingleCore, Benchmark::MultiCore]; static ACTIVE: AtomicBool = AtomicBool::new(false); -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -#[repr(u32)] -pub enum Benchmark { - SingleCore = 1, - MultiCore = 2, -} - -impl Benchmark { - pub fn name(self) -> &'static str { - match self { - Self::SingleCore => "single_core", - Self::MultiCore => "multi_core", - } - } -} - #[derive(Clone, Copy, Debug)] pub struct Options { pub warmup: Duration, @@ -163,28 +145,12 @@ pub struct SuiteResult { impl SuiteResult { pub fn new(results: Vec) -> Self { - let complete = results.len() == BENCHMARKS.len() - && results.iter().zip(BENCHMARKS).all(|(result, benchmark)| { - result.benchmark == benchmark - && result.status == Status::Completed - && result - .pi_equivalent - .is_some_and(|value| value.is_finite() && value >= 0.0) - }); - let pi_equivalent = complete.then(|| { - if results - .iter() - .any(|result| result.pi_equivalent == Some(0.0)) - { - return 0.0; - } - (results - .iter() - .map(|result| result.pi_equivalent.unwrap().ln()) - .sum::() - / results.len() as f64) - .exp() - }); + let pi_equivalent = aggregate( + results.len(), + |index, expected| results[index].benchmark == expected, + |index| results[index].status as u32, + |index| results[index].pi_equivalent, + ); Self { results, aggregated_scores: AggregatedScores { pi_equivalent }, @@ -192,6 +158,99 @@ impl SuiteResult { } } +#[derive(Clone, Copy)] +#[repr(u32)] +enum AggregateOperator { + GreaterEqual = 2, + Equal = 3, +} + +fn finite(value: f64) -> bool { + value.to_bits() & 0x7ff0_0000_0000_0000 != 0x7ff0_0000_0000_0000 +} + +fn zero(value: f64) -> bool { + value.to_bits() << 1 == 0 +} + +fn nonnegative(value: f64) -> bool { + let bits = value.to_bits(); + bits <= 0x7fef_ffff_ffff_ffff || bits == 0x8000_0000_0000_0000 +} + +fn positive(value: f64) -> bool { + value.to_bits().wrapping_sub(1) < 0x7fef_ffff_ffff_ffff +} + +trait AggregateValue { + fn number(&self) -> f64; + fn compare(&self, operator: AggregateOperator, operand: u32) -> Option; + fn logarithm(&self) -> Option { + Some(self.number().ln()) + } + + fn exponential(&self, exponent: f64) -> Option { + Some(exponent.exp()) + } +} + +impl AggregateValue for f64 { + fn number(&self) -> f64 { + *self + } + + fn compare(&self, operator: AggregateOperator, _operand: u32) -> Option { + // The calculator compares only with integer zero; do not signal on NaNs. + Some(match operator { + AggregateOperator::GreaterEqual => nonnegative(*self), + AggregateOperator::Equal => zero(*self), + }) + } +} + +fn aggregate( + count: usize, + mut matches: impl FnMut(usize, Benchmark) -> bool, + mut status: impl FnMut(usize) -> u32, + mut ratio: impl FnMut(usize) -> Option, +) -> Option { + // Tuple identity comparison visits the common prefix before length differs. + if !BENCHMARKS + .into_iter() + .take(count) + .enumerate() + .all(|(index, expected)| matches(index, expected)) + || count != BENCHMARKS.len() + { + return None; + } + // Host values may require fallible conversion. Read them only when needed, + // preserving structural and left-to-right short-circuiting in the core. + let values = (0..count) + .map(|index| { + if status(index) != Status::Completed as u32 { + return None; + } + let value = ratio(index)?; + if !finite(value.number()) || !value.compare(AggregateOperator::GreaterEqual, 0)? { + return None; + } + Some(value) + }) + .collect::>>()?; + // Preserve the caller's floating environment: a zero skips every logarithm. + for value in &values { + if value.compare(AggregateOperator::Equal, 0)? { + return Some(0.0); + } + } + let mut logarithms = 0.0; + for value in &values { + logarithms += value.logarithm()?; + } + values[0].exponential(logarithms / BENCHMARKS.len() as f64) +} + /// Run every benchmark in order, stopping after failure or cancellation. /// Progress includes the benchmark being measured and runs on the calling thread. pub fn run_all( @@ -207,10 +266,18 @@ pub fn run_all( } fn run_suite(mut runner: impl FnMut(Benchmark) -> BenchmarkResult) -> SuiteResult { + run_suite_with(|benchmark| Some((runner(benchmark), true))) +} + +fn run_suite_with( + mut runner: impl FnMut(Benchmark) -> Option<(BenchmarkResult, bool)>, +) -> SuiteResult { let mut results = Vec::new(); for benchmark in BENCHMARKS { - let result = runner(benchmark); - let completed = result.status == Status::Completed; + let Some((result, proceed)) = runner(benchmark) else { + break; + }; + let completed = result.status == Status::Completed && proceed; results.push(result); if !completed { break; diff --git a/native/src/observed_suite_tests.rs b/native/src/observed_suite_tests.rs new file mode 100644 index 0000000..01f67ef --- /dev/null +++ b/native/src/observed_suite_tests.rs @@ -0,0 +1,179 @@ +use super::*; +use crate::tests::TEST_LOCK; +use std::mem::{size_of, MaybeUninit}; + +struct Events { + suite: *mut SuiteHandle, + results: Vec<(u32, u32, u64)>, + caller: std::thread::ThreadId, + stop: u32, + cancel_at: usize, +} + +unsafe extern "C" fn progress(value: *const CSuiteProgress, user: *mut c_void) { + let events = &*user.cast::(); + assert_eq!(std::thread::current().id(), events.caller); + // Seeing progress for benchmark N requires its entire preceding prefix to + // have been published already, not replayed after the native suite finishes. + assert_eq!( + (*value).benchmark, + crate::BENCHMARKS[events.results.len()] as u32 + ); +} + +unsafe extern "C" fn result(benchmark: u32, handle: *const Handle, user: *mut c_void) -> u32 { + let events = &mut *user.cast::(); + assert_eq!(std::thread::current().id(), events.caller); + let mut view = MaybeUninit::::uninit(); + assert_eq!( + benchset_result_v2(handle, view.as_mut_ptr(), size_of::() as u32), + 1 + ); + let view = view.assume_init(); + assert_eq!(benchmark, view.base.benchmark); + assert_eq!(benchmark, crate::BENCHMARKS[events.results.len()] as u32); + assert!(!benchset_reason(handle).is_null()); + for index in 0..view.base.sample_count { + let mut sample = MaybeUninit::::uninit(); + assert_eq!(benchset_sample(handle, index, sample.as_mut_ptr()), 1); + } + events + .results + .push((benchmark, view.base.status, view.base.iterations)); + if events.cancel_at == events.results.len() { + benchset_suite_cancel(events.suite); + } + events.stop +} + +/// Observers see owned native diagnostics before later work on the calling thread. +/// Stopping keeps the completed prefix; cancellation is sticky across child handles. +#[test] +fn observation_streams_results_and_controls_later_work_without_rewriting_them() { + let _guard = TEST_LOCK.lock().unwrap(); + unsafe { + for (stop, cancel_at, count) in [ + (0, 0, 2), + (1, 0, 1), + (7, 0, 1), + (u32::MAX, 0, 1), + (0, 1, 2), + (0, 2, 2), + (7, 1, 1), + ] { + let suite = benchset_suite_create(); + let mut config = MaybeUninit::::uninit(); + assert_eq!(benchset_default_config(config.as_mut_ptr()), 1); + let mut config = config.assume_init(); + config.benchmark = 999; + config.warmup_seconds = 0.0; + // Observing a completed prefix requires time for scheduled workers. + config.measurement_seconds = 0.2; + let mut events = Events { + suite, + results: Vec::new(), + caller: std::thread::current().id(), + stop, + cancel_at, + }; + let status = benchset_run_all_observed( + suite, + &config, + Some(progress), + Some(result), + (&mut events as *mut Events).cast(), + ); + assert_eq!(config.benchmark, 999); + assert_eq!(events.results.len(), count); + assert_eq!( + events.results[0].1, + Status::Completed as u32, + "{:?}", + (*suite) + .result + .lock() + .unwrap() + .as_ref() + .unwrap() + .result + .results + ); + assert!(events.results[0].2 > 0); + assert_eq!( + status, + if cancel_at == 1 && stop == 0 { + Status::Cancelled + } else { + Status::Completed + } as u32 + ); + if cancel_at == 1 && stop == 0 { + assert_eq!( + events.results[1], + (Benchmark::MultiCore as u32, Status::Cancelled as u32, 0) + ); + } + let mut summary = MaybeUninit::::uninit(); + assert_eq!( + benchset_suite_result( + suite, + summary.as_mut_ptr(), + size_of::() as u32 + ), + 1 + ); + let summary = summary.assume_init(); + assert_eq!(summary.result_count, count as u64); + assert_eq!( + summary.aggregated_scores.has_pi_equivalent != 0, + stop == 0 && cancel_at != 1 + ); + assert_eq!( + benchset_run_all_observed( + suite, + &config, + None, + Some(result), + (&mut events as *mut Events).cast() + ), + Status::Failed as u32 + ); + assert_eq!(events.results.len(), count); + benchset_suite_free(suite); + } + } +} + +/// Failed validation still publishes its first diagnostic, without progress or later work. +#[test] +fn observation_reports_invalid_configuration_and_rejects_null_suite() { + unsafe { + assert_eq!( + benchset_run_all_observed(ptr::null_mut(), ptr::null(), None, None, ptr::null_mut()), + Status::Failed as u32 + ); + let suite = benchset_suite_create(); + let mut events = Events { + suite, + results: Vec::new(), + caller: std::thread::current().id(), + stop: 0, + cancel_at: 0, + }; + assert_eq!( + benchset_run_all_observed( + suite, + ptr::null(), + Some(progress), + Some(result), + (&mut events as *mut Events).cast() + ), + Status::Failed as u32 + ); + assert_eq!( + events.results, + [(Benchmark::SingleCore as u32, Status::Failed as u32, 0)] + ); + benchset_suite_free(suite); + } +} diff --git a/native/src/suite_ffi.rs b/native/src/suite_ffi.rs index 419b535..bde87b0 100644 --- a/native/src/suite_ffi.rs +++ b/native/src/suite_ffi.rs @@ -1,5 +1,5 @@ use super::*; -use crate::{run_suite, SuiteResult}; +use crate::{run_suite_with, SuiteResult}; #[repr(C)] pub struct CAggregatedScores { @@ -21,6 +21,8 @@ pub struct CSuiteProgress { } pub type SuiteCallback = Option; +pub type ResultCallback = Option u32>; +pub type ConfigCallback = Option u32>; struct StoredSuite { result: SuiteResult, @@ -84,6 +86,51 @@ pub unsafe extern "C" fn benchset_run_all( config: *const Config, callback: SuiteCallback, user: *mut c_void, +) -> u32 { + benchset_run_all_observed(handle, config, callback, None, user) +} + +/// Publish borrowed child snapshots before the next benchmark starts. +#[no_mangle] +pub unsafe extern "C" fn benchset_run_all_observed( + handle: *mut SuiteHandle, + config: *const Config, + callback: SuiteCallback, + on_result: ResultCallback, + user: *mut c_void, +) -> u32 { + benchset_run_all_configured(handle, config, callback, on_result, None, user) +} + +/// Configure each borrowed child copy before validation, or abort without a child result. +#[no_mangle] +pub unsafe extern "C" fn benchset_run_all_configured( + handle: *mut SuiteHandle, + config: *const Config, + callback: SuiteCallback, + on_result: ResultCallback, + on_config: ConfigCallback, + user: *mut c_void, +) -> u32 { + execute_suite( + handle, + config, + callback, + on_result, + on_config, + user, + &mut |b, o, c, p| run(b, o, c, p), + ) +} + +unsafe fn execute_suite( + handle: *mut SuiteHandle, + config: *const Config, + callback: SuiteCallback, + on_result: ResultCallback, + on_config: ConfigCallback, + user: *mut c_void, + runner: &mut RustRunner<'_>, ) -> u32 { let Some(handle) = handle.as_ref() else { return Status::Failed as u32; @@ -92,12 +139,8 @@ pub unsafe extern "C" fn benchset_run_all( return Status::Failed as u32; } let mut reasons = Vec::new(); - let result = run_suite(|benchmark| { - let mut current = Handle { - cancellation: handle.cancellation.clone(), - used: AtomicBool::new(false), - result: Mutex::new(None), - }; + let mut configuration_aborted = false; + let result = run_suite_with(|benchmark| { // Inspect the fixed prefix before copying the whole config, just as run does. let mut copied; let config = if !config.is_null() @@ -105,11 +148,21 @@ pub unsafe extern "C" fn benchset_run_all( && ptr::addr_of!((*config).struct_size).read() == std::mem::size_of::() as u32 { copied = config.read(); + if on_config.is_some_and(|callback| callback(benchmark as u32, &mut copied, user) == 0) + { + configuration_aborted = true; + return None; + } copied.benchmark = benchmark as u32; &copied } else { config }; + let mut current = Handle { + cancellation: handle.cancellation.clone(), + used: AtomicBool::new(false), + result: Mutex::new(None), + }; let mut context = CallbackContext { benchmark, callback, @@ -120,13 +173,23 @@ pub unsafe extern "C" fn benchset_run_all( config, Some(progress_callback), (&mut context as *mut CallbackContext).cast(), - &mut |b, o, c, p| run(b, o, c, p), + runner, ); + let proceed = on_result.map_or(true, |callback| { + callback(benchmark as u32, ¤t, user) == 0 + }); let stored = current.result.into_inner().unwrap().unwrap(); reasons.push(stored.reason); - stored.result + Some((stored.result, proceed)) }); - let status = result.results.last().unwrap().status as u32; + let status = if configuration_aborted { + Status::Failed as u32 + } else { + result + .results + .last() + .map_or(Status::Failed, |result| result.status) as u32 + }; let views = result.results.iter().map(linked_result).collect(); *handle.result.lock().unwrap_or_else(|e| e.into_inner()) = Some(StoredSuite { result, @@ -241,3 +304,11 @@ pub unsafe extern "C" fn benchset_suite_free(handle: *mut SuiteHandle) { #[cfg(test)] #[path = "suite_ffi_tests.rs"] mod tests; + +#[cfg(test)] +#[path = "observed_suite_tests.rs"] +mod observed_tests; + +#[cfg(test)] +#[path = "configured_suite_tests.rs"] +mod configured_tests; diff --git a/native/src/suite_tests.rs b/native/src/suite_tests.rs index e413d26..d5714e8 100644 --- a/native/src/suite_tests.rs +++ b/native/src/suite_tests.rs @@ -1,5 +1,12 @@ use super::*; +/// Keep the existing public type path, including when users inspect nested types. +/// Sharing the catalog must not move Benchmark's nominal definition into a module. +#[test] +fn public_benchmark_retains_root_type_path() { + assert_eq!(std::any::type_name::(), "benchset::Benchmark"); +} + fn completed(benchmark: Benchmark, ratio: f64) -> BenchmarkResult { let mut result = BenchmarkResult::failed(benchmark, Options::default(), String::new()); result.status = Status::Completed; @@ -89,6 +96,75 @@ fn incomplete_invalid_or_failed_suites_have_no_full_score() { } } +/// Rejects unsuccessful status even when caller-built snapshots retain valid coefficients. +/// Both positions prevent a missing status check from hiding behind absent-ratio validation. +#[test] +fn unsuccessful_results_with_present_ratios_have_no_aggregate() { + for status in [Status::Failed, Status::Cancelled] { + for position in 0..BENCHMARKS.len() { + let mut results = vec![ + completed(Benchmark::SingleCore, 0.5), + completed(Benchmark::MultiCore, 2.0), + ]; + results[position].status = status; + assert!(SuiteResult::new(results) + .aggregated_scores + .pi_equivalent + .is_none()); + } + } +} + +/// Checks every missing-ratio position independently, including both missing and zero beside absence. +#[test] +fn every_missing_ratio_prevents_aggregation() { + for positions in [&[0][..], &[1][..], &[0, 1][..]] { + let mut results = vec![ + completed(Benchmark::SingleCore, 0.0), + completed(Benchmark::MultiCore, 2.0), + ]; + for &position in positions { + results[position].pi_equivalent = None; + } + assert!(SuiteResult::new(results) + .aggregated_scores + .pi_equivalent + .is_none()); + } +} + +/// Retains true double endpoints in log space and canonicalizes a present negative zero at either position. +#[test] +fn aggregate_float_endpoints_and_negative_zero() { + let smallest = f64::from_bits(1); + for (single, multi, expected) in [ + (f64::MAX, f64::MAX, f64::MAX), + (smallest, smallest, smallest), + (f64::MAX, smallest, f64::MAX.sqrt() * smallest.sqrt()), + ] { + let value = SuiteResult::new(vec![ + completed(Benchmark::SingleCore, single), + completed(Benchmark::MultiCore, multi), + ]) + .aggregated_scores + .pi_equivalent + .unwrap(); + assert!((value / expected - 1.0).abs() < 1e-12); + } + for position in 0..2 { + let mut results = vec![ + completed(Benchmark::SingleCore, 0.5), + completed(Benchmark::MultiCore, 2.0), + ]; + results[position].pi_equivalent = Some(-0.0); + let value = SuiteResult::new(results) + .aggregated_scores + .pi_equivalent + .unwrap(); + assert_eq!(value.to_bits(), 0.0_f64.to_bits()); + } +} + /// Each unsuccessful position stops later work without publishing an aggregate. #[test] fn suite_order_and_stop_after_every_unsuccessful_position() { diff --git a/pyproject.toml b/pyproject.toml index edd9c7d..7b3fe35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ backend-path = ["."] [project] name = "benchset" -version = "0.0.6" +version = "0.0.7" authors = [ { name="Evgeniy Blinov", email="zheni-b@yandex.ru" }, ] @@ -42,13 +42,13 @@ keywords = [ ] [tool.setuptools.package-data] -"benchset" = ["py.typed"] +"benchset" = ["py.typed", "_metadata.json"] [project.scripts] benchset = "benchset.cli:main" [[tool.mypy.overrides]] -module = ["benchset._native", "benchset.cli"] +module = ["benchset._native", "benchset._aggregate", "benchset.cli"] disallow_any_expr = false disallow_any_explicit = false disallow_any_unimported = false diff --git a/scripts/build_native.py b/scripts/build_native.py index 60cb43e..ccc48c9 100644 --- a/scripts/build_native.py +++ b/scripts/build_native.py @@ -1,29 +1,19 @@ """Developer build: put the native library beside the Python source package.""" -import shutil -import subprocess import sys -import tempfile from pathlib import Path if not __package__: sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from scripts.cargo import target_directory +from scripts.cargo import build, publish, target_directory from scripts.version import sync_version ROOT = Path(__file__).resolve().parents[1] if __name__ == '__main__': sync_version(ROOT) - subprocess.run(['cargo', 'build', '--release', '--locked'], cwd=ROOT, check=True) + metadata = build(ROOT, ['cargo', 'build', '--release', '--locked']) filename = 'benchset.dll' if sys.platform == 'win32' else ( 'libbenchset.dylib' if sys.platform == 'darwin' else 'libbenchset.so' ) - destination = ROOT / 'benchset/_native' - destination.mkdir(exist_ok=True) - # Never truncate a library that another process may be loading or using. - with tempfile.TemporaryDirectory(dir=destination) as temporary: - staged = Path(temporary) / filename - shutil.copy2(target_directory(ROOT) / 'release' / filename, staged) - staged.replace(destination / filename) - print(destination / filename) + print(publish(ROOT / 'benchset', target_directory(ROOT) / 'release' / filename, metadata)) diff --git a/scripts/cargo.py b/scripts/cargo.py index ce7704b..0d5aace 100644 --- a/scripts/cargo.py +++ b/scripts/cargo.py @@ -1,11 +1,111 @@ """Cargo output paths shared by build and release consumers.""" +import json import os +import shutil +import signal +import subprocess +import tempfile +from contextlib import ExitStack, contextmanager +from errno import EACCES from pathlib import Path -from typing import Mapping, Optional +from threading import current_thread, main_thread +from time import sleep +from typing import Any, Iterator, Mapping, Optional, cast def target_directory(root: Path, environment: Optional[Mapping[str, str]] = None) -> Path: """Resolve CARGO_TARGET_DIR against Cargo's working directory, or use target.""" environment = os.environ if environment is None else environment return root / environment.get('CARGO_TARGET_DIR', 'target') + + +def build(root: Path, arguments: list, environment: Optional[Mapping[str, str]] = None) -> bytes: + """Build native code and read its exact Rust-generated metadata snapshot.""" + result = subprocess.run([*arguments, '--message-format=json-render-diagnostics'], cwd=root, + env=environment, stdout=subprocess.PIPE, encoding='utf-8', check=True) + paths = [Path(os.fsdecode(bytes.fromhex(value))) for line in result.stdout.splitlines() + for message in [json.loads(line)] if message.get('reason') == 'build-script-executed' + for key, value in message['env'] if key == 'BENCHSET_METADATA_PATH_HEX'] + if len(paths) != 1: + raise RuntimeError('Cargo did not report exactly one benchset metadata snapshot') + return paths[0].read_bytes() + + +@contextmanager +def _package_lock(path: Path) -> Iterator[None]: + """Keep the kernel lock owned by a file, including interrupted acquisition.""" + with path.open('a+b') as lock, ExitStack() as cleanup: + if os.name == 'nt': + import msvcrt # noqa: PLC0415 — Platform-specific standard library. + + windows = cast(Any, msvcrt) + while True: + lock.seek(0) + try: + windows.locking(lock.fileno(), windows.LK_NBLCK, 1) + break + except OSError as error: + if error.errno != EACCES: + raise + sleep(.05) + else: + import fcntl # noqa: PLC0415 — Platform-specific standard library. + + try: + os.fchmod(lock.fileno(), 0o666) + except PermissionError: + pass # Another UID may own this writable persistent lock. + cleanup.callback(fcntl.flock, lock, fcntl.LOCK_UN) + fcntl.flock(lock, fcntl.LOCK_EX) + yield + + +def publish(package: Path, library: Path, metadata: bytes) -> Path: + """Serialize a package's complete publication, including its rollback snapshot.""" + package.mkdir(parents=True, exist_ok=True) + with _package_lock(package.resolve() / '.benchset.publish.lock'): + return _publish(package, library, metadata) + + +def _publish(package: Path, library: Path, metadata: bytes) -> Path: + """Stage both products and retain a complete generation after failed publication.""" + destination = package / '_native' + destination.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=destination) as temporary: + staged = Path(temporary) + previous = staged / 'previous-library' + installed = destination / library.name + if installed.exists(): + shutil.copy2(installed, previous) + shutil.copy2(library, staged / library.name) + (staged / '_metadata.json').write_bytes(metadata) + interrupted = False + + def interrupt(_signum: int, _frame: object) -> None: + nonlocal interrupted + interrupted = True + + # Check Ctrl+C between commits; finish any rollback before propagating it. + handles_signals = (current_thread() is main_thread() + and signal.getsignal(signal.SIGINT) is signal.default_int_handler) + handler = signal.signal(signal.SIGINT, interrupt) if handles_signals else None + try: + (staged / library.name).replace(installed) + if interrupted: + raise KeyboardInterrupt + (staged / '_metadata.json').replace(package / '_metadata.json') + except BaseException: + # A rename may finish before SIGINT: roll back only an incomplete pair. + if (staged / '_metadata.json').exists() and not (staged / library.name).exists(): + if previous.exists(): + previous.replace(installed) + else: + installed.unlink() + raise + finally: + if handler is not None: + signal.signal(signal.SIGINT, handler) + if interrupted: + raise KeyboardInterrupt + return destination / library.name diff --git a/scripts/check_source_install.py b/scripts/check_source_install.py index 572d147..42b5edb 100644 --- a/scripts/check_source_install.py +++ b/scripts/check_source_install.py @@ -35,7 +35,7 @@ def forbidden(*args, **kwargs): annotations = get_type_hints(BenchmarkResult) assert annotations['reference_machine'] is ReferenceMachine assert annotations['reference_run'] is ReferenceRun -for result in (BenchmarkResult('single_core'), run('unknown')): +for result in (BenchmarkResult('single_core'),): assert result.reference_machine is PI_REFERENCE_MACHINE assert result.reference_run is PI_REFERENCE_RUN payload = json.loads(json.dumps(asdict(result), allow_nan=False)) @@ -43,6 +43,7 @@ def forbidden(*args, **kwargs): assert payload['reference_run']['started_at_unix_seconds'] == 1792767590.517391 assert 'started_at' not in payload['reference_run'] ctypes.CDLL = load_native +assert run('unknown').reason == 'unknown benchmark: unknown' machine = asdict(PI_REFERENCE_MACHINE) assert machine == { 'name': 'Raspberry Pi 5 Model B Rev 1.0', 'short_name': 'pi', diff --git a/setup.py b/setup.py index 28dd9bd..8b4bfda 100644 --- a/setup.py +++ b/setup.py @@ -1,19 +1,20 @@ """Build an interpreter-independent platform wheel containing the Rust CDylib.""" +import os import platform import shutil import struct -import subprocess import sys -import tempfile from pathlib import Path from setuptools import Distribution, setup from setuptools.command.build_py import build_py from setuptools.command.develop import develop +from setuptools.command.install_lib import install_lib +from setuptools.errors import FileError from wheel.bdist_wheel import bdist_wheel from scripts.build_backend import rust_environment -from scripts.cargo import target_directory +from scripts.cargo import build, publish, target_directory from scripts.version import sync_version ROOT = Path(__file__).parent.resolve() @@ -57,19 +58,12 @@ def run(self): if tuple(map(int, requested.split('.'))) < tuple(map(int, minimum.split('.'))): requested = minimum environment['MACOSX_DEPLOYMENT_TARGET'] = requested - subprocess.run([cargo, 'build', '--release', '--locked', '--target', target], - cwd=ROOT, env=environment, check=True) + package = ROOT / 'benchset' if self.editable_mode else Path(self.build_lib) / 'benchset' + metadata = build(ROOT, [cargo, 'build', '--release', '--locked', '--target', target], environment) filename = 'benchset.dll' if sys.platform == 'win32' else ( 'libbenchset.dylib' if sys.platform == 'darwin' else 'libbenchset.so' ) - package = ROOT / 'benchset' if self.editable_mode else Path(self.build_lib) / 'benchset' - destination = package / '_native' - destination.mkdir(parents=True, exist_ok=True) - # Readers keep the old complete library until its replacement is ready. - with tempfile.TemporaryDirectory(dir=destination) as temporary: - staged = Path(temporary) / filename - shutil.copy2(target_directory(ROOT, environment) / target / 'release' / filename, staged) - staged.replace(destination / filename) + destination = publish(package, target_directory(ROOT, environment) / target / 'release' / filename, metadata).parent for old in destination.iterdir(): if old.name != filename and old.suffix in ('.so', '.dll', '.dylib'): old.unlink(missing_ok=True) @@ -97,6 +91,41 @@ def run(self): super().run() +class InstallNative(install_lib): + def get_exclusions(self): + return super().get_exclusions() | {self._exclude_pkg_path('benchset', '.benchset.publish.lock')} + + def copy_tree( # noqa: PLR0913 — Preserve the setuptools command signature. + self, infile, outfile, preserve_mode=True, preserve_times=True, preserve_symlinks=False, level=1, + ): + """Filter before copying while retaining the normal command's file policies.""" + assert preserve_mode + assert preserve_times + assert not preserve_symlinks + if not self.dry_run and not Path(infile).is_dir(): + raise FileError(f"cannot copy tree '{infile}': not a directory") + + def failed(error): + if not self.dry_run: + raise FileError(f"error listing files in '{error.filename}': {error.strerror}") + + excluded, outputs = self.get_exclusions(), [] + for directory, dirs, files in os.walk(infile, followlinks=True, onerror=failed): + dirs[:] = [name for name in dirs if not name.startswith('.nfs')] + relative = os.path.relpath(directory, infile) + target = outfile if relative == os.curdir else os.path.join(outfile, relative) + if not self.dry_run: + self.mkpath(target) + for name in files: + destination = os.path.join(target, name) + if not name.startswith('.nfs') and destination not in excluded: + if not self.dry_run: + self.copy_file(os.path.join(directory, name), destination, + preserve_mode, preserve_times, level=level) + outputs.append(destination) + return outputs + + setup(distclass=NativeDistribution, cmdclass={ - 'build_py': BuildNative, 'bdist_wheel': PlatformWheel, 'develop': DevelopNative, + 'build_py': BuildNative, 'bdist_wheel': PlatformWheel, 'develop': DevelopNative, 'install_lib': InstallNative, }) diff --git a/tests/core-centralization-plan.md b/tests/core-centralization-plan.md new file mode 100644 index 0000000..2dbf0b5 --- /dev/null +++ b/tests/core-centralization-plan.md @@ -0,0 +1,273 @@ +# Rust core centralization regression matrix + +The six groups below combine the independent regression reviews of moving +aggregation, suite execution, validation, the benchmark catalog, defaults and +calibration into Rust. Tests check observable behavior and transport separately: +matching arithmetic alone cannot detect a wrapper that recalculates a native +result, and an unchanged fixture cannot expose a handwritten metadata copy. + +## Review and evidence inventory + +| Group | Independent review set | +|---|---| +| 1. Aggregation | `aggregate_review_a`, `aggregate_review_b`, `aggregate_review_c` | +| 2. Suite execution | `suite_review_a`, `suite_review_b`, `suite_review_c` | +| 3. Validation | `validation_review_a`, `validation_review_b`, `validation_review_c` | +| 4. Catalog | `registry_review_a`, `registry_review_b`, `registry_review_c` | +| 5. Defaults | `defaults_review_a`, `defaults_review_b`, `defaults_review_c` | +| 6. Calibration | `calibration_review_a`, `calibration_review_b`, `calibration_review_c` | + +Local mutation evidence is deliberately outside the source distribution: + +| Label | Reports and logs under `target/` | +|---|---| +| A | `aggregation-audit/results.json`, `python-results.json`, `swift-results.json`, `audit-gap-results.json`; adjacent baseline and individual mutant logs | +| M | `metadata-mutations/results.json`; each record identifies its exact test and log | +| S | `swift-suite-mutations/results.json`; adjacent baseline and individual mutant logs | +| D | `swift-defaults-mutations/results.json`; native-defaults baseline and individual mutant logs | +| C | `core-centralization-audit/results.json`; baseline and individual mutant logs | +| P | `python-core-audit/mutation-results.json`; each record identifies exact tests and its adjacent mutant log | +| W | `core-centralization-audit/artifact-mutation.json`; real stale-snapshot wheel rebuild and its failing assertion | + +Reports record a successful unmodified baseline and individual deliberately +regressed copies. All 210 directed checks were detected by regression tests: +A 60 (21 Rust/storage, 13 Python transport, 6 Swift, 20 audit-gap checks), M 44, +S 23, D 11, C 24, P 47 and W 1. Infrastructure failures are not counted as +killed mutants. Two Swift mutations (`force_unwrap_progress` and +`double_free_suite_handle`) triggered runtime traps inside the designated tests +after successful compilation; the other 208 were detected by assertions. +New tests have docstrings or doc-comments explaining their regression target. +Final verification and local platform scope are recorded below. + +Pre-transfer Python characterization includes direct-constructor storage +(`A/baseline-python-storage.log`), unknown-name and surrogate behavior +(`A/prechange-unknown-names.log`, `prechange-surrogate-names.log`, +`prechange-validation-surrogates.log`). Existing CLI, duration, ownership, +metadata and packaged-consumer tests supplied the other original contracts. +Boundary/exporter tests additionally cover capabilities introduced by the +transfer; their unmodified baselines precede their directed mutations. +The later aggregation audit cases were also run against original source: +`aggregation-audit/audit-gap-head-baseline.log` records 13 passing cases. +`A/baseline-rust.log` verifies retained-ratio/status contracts alongside the +new FFI tests; it is an unmutated implementation baseline, not a separate +record of the pre-transfer Rust tree. + +## 1. Aggregation + +| Scenarios | Exact tests | Directed mutations / evidence | +|---|---|---| +| Complete ordered set only; no partial, duplicate, reordered or extra result; failed/cancelled/unknown status remains unavailable even with retained coefficients | Rust `incomplete_invalid_or_failed_suites_have_no_full_score`, `unsuccessful_results_with_present_ratios_have_no_aggregate`; Python `test_no_full_score_for_partial_missing_or_invalid_suite`, `test_aggregate_rejects_unsuccessful_results_with_retained_ratios`; Swift `aggregateRejectsUnsuccessfulResultsWithRetainedRatios`, `fullSuiteGeometricMeanAndForwarding` | A: `length`, `order`, `status-failed-single`, `status-failed-multi`, `status-cancelled-single`, `status-cancelled-multi`, six `swift-status-*` | +| Equal weights; original stored ratios rather than scores or rounded displays; unstable completed results included; zero remains present; validate all values before zero short-circuit; reject negative/NaN/infinite values at either position | Rust `aggregate_is_geometric_mean_of_ratios`, `geometric_mean_handles_zeros_and_extreme_ratios_without_multiplication`; Python `test_suite_aggregates_ratios_and_preserves_results`, `test_suite_rejects_invalid_ratios`; Swift `fullSuiteLogarithmicRangeZerosAndInvalidRatios` | A: `finite`, `negative`, `missing`, `zero`, `zero-hides-invalid`, `ratio-rounding`, `weight` | +| Marshal every input field/count; preserve unrelated native output, nonstandard nonzero presence flags, absent values, zero and precision; never recalculate in Python | Rust `aggregate_ffi_preserves_raw_fields_and_presence`; Python `test_aggregate_boundary_preserves_inputs_and_native_output`, `test_aggregate_boundary_reports_unavailable_native_output` | A: `ffi-nonzero-presence`, `ffi-ignore-presence`, `ffi-status`, `ffi-ratio`; `python-wire-*`, `python-output-*`, `python-getter-rejection` | +| Unknown strings never alias registered members, including UTF-8, embedded NUL and lone surrogates | Python `test_aggregate_does_not_alias_unknown_benchmark_names` | A: `python-wire-benchmark`, `python-native-nul`, `python-native-surrogates` | +| Python storage constructor retains supplied tuple and scores; `from_results` preserves original result objects; import/storage inspection remains native-free | `test_direct_suite_constructor_stores_supplied_fields_without_aggregation`, `test_direct_suite_constructor_remains_native_free_in_a_fresh_interpreter`, `test_suite_aggregates_ratios_and_preserves_results`, `test_calibration_imports_and_serialization_do_not_load_native_code` | A: `constructor-recomputes`, `constructor-copies-results`, `gap-nativefree-constructor` | +| C NULL/count/output contract, including nonzero NULL counts 1/2/3/MAX; no writes on rejection or surrounding canary damage; valid caller-owned buffers only | Rust `aggregate_pointer_and_count_contract` | A: `ffi-empty-count`, `ffi-null-contract`, `gap-ffi-null-output-write`, `gap-ffi-output-canary` | +| Independent public C header layout, offsets, field/count widths and direct aggregate call preserve nonstandard presence flags | `test_public_native_consumers[suite.c-C full suite smoke passed]`, `tests/native/suite.c` static assertions and direct `benchset_aggregate` call | A: `gap-c-count-width`, `gap-c-field-width` | +| First and both ratios missing; negative zero becomes present positive zero; minimum subnormal and maximum finite inputs; any zero skips every logarithm and preserves existing floating-point exception flags | Rust `every_missing_ratio_prevents_aggregation`, `aggregate_float_endpoints_and_negative_zero`; Python `test_aggregate_requires_each_ratio_independently`, `test_aggregate_negative_zero_is_present_canonical_positive_zero`, `test_aggregate_preserves_actual_finite_float_endpoints`, `test_zero_aggregate_preserves_floating_environment` | A: six `gap-python/rust-missing-*` checks, `gap-python-zero-input`, `gap-python-zero-sign`, `gap-rust-zero-sign`, `gap-rust-endpoint-max`, `gap-rust-endpoint-min`, `gap-fenv-all-logs`, `gap-fenv-nonzero-logs` | +| Pure aggregation remains usable inside active measurement callbacks and does not disturb reservation/cancellation/reuse | `test_aggregation_is_available_inside_active_benchmark_callbacks` | A: `gap-active-reservation`; `gap-active-head-baseline` verifies original behavior | +| Same-version older library must reject missing aggregate symbol before allocating | `test_aggregate_is_a_required_native_symbol_before_allocation` | A: `gap-required-symbol` | + +## 2. Suite execution + +| Scenarios | Exact tests | Directed mutations / evidence | +|---|---|---| +| Rust owns sequence and early stop at either unsuccessful position; retain complete prefix and unsuccessful diagnostics; observer sees results before subsequent work; every nonzero observer result stops; caller-thread callbacks | Rust `suite_order_and_stop_after_every_unsuccessful_position`, `public_suite_progress_cancellation_and_invalid_duration`, `observation_streams_results_and_controls_later_work_without_rewriting_them`, `observation_reports_invalid_configuration_and_rejects_null_suite`; Python `test_run_all_returns_suite_and_stops_on_failure`, `test_public_api_forwards_durations_and_callback_without_running_long_benchmarks` | C: `observer_drop`, `observer_ignore_stop`, `observer_wrong_id`, `observer_cancellation`, `observer_nonzero_stop`; P: `explicit_suite_mode`, `suite_free_mode` | +| Copy every child result/reason/sample/ratio before borrowed buffers disappear; decoding failure at either child keeps earlier results and frees suite once | Swift `suiteSnapshotsOwnEachNativePayload`; Python `test_suite_child_decoder_failure_keeps_prefix_and_frees_once`; native `actual_suite_getters_progress_cancellation_and_owned_results`, `suite_getters_preserve_zero_aggregate_and_indexed_samples` | S: `drop_first_child`, `drop_reason`, `drop_iterations`, `drop_ratio`, `drop_sample_time`, `drop_sample_iterations`, `ignore_result_getter_failure`, `ignore_sample_getter_failure` | +| Callback BaseExceptions stop future notifications/work; terminal callback failure cannot publish completed score; preserve prefix; no host exception unwinds through C; malformed contexts/IDs/child handles rejected after return | Python `test_suite_callback_failure_stops_notifications_and_keeps_completed_prefix`, `test_callback_base_exceptions_stop_notifications_and_allow_reuse`; Swift `suiteCallbackBoundaryRejectsMalformedResultsWithoutUnwinding` | S: `continue_after_callback_error`, `accept_nil_result_context`, `force_unwrap_progress`, `accept_unknown_result`, `accept_nil_child`, `ignore_progress_id`, `ignore_result_error`; P: `suite_callback_base_exceptions_do_not_cross_c`, `suppress_notifications_after_callback_failure`, `host_failure_overrides_completed_status`, `terminal_callback_failure_stops_suite`, `host_failure_hides_native_aggregate` | +| Cancellation reaches proper single/suite API; sticky between children; concurrent/reentrant owners remain excluded; cleanup waits for cancellation; no double-free/leak; captures and runner released on every exit | Swift `cancellationUsesTheActiveNativeHandleKind`, `suiteReservationSurvivesNativeHandleReplacement`, `concurrentSuitesKeepSingleOwnerDuringCancellation`, `suiteCallbackCapturesReleaseOnAllExits`, `fullSuiteRejectsReentryAndRecoversAfterError`, `fullSuiteCancellationSurvivesCompletedBenchmark`, `cleanupWaitsForAnInFlightNativeCancellation`; Python existing ownership, SIGINT and cleanup tests | S: `wrong_cancel_api`, `wrong_free_api`, `retain_context`, `leak_suite_handle`, `double_free_suite_handle`, `ignore_allocation_failure`, `ignore_abi`, `ignore_config_failure`; C: `observer_cancellation` | +| CLI flushes each result on main thread before next native progress, preserves stderr/stdout and exits 0/1/2/130; intermediate write failure releases blocked observer and joins workers; late SIGINT suppresses aggregate; interruption during first rendering retains already printed output, exits 130 without an extra footer | `test_redirected_result_is_flushed_before_the_next_benchmark_starts`, `test_real_native_cli_flushes_first_result_before_second_progress`, `test_real_native_cli_render_failure_releases_blocked_observer`, `test_cli_interrupt_during_first_result_render_stops_the_suite`, `test_cli_exit_stream_and_early_stop`, `test_interrupted_completed_cli_suite_has_no_aggregate`, `test_cli_repeated_interrupt_has_priority_over_completed_result`, `test_sigint_at_worker_lifecycle_boundaries_waits_for_teardown`, `test_real_ctrl_c_cancels_each_phase_and_stops_the_suite` | P: `intermediate_result_is_flushed`, `native_next_waits_until_render_ack`, `rendering_stays_on_main_thread`, `render_failure_cancels_owned_suite`, `intermediate_render_interrupt_is_quiet` | +| Final summary failure/count mismatch or a prefix-only backend cannot become CLI success; preserve unrelated native aggregate, zero and noncanonical presence without recalculation | `test_cli_suite_getter_failure_after_completed_children_is_not_success`, `test_suite_rejects_mismatched_native_snapshot_count`, `test_suite_forwards_native_aggregate_without_recalculation`, `test_cli_rejects_a_prefix_only_backend_and_restores_worker_ownership` | P: `failed_suite_observer_is_not_skipped`, `native_snapshot_count_checked`, `native_presence_not_truthiness_of_score`, `nonzero_native_presence_flags`, `native_aggregate_not_rounded`, `native_aggregate_not_recomputed`, `prefix_only_backend_cannot_report_success` | +| Suite handle kind used for cancellation/SIGINT/free; loading/allocation/execution/free lifecycle leaves no live handle or lost owner; callback/getter/result-observer errors release captures without GC after native return | `test_suite_cancellation_targets_its_owned_native_handle`, `test_sigint_defers_through_handle_lifetime_and_restores_runner`, `test_suite_child_decoder_failure_keeps_prefix_and_frees_once`, `test_suite_getter_failures_release_callback_captures_without_gc`, `test_live_callback_capture_released_without_gc`, `test_result_observer_exceptions_wait_for_native_return_and_release_captures` | P: `suite_cancel_mode`, `suite_interrupt_cancel_mode`, `suite_free_mode`, `signal_cannot_cancel_freeing_handle`, `suite_callback_failure_cancels_native`, `child_decode_failure_stops_and_keeps_prefix`, `suite_callback_traceback_cycles_cleared`, five `result_observer_*` checks | + +## 3. Validation + +| Scenarios | Exact tests | Directed mutations / evidence | +|---|---|---| +| Unknown benchmark before warmup before measurement; exact API/CLI errors; zero warmup allowed, zero/negative-zero measurement rejected; finite, negative, huge/subnanosecond and NaN cases retain requested values | `test_validation_preserves_exact_errors_precedence_and_requested_values`, `test_native_duration_limits_preserve_requested_diagnostics`, `test_programmatic_cli_rejects_invalid_durations_with_option_diagnostic`; Rust `validation_rules_and_messages_preserve_each_interface`, `invalid_config_matrix`; Swift `invalidDurationsAndRecovery` | C: `unknown_id`, `warmup_finite`, `measurement_finite`, `warmup_zero`, `measurement_zero`, `warmup_reason`, `measurement_reason`, `cli_reason` | +| Invalid reentry retains option-error precedence; numeric conversion of both arguments precedes loading/validation; bad benchmark types (including None) raise TypeError instead of selecting a suite; invalid suite options do not allocate; CLI warmup error precedes measurement | `test_reentrant_invalid_request_keeps_validation_precedence`, `test_invalid_python_types_raise_before_start_and_do_not_poison_runner`, `test_all_entrypoints_convert_both_numbers_before_validation_or_loading`, `test_invalid_benchmark_types_cannot_select_a_suite`, `test_numeric_conversion_keeps_priority_over_invalid_benchmark_type`, `test_suite_invalid_durations_keep_exact_diagnostics_without_allocation`, `test_cli_invalid_warmup_precedes_invalid_measurement` | P: `invalid_name_types_rejected`, `type_check_after_numeric_conversion`, `both_float_conversions_precede_native_load`, `warmup_conversion_precedes_measurement`, `measurement_conversion_not_skipped`, `validation_before_busy_diagnostic`, `validate_before_handle_allocation`, `cli_duration_argument_order` | +| Loading failure wins before a core is published, without releasing another call's owner; an executing runner reuses its existing core for validation/busy diagnostics rather than attempting a failing reload; ordinary loader errors remain failed results/clean CLI output | `test_busy_initialization_load_failure_keeps_priority_without_releasing_owner`, `test_busy_runner_uses_its_loaded_core_when_new_loading_fails`, `test_loading_failure_has_priority_over_invalid_arguments`, `test_cli_loader_failure_is_rendered_without_a_traceback` | P: `loading_priority_during_busy_initialization`, `busy_loaded_core_reuse`, `loser_must_not_release_owner`, `runtime_load_error_is_failed_result`, `cli_validation_runtime_load_error_is_quiet` | +| Existing C vocabulary/validation precedence remains exact, including reserved/unknown/duration/subnanosecond failure and requested NaN/signed-zero bits | Rust `existing_c_calls_keep_their_exact_validation_diagnostics` | C: `legacy_unknown_reason`, `legacy_reserved_reason`, `legacy_duration_reason`, `legacy_tiny_reason`, `legacy_warmup_bits`, `legacy_measurement_bits` | +| Names are exact bytes: no trimming, case folding, hyphen replacement, prefix/suffix matching or NUL truncation; UTF-8/surrogates rejected as unknown without encoding errors | Rust `names_are_length_delimited_and_catalog_driven`; Python `test_validation_preserves_exact_errors_precedence_and_requested_values`, `test_python_transports_native_registry_ids_and_utf8_byte_lengths` | C: `name_trim`, `name_case`, `name_hyphen`, `name_nul`, `name_len`; A: `python-native-surrogates`, `python-native-nul` | + +Native-free imports, calibration, signatures, direct storage constructors and +CLI help/version are separate from runtime validation, which may load Rust. +The loading-first failure contract is explicitly tested rather than retaining +handwritten Python validation as a fallback. Help/version tests forbid both +Python Native aliases; P `help_does_not_initialize_native` catches eager loading. + +## 4. Benchmark catalog + +| Scenarios | Exact tests | Directed mutations / evidence | +|---|---|---| +| Rust name/order changes regenerate metadata; checked snapshot is fresh; changed values reach fresh Python import/models/signatures and CLI settings rather than handwritten copies | `test_rust_generated_snapshot_tracks_registry_defaults_and_calibration`, `test_python_metadata_and_signature_defaults_are_native_free` | M: `freeze-generated-benchmark-order`, `reverse-public-registry`, `reuse-stale-generated-snapshot`, `handwritten-python-registry` | +| Binding accepts arbitrary native-returned ID and full UTF-8 byte length, including NUL/surrogates; does not reconstruct 1/2 from strings | `test_python_transports_native_registry_ids_and_utf8_byte_lengths`; Rust `names_are_length_delimited_and_catalog_driven` | C: `name_len`, `unknown_id`, `name_nul`; P: `native_id_used_without_python_registry`, `utf8_byte_length`, `unknown_name_surrogates_remain_unknown` | +| Execution/aggregation follow the Rust catalog; old C identities/layouts remain stable | Rust `suite_order_and_stop_after_every_unsuccessful_position`, `incomplete_invalid_or_failed_suites_have_no_full_score`; existing unpacked Rust/C consumers | C observer mutations; legacy ABI/platform gates below | + +## 5. Default options + +| Scenarios | Exact tests | Directed mutations / evidence | +|---|---|---| +| Native-free Python constants/signatures/dataclass defaults remain real floats, not sentinels; current and altered Rust defaults reach APIs, models and CLI; explicit values bypass defaults without rounding | `test_python_metadata_and_signature_defaults_are_native_free`, `test_rust_generated_snapshot_tracks_registry_defaults_and_calibration`, `test_public_api_forwards_durations_and_callback_without_running_long_benchmarks`, `test_defaults_repeated_options_and_environment_independence` | M: `integer-default-instead-of-float`, `default-signature-sentinel`, `freeze-generated-defaults`, `round-generated-float-defaults`, `round-measurement-default`, `handwritten-python-warmup`, `handwritten-python-measurement` | +| Swift obtains omitted windows from ABI-checked native config; getter failure publishes nothing; both entry points preserve explicit signed-zero bits and partially omitted arguments; rebuilt native defaults prove no current-value fallback | Swift `defaultLoaderChecksABIAndGetterBeforePublishingValues`, `defaultsFollowNativeConfigAndPreserveExplicitSignedZero`; Rust `defaults_versions_and_layout`, `public_defaults_names_and_shared_cancellation` | D: `freeze_public_warmup`, `freeze_public_measurement`, `freeze_suite_arguments`, `freeze_single_arguments`, `warmup_zero_sentinel`, `measurement_zero_sentinel`, `lose_warmup_signed_zero`, `lose_measurement_signed_zero`, `ignore_abi_guard`, `ignore_getter_failure`, `publish_failed_getter_payload`; altered-core baseline uses 0.125/0.375 | + +## 6. Calibration and build publication + +| Scenarios | Exact tests | Directed mutations / evidence | +|---|---|---| +| Generated values derive from Rust, survive a real incremental rebuild, and need no historical JSON; fresh Python consumes regenerated data | `test_rust_generated_snapshot_tracks_registry_defaults_and_calibration`; `test_calibration_fixture_python_and_native_metadata_agree_without_a_handle` | M: `freeze-reference-machine-name`, `round-generated-reference-score`, `reuse-stale-generated-snapshot`, `handwritten-python-reference-score` | +| Precise float types/bits including integral floats, negative zero, subnormal/max finite; large integer capacities remain integers; UTF-8, quote/backslash/control escaping; source/output Unicode paths | `test_rust_generated_snapshot_tracks_registry_defaults_and_calibration`, `test_python_metadata_and_signature_defaults_are_native_free` | M: `integer-or-signedzero-reference-score`, `normalize-negative-zero-score`, `underflow-subnormal-score`, `clamp-extreme-score`, `truncate-u64-capacity`, `round-reference-warmup`, `drop-unicode-metadata-characters`, `decode-snapshot-as-ascii`, `drop-json-quote-escaping`, `drop-json-backslash-escaping`, `drop-json-control-escaping` | +| Shared global identity, frozen nested tuples, serializable copies, UTC epoch/date precision, every terminal status, callback/thread reads and retained lifetime | `test_result_reference_fields_are_public_required_types_with_global_defaults`, `test_reference_models_are_frozen_including_nested_results_and_tuple`, `test_result_reference_serialization_copies_data_without_mutating_globals`, `test_reference_timestamp_is_aware_utc_and_json_keeps_primitive_epoch_seconds`, `test_globals_remain_identical_and_readable_during_callbacks_and_after_terminal_states`, `test_retained_result_metadata_outlives_runner_and_is_readable_during_later_run`; Swift `globalReferenceMetadataMatchesRecordedSeries`, `metadataCopiesStringsBeforeNextGetterAndOwnsEverySnapshot` | M: `mutable-reference-results`, `rounded-reference-score`, `string-memory-capacity`, `lose-shared-reference-identity`; existing native/reference ownership tests | +| Use exact Cargo-reported path and UTF-8 JSON; reject missing/ambiguous reports and stale conventional output; retain build environment/working directory | `test_build_publishes_exact_reported_metadata_atomically` | M: `drop-cargo-json-format`, `lose-build-environment`, `wrong-build-working-directory`, `pick-stale-conventional-output`, `ignore-ambiguous-output-report`, `lose-utf8-cargo-decoding`, `publish-empty-snapshot` | +| Stage library and metadata before publishing; each copy/write/rename failure restores old complete files; first-install failure/KeyboardInterrupt removes new library; no staging remnants | `test_build_publishes_exact_reported_metadata_atomically`, `test_first_install_metadata_failure_removes_unpublished_library`, `test_developer_build_copies_the_fresh_host_library_after_successful_cargo`, `test_native_build_installs_fresh_target_binary_in_wheel_or_editable_package` | M: `publish-in-place-before-replace`, `ignore-metadata-publication-error`, `leave-staging-file`, `replace-library-before-staging-metadata`, `skip-backup-restoration`, `skip-first-install-rollback`, `ignore-interruption-rollback` | +| Package generated snapshot, build script and catalog in source/wheel installs; poison source snapshot before real sdist-to-wheel build so stale checked metadata cannot ship; native/Swift consumers remain Python-independent | `test_sdist_builds_an_installable_wheel_with_api_and_both_cli_entrypoints`; existing crate/native ZIP/offline Swift consumer tests in `test_reference_artifacts.py`; `scripts/check_source_install.py` | W: `artifact-stale-snapshot`, killed at wheel `reference_machine` assertion; unpacked source/wheel/native/Swift consumer gates passed | + +Publication uses atomic replacement of each file and rollback on a failed +publication. Two separate renames do not promise an arbitrary concurrent reader +an atomic transaction over the library/snapshot pair. + +## Cross-cutting gates and scope + +- Public typing: `test_result_reference_public_type_contract` and + `test_positive_result_reference_consumer_imports_without_native_code`, plus + strict mypy/lint. Metadata constants keep `Tuple[str, ...]` and `float` types. +- Compatibility: Rust 1.74 library/example CI; existing C/C++ and legacy ABI-1/V2 + consumers, explicit wire-layout tests, unpacked distribution consumers and the + normal Swift deployment target. New APIs are additive. +- Manual C declarations, Swift enum cases and ctypes wire structures are fixed + ABI marshaling, not independent benchmark policy. The historical + `native/reference-calibration.json` remains independent provenance evidence; + runtime generation does not read it. +- Arbitrary object/complex-valued fields outside the annotated public model + types, undefined C pointers/alignment/lifetimes, genuine OOM, speed thresholds + and an atomic transaction for arbitrary concurrent two-file readers are not + claimed contracts. Real supported float/string/status values and ordinary + Python numeric conversion errors remain covered. + +## Centralization validation before iterative review + +Verified on macOS ARM64; the repository's Linux/Windows/other-architecture CI +matrix remains necessary for those platforms. Counts below describe separate +recorded selections, not an invented combined collection total. + +| Check | Verified result / evidence | +|---|---| +| Python excluding artifact consumers | 1326 passed, plus 5 subsequently added resource/core cases passed; `target/core-centralization-audit/python-coverage-tests.log` | +| Artifact and native consumers | 144 passed, 1 legacy-Python compatibility skip; `target/core-centralization-audit/artifact-tests.log` | +| Python runtime statement/branch coverage | 591 statements and 108 branches, all covered; `target/core-centralization-audit/python-coverage.txt` | +| Rust unit tests and native coverage | 71 tests passed; 100% lines, regions, functions and instantiations; `target/core-centralization-audit/rust-coverage.log` | +| Swift tests and native coverage | 41 tests passed; 100% lines, regions, functions and instantiations; `target/core-centralization-audit/swift-coverage.log` | +| Thread Sanitizer | Actual concurrent-suite ownership run passed; `target/swift-suite-thread-sanitizer.log` | +| Build/package utilities | 359 passed; `target/core-centralization-audit/build-tests.log` | +| Minimum Rust and static checks | Rust 1.74 library/example, cargo fmt/Clippy, Ruff, strict mypy for package/tests and git diff whitespace checks passed | + +LLVM's native reports have no branch counters: native 100% coverage above does +not claim branch coverage. Generated-package subprocesses remove inherited +coverage variables so independent library copies cannot inflate the runtime +coverage inventory. + +## Iterative post-transfer review + +Eight rounds each used three fresh, independent, read-only reviewers. The last +two rounds inspected the final production implementation and both returned zero +new confirmed regressions. Speculation, resolved duplicates and the unsupported +inputs listed above were not counted as useful findings. + +| Round | Useful findings | Confirmed scenario and correction | +|---|---:|---| +| 1 | 1 | Signal reentry before ownership publication overwrote the single/suite handle kind; claim ownership before publishing the mode. | +| 2 | 1 | SIGINT after a completed rename mixed publication generations; protect both renames and roll back only an incomplete pair. | +| 3 | 1 | A backend error after aborted CLI output published a second result to an absent consumer; stop publishing after abort. | +| 4 | 1 | Ctrl+C could not interrupt intermediate output blocked by a full pipe; preserve interruptible render/flush/separator operations and bypass stdout for a pending interruption. | +| 5 | 1 | SIGINT immediately after installing the output handler skipped lifetime-handler restoration; protect the transition itself. | +| 6 | 2 | SIGINT could also interrupt handler restoration; latch the first output Ctrl+C before raising and defer repeats through cleanup. SIGINT during publication rollback destroyed the backup; defer default main-thread SIGINT through commit/rollback, check between commits and rethrow after restoring consistency. | +| 7 | 0 | Python runtime, Rust/C/Swift boundaries and integration/build review. | +| 8 | 0 | Independent repeat of the full review; second consecutive clean round. | + +Each confirmed finding had a failing regression assertion before its production +fix and passing assertions afterward. New tests document their purpose; signal +and blocked-output probes use real signals, native calls or filesystem mutations +with explicit synchronization and bounded cleanup. + +| Contract | Regression tests | +|---|---| +| Reentrant signal preserves native handle kind and runner reuse | `test_reentrant_signal_before_suite_claim_preserves_native_handle_kind` | +| Publication stays consistent after either rename and during rollback, for existing and first installations | `test_publication_signal_after_rename_keeps_one_complete_generation`, `test_sigint_during_publication_rollback_keeps_complete_files` | +| Publication preserves background-thread and custom/ignored signal support | `test_publication_preserves_thread_and_signal_handler_support` | +| Aborted output cannot deadlock a second backend publication | `test_cli_output_abort_does_not_wait_for_a_second_backend_error` | +| Ctrl+C interrupts blocked render, flush and separator; pending interruption bypasses stdout | `test_sigint_interrupts_blocked_intermediate_output` | +| Repeated Ctrl+C across output-handler transitions waits for actual native cleanup | `test_repeated_sigint_at_output_handoff_waits_for_native_cleanup` | +| Custom/ignored output signal policy, background CLI and late-interrupt precedence remain intact | `test_intermediate_output_preserves_caller_sigint_policy`, `test_cli_can_run_outside_main_thread`, `test_cli_rejects_a_prefix_only_backend_and_restores_worker_ownership` | + +All 22 additional directed mutations were killed by regression assertions +(rounds 1–6: 1, 2, 1, 4, 2, 12). Baselines, individual mutant logs, RED/GREEN +logs and the round ledger are under `target/post-transfer-review/`; each +`round-N/evidence.json` records its mutation results, and `rounds.json` records +the reviewers and the two-round clean streak. These are separate from the +210 centralization checks recorded earlier. + +Final verification on macOS ARM64: + +- Complete Python selection: 1500 passed, 1 legacy-Python compatibility skip; + `final-tests.log`. Two subsequently added caller-policy cases and the two-case + prefix/late-interrupt selection passed in `final-cli-policy-tests.log` and + `final-late-interrupt-tests.log` (the latter includes one previously existing + case). These selections are reported separately to avoid double counting. +- Python runtime coverage: all 605 statements and 118 branches covered; + `final-coverage.txt`. +- Ruff, strict package mypy, test mypy and git whitespace checks passed. +- Rust and Swift implementation files were unchanged by these review fixes; + their native tests, coverage and consumer checks are recorded in the preceding + validation table. The full Python run includes artifact/consumer checks. + +## Original-requirements completion review + +Three independent reviewers (`requirements_architecture`, +`requirements_bindings`, `requirements_evidence`) each received the full +original task and all six duplication groups. All three checked every group, +README, binary C independence, thin binding boundaries, preserved behavior and +the test/mutation evidence. No remaining production duplication or behavioral +defect was confirmed. Fixed wire declarations, DTO construction, presentation +and generated snapshots do not independently implement benchmark rules. + +Two further rounds each used three fresh reviewers, again receiving the entire +task and all six original duplication locations. Both returned zero new +confirmed findings after checking the corrections below. In total, nine +independent reviewers checked all six groups; the last two rounds were clean. + +| Round | Reviewers | New production defects | New evidence findings | +|---|---|---:|---:| +| 1 | `requirements_architecture`, `requirements_bindings`, `requirements_evidence` | 0 | 2 | +| 2 | `requirements_round2_a`, `requirements_round2_b`, `requirements_round2_c` | 0 | 0 | +| 3 | `requirements_round3_a`, `requirements_round3_b`, `requirements_round3_c` | 0 | 0 | + +This review identified an evidence gap: the Python finite-endpoint test had an +unmutated baseline, but the endpoint mutations selected only its Rust analogue. +Two renamed existing tests also lacked recorded failures under their own names. +The isolated audit `target/requirements-review/check_mutations.py` closes all +three targets without changing production code or weakening tests: + +| Test | Individually reproduced regressions | +|---|---| +| `test_aggregate_preserves_actual_finite_float_endpoints` | Input/output maximum clamping, subnormal flushing and float32 narrowing; Python direct-product aggregation (7 mutations). | +| `test_invalid_durations_do_not_allocate_a_native_handle` | Allocation before validation, generic diagnostics, lost requested durations, invalid status or present score/ratio (7 mutations). | +| `test_positive_result_reference_consumer_imports_without_native_code` | Eager native loading on import or storage construction (2 mutations). | + +The unmodified selection passed all 13 cases; every one of the 16 mutations +was detected by its designated test: 15 assertion failures and one explicit +`pytest.fail` on allocation before validation, with no collection/build failures. +Logs and exact selections are recorded in +`target/requirements-review/mutation-results.json`. These checks supplement +the earlier 210 + 22; they do not inflate those historical totals. No test +needed strengthening. This review also corrected the distinction between +assertion failures and the two expected Swift runtime traps above. +The same 13 selected cases passed again in the original checkout after the +isolated mutations (`target/requirements-review/final-verification.log`). No +production or test sources changed during this completion audit. Review +outcomes and the clean-round ledger are recorded in +`target/requirements-review/round-1-reviewers.json`, `round-2-reviewers.json`, +`round-3-reviewers.json` and `review-results.json`. + +Original-source baselines and mutation logs establish the tested contracts and +outcomes. The complete historical order of edits and the original 18 reviewer +conversations additionally rely on task history; the repository does not +archive those conversations, and coverage alone is not a proof of completeness. diff --git a/tests/helpers.py b/tests/helpers.py index 8432ba2..3ebb1b0 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,6 +1,11 @@ """Shared checks for result metadata and reusable native runners.""" +from typing import Callable, List, Optional + +import pytest + from benchset import ( + BENCHMARKS, PI_REFERENCE_MACHINE, PI_REFERENCE_RUN, BenchmarkResult, @@ -8,9 +13,37 @@ Progress, ReferenceMachine, ReferenceRun, + SuiteResult, ) +def mock_suite(monkeypatch: pytest.MonkeyPatch) -> None: + """Adapt single-result test fakes to the native suite callback boundary. + + These fakes exercise host forwarding/rendering; Rust's real suite ordering + and stopping are covered independently by core and integration tests. + """ + def execute( + runner: BenchmarkRunner, warmup_seconds: float, measurement_seconds: float, + on_progress: Optional[Callable[[Progress], None]], + on_result: Optional[Callable[[BenchmarkResult], bool]] = None, + ) -> SuiteResult: + results: List[BenchmarkResult] = [] + for name in BENCHMARKS: + try: + result = runner.run(name, warmup_seconds=warmup_seconds, + measurement_seconds=measurement_seconds, on_progress=on_progress) + except Exception as error: # noqa: BLE001 — Emulate the public runner failure boundary. + result = BenchmarkResult(name, reason=str(error)) + results.append(result) + stop = on_result(result) if on_result is not None else False + if not result.success or stop: + break + return SuiteResult.from_results(tuple(results)) + + monkeypatch.setattr(BenchmarkRunner, '_run_all', execute) + + def assert_result_references(result: BenchmarkResult) -> None: assert isinstance(result.reference_machine, ReferenceMachine) assert isinstance(result.reference_run, ReferenceRun) diff --git a/tests/native/suite.c b/tests/native/suite.c index 2e57da4..0ea9457 100644 --- a/tests/native/suite.c +++ b/tests/native/suite.c @@ -4,6 +4,15 @@ #include _Static_assert(sizeof(BenchsetAggregatedScores) == 16, "aggregate layout"); +_Static_assert(sizeof(BenchsetAggregateInput) == 24, "aggregate input layout"); +_Static_assert(offsetof(BenchsetAggregateInput, pi_equivalent) == 8, "ratio offset"); +_Static_assert(offsetof(BenchsetAggregateInput, has_pi_equivalent) == 16, "ratio presence offset"); +/** Keep aggregate counts 64-bit and preserve const input/writable output in the public declaration. */ +_Static_assert( + _Generic(&benchset_aggregate, + int32_t (*)(const BenchsetAggregateInput *, uint64_t, BenchsetAggregatedScores *): 1, + default: 0), + "aggregate signature"); _Static_assert(sizeof(BenchsetSuiteResult) == 32, "suite layout"); _Static_assert(offsetof(BenchsetSuiteProgress, progress) == 8, "progress layout"); @@ -21,6 +30,14 @@ static void cancel(const BenchsetSuiteProgress *progress, void *handle) { /** Checks suite layout, getters and cancellation; benchmark override must leave const config unchanged. */ int main(void) { + /* Independent C layout/call: nonzero presence flags and caller-built ratios must survive the ABI. */ + const BenchsetAggregateInput inputs[] = { + {BENCHSET_SINGLE_CORE, BENCHSET_COMPLETED, 0.5, 7}, + {BENCHSET_MULTI_CORE, BENCHSET_COMPLETED, 2.0, UINT32_MAX} + }; + BenchsetAggregatedScores aggregate; + assert(benchset_aggregate(inputs, 2, &aggregate)); + assert(aggregate.has_pi_equivalent && aggregate.pi_equivalent == 1.0); BenchsetConfig config; assert(benchset_default_config(&config)); config.benchmark = 999; /* Ignored by the suite entrypoint. */ diff --git a/tests/swift/AggregationTests.swift b/tests/swift/AggregationTests.swift new file mode 100644 index 0000000..ff69e5e --- /dev/null +++ b/tests/swift/AggregationTests.swift @@ -0,0 +1,25 @@ +@testable import Benchset +import CBenchset +import Testing + +private func aggregateInput(_ benchmark: Benchmark, _ status: UInt32, _ ratio: Double?) -> BenchmarkResult { + BenchmarkResult( + benchmark: benchmark, version: "test", status: status, reason: nil, score: 999, + workers: 1, warmupSeconds: 0, measurementSeconds: 1, elapsedSeconds: 1, + iterations: 1, checksum: 0, samples: [], variationPercent: nil, unstable: false, + piEquivalent: ratio + ) +} + +/// Rejects status independently of a retained ratio, including unknown native status codes. +/// Both positions guard caller-built snapshots that must not publish a successful aggregate. +@Test func aggregateRejectsUnsuccessfulResultsWithRetainedRatios() { + for status: UInt32 in [BENCHSET_FAILED, BENCHSET_CANCELLED, UInt32.max] { + for position in 0..<2 { + var results = [aggregateInput(.singleCore, BENCHSET_COMPLETED, 0.5), + aggregateInput(.multiCore, BENCHSET_COMPLETED, 2)] + results[position] = aggregateInput(results[position].benchmark, status, results[position].piEquivalent) + #expect(SuiteResult(results: results).aggregatedScores.piEquivalent == nil) + } + } +} diff --git a/tests/swift/BenchsetTests.swift b/tests/swift/BenchsetTests.swift index f2b1547..d883aa6 100644 --- a/tests/swift/BenchsetTests.swift +++ b/tests/swift/BenchsetTests.swift @@ -6,6 +6,7 @@ import Testing private final class NativeStub { let pointer = OpaquePointer(bitPattern: 0x1234)! + let suitePointer = OpaquePointer(bitPattern: 0x5678)! var abi: UInt32 = BENCHSET_ABI_VERSION var abiChecks = 0 var allocationFails = false @@ -15,6 +16,10 @@ private final class NativeStub { var creates = 0 var frees = 0 var cancels = 0 + var suiteCreates = 0 + var suiteFrees = 0 + var suiteCancels = 0 + var betweenBenchmarks: (() -> Void)? var configs: [BenchsetConfig] = [] var value = BenchsetResult() var piEquivalent = 0.12345678901234567 @@ -57,7 +62,7 @@ private final class NativeStub { reasonPointer = string.map { strdup($0)! } } var api: NativeAPI { - NativeAPI( + let single = NativeAPI( abiVersion: { self.abiChecks += 1; return self.abi }, create: { self.creates += 1; return self.allocationFails ? nil : self.pointer }, defaultConfig: { out in @@ -84,7 +89,8 @@ private final class NativeStub { return self.value.status }, cancel: { handle in #expect(handle == self.pointer); self.cancels += 1 }, - result: { _, out, size in + result: { handle, out, size in + #expect(handle == self.pointer) self.resultSizes.append(size) #expect(size == UInt32(MemoryLayout.size)) self.value.sample_count = UInt64(self.samples.count) @@ -92,18 +98,63 @@ private final class NativeStub { has_pi_equivalent: self.hasPiEquivalent) return self.resultAvailable ? 1 : 0 }, - sample: { _, index, out in + sample: { handle, index, out in + #expect(handle == self.pointer) if self.missingSample == index { return 0 } out.pointee = self.samples[Int(index)] return 1 }, - reason: { _ in self.reasonPointer.map { UnsafePointer($0) } }, + reason: { handle in + #expect(handle == self.pointer) + return self.reasonPointer.map { UnsafePointer($0) } + }, version: { UnsafePointer(self.versionPointer) }, free: { handle in #expect(handle == self.pointer); self.frees += 1 } ) + var native = single + native.suite = SuiteAPI( + create: { self.suiteCreates += 1; return self.allocationFails ? nil : self.suitePointer }, + run: { handle, config, callback, resultCallback, user in + #expect(handle == self.suitePointer) + for id in [BENCHSET_SINGLE_CORE, BENCHSET_MULTI_CORE] { + var current = config.pointee + current.benchmark = id + let context = StubSuiteCallbackContext(benchmark: id, callback: callback, user: user) + _ = withExtendedLifetime(context) { + single.run(self.pointer, ¤t, forwardStubSuiteProgress, + Unmanaged.passUnretained(context).toOpaque()) + } + let stop = resultCallback(id, self.pointer, user) + if self.value.status != BENCHSET_COMPLETED || stop != 0 { break } + if id == BENCHSET_SINGLE_CORE { self.betweenBenchmarks?() } + } + return self.value.status + }, + cancel: { handle in #expect(handle == self.suitePointer); self.suiteCancels += 1 }, + free: { handle in #expect(handle == self.suitePointer); self.suiteFrees += 1 } + ) + return native } } +private final class StubSuiteCallbackContext { + let benchmark: UInt32 + let callback: NativeSuiteCallback + let user: UnsafeMutableRawPointer + init(benchmark: UInt32, callback: @escaping NativeSuiteCallback, user: UnsafeMutableRawPointer) { + self.benchmark = benchmark + self.callback = callback + self.user = user + } +} + +private func forwardStubSuiteProgress(_ pointer: UnsafePointer?, _ user: UnsafeMutableRawPointer?) { + guard let value = pointer?.pointee, let user = user else { return } + let context = Unmanaged.fromOpaque(user).takeUnretainedValue() + var progress = BenchsetSuiteProgress(benchmark: context.benchmark, progress: value) + withUnsafePointer(to: &progress) { context.callback($0, context.user) } +} + private final class LockedBox { private let lock = NSLock() private var storage: T @@ -131,8 +182,193 @@ func expectRunnerReusable(_ runner: BenchmarkRunner, warmupSeconds: Double = 10) @Suite(.serialized) struct BenchsetTests { - /// Keep the suite reservation across native-handle teardown and replacement. - /// A contender queued during free must never run between the owner's two + /// Checks the defaults loader's ABI before a layout-sensitive getter call, + /// discards getter failure, and preserves full-precision values on retry. + /// Public default arguments must never consume an unchecked C structure. + @Test func defaultLoaderChecksABIAndGetterBeforePublishingValues() throws { + let stub = NativeStub() + var native = stub.api + var calls = 0 + let warmup = 1.2345678901234567 + let measurement = 9.876543210987654 + var available: Int32 = 1 + native.defaultConfig = { out in + calls += 1 + out.pointee.warmup_seconds = warmup + out.pointee.measurement_seconds = measurement + return available + } + for abi: UInt32 in [0, 2, UInt32.max] { + stub.abi = abi + let defaults = loadDefaults(using: native) + #expect(defaults.warmup_seconds == 0 && defaults.measurement_seconds == 0) + #expect(calls == 0) + } + stub.abi = BENCHSET_ABI_VERSION + available = 0 + let unavailable = loadDefaults(using: native) + #expect(unavailable.warmup_seconds == 0 && unavailable.measurement_seconds == 0) + #expect(calls == 1) + available = 1 + let defaults = loadDefaults(using: native) + #expect(defaults.warmup_seconds.bitPattern == warmup.bitPattern) + #expect(defaults.measurement_seconds.bitPattern == measurement.bitPattern) + #expect(calls == 2) + } + + /// Derives omitted windows from the linked Rust defaults while preserving + /// explicit signed zero. Changing Rust's defaults must update both Swift + /// entry points without introducing a hardcoded fallback or zero sentinel. + @Test func defaultsFollowNativeConfigAndPreserveExplicitSignedZero() throws { + var expected = BenchsetConfig() + #expect(benchset_default_config(&expected) == 1) + #expect(defaultWarmupSeconds.bitPattern == expected.warmup_seconds.bitPattern) + #expect(defaultMeasurementSeconds.bitPattern == expected.measurement_seconds.bitPattern) + let stub = NativeStub() + let runner = BenchmarkRunner(native: stub.api) + for suite in [false, true] { + stub.configs.removeAll() + if suite { _ = try runner.runAll() } + else { _ = try runner.run(.singleCore) } + #expect(stub.configs.count == (suite ? 2 : 1)) + #expect(stub.configs.allSatisfy { + $0.warmup_seconds.bitPattern == expected.warmup_seconds.bitPattern + && $0.measurement_seconds.bitPattern == expected.measurement_seconds.bitPattern + }) + stub.configs.removeAll() + if suite { _ = try runner.runAll(warmupSeconds: -0.0, measurementSeconds: -0.0) } + else { _ = try runner.run(.singleCore, warmupSeconds: -0.0, measurementSeconds: -0.0) } + #expect(stub.configs.count == (suite ? 2 : 1)) + #expect(stub.configs.allSatisfy { + $0.warmup_seconds.bitPattern == (-0.0 as Double).bitPattern + && $0.measurement_seconds.bitPattern == (-0.0 as Double).bitPattern + }) + } + } + + /// Copies every individual snapshot before native storage is reused or freed. + /// Distinct reasons, samples and optional ratios catch retaining a borrowed + /// child handle or decoding all results from the final child's buffers. + @Test func suiteSnapshotsOwnEachNativePayload() throws { + let stub = NativeStub() + stub.action = { stub, _, _ in + let ordinal = stub.configs.count + stub.setReason("result \(ordinal)") + stub.value.iterations = UInt64(ordinal * 8) + stub.piEquivalent = ordinal == 1 ? 0.5 : 2 + stub.samples = [BenchsetSample(elapsed_seconds: Double(ordinal), duration_seconds: 0.5, + iterations: UInt64(ordinal * 8), iterations_per_second: Double(ordinal * 16))] + } + let suite = try BenchmarkRunner(native: stub.api).runAll() + stub.setReason(nil) + stub.samples.removeAll() + stub.piEquivalent = 99 + #expect(suite.results.map { $0.reason } == ["result 1", "result 2"]) + #expect(suite.results.map { $0.iterations } == [8, 16]) + #expect(suite.results.map { $0.piEquivalent } == [0.5, 2]) + #expect(suite.results.map { $0.samples.count } == [1, 1]) + #expect(suite.results.map { $0.samples[0].elapsedSeconds } == [1, 2]) + #expect(suite.results.map { $0.samples[0].iterations } == [8, 16]) + #expect(suite.aggregatedScores.piEquivalent == 1) + } + + /// Releases suite progress captures, runner and native owner on every exit. + /// Getter failures must unwind only after native code returns, free the + /// suite exactly once, and release the context and native function table. + @Test func suiteCallbackCapturesReleaseOnAllExits() throws { + final class Probe { var calls = 0 } + for stage in ["abi", "allocation", "config", "result", "sample", "completed"] { + weak var observedStub: NativeStub? + weak var observed: Probe? + weak var observedRunner: BenchmarkRunner? + func exercise() throws { + let stub = NativeStub() + observedStub = stub + stub.abi = stage == "abi" ? 0 : BENCHSET_ABI_VERSION + stub.allocationFails = stage == "allocation" + stub.configAvailable = stage != "config" + stub.resultAvailable = stage != "result" + stub.missingSample = stage == "sample" ? 0 : nil + func makeCallback() -> (Benchmark, Benchset.Progress) -> Void { + let probe = Probe() + observed = probe + return { _, _ in probe.calls += 1 } + } + let runner = BenchmarkRunner(native: stub.api) + observedRunner = runner + if stage == "completed" { + #expect(try runner.runAll(onProgress: makeCallback()).results.count == 2) + } else { + let error: BenchsetError = stage == "allocation" ? .allocationFailed : .resultUnavailable + #expect(throws: error) { try runner.runAll(onProgress: makeCallback()) } + } + #expect(observed == nil) + #expect(stub.suiteFrees == (stage == "abi" || stage == "allocation" ? 0 : 1)) + } + try exercise() + #expect(observedRunner == nil) + #expect(observedStub == nil) + } + } + + /// Accepts absent progress pointers/context and contains malformed result + /// identifiers or missing child handles inside C callbacks. Swift errors + /// must surface only after native code returns, with the suite freed once. + @Test func suiteCallbackBoundaryRejectsMalformedResultsWithoutUnwinding() throws { + let stub = NativeStub() + var native = stub.api + let run = native.suite.run + native.suite.run = { handle, config, progress, result, user in + progress(nil, user) + progress(nil, nil) + var value = BenchsetSuiteProgress() + withUnsafePointer(to: &value) { progress($0, nil) } + #expect(result(BENCHSET_SINGLE_CORE, stub.pointer, nil) == 1) + return run(handle, config, progress, result, user) + } + #expect(try BenchmarkRunner(native: native).runAll().results.count == 2) + #expect(stub.suiteFrees == 1) + for malformed in ["progress-id", "result-id", "handle"] { + native.suite.run = { _, _, progress, result, user in + if malformed == "progress-id" { + var value = BenchsetSuiteProgress(benchmark: UInt32.max, progress: BenchsetProgress()) + withUnsafePointer(to: &value) { progress($0, user) } + } + #expect(result(malformed == "result-id" ? UInt32.max : BENCHSET_SINGLE_CORE, + malformed == "handle" ? nil : stub.pointer, user) == 1) + return BENCHSET_COMPLETED + } + let freed = stub.suiteFrees + #expect(throws: BenchsetError.resultUnavailable) { try BenchmarkRunner(native: native).runAll() } + #expect(stub.suiteFrees == freed + 1) + } + } + + /// Dispatches cancellation and cleanup to the active handle's native API. + /// Distinct single/suite addresses prevent a wrong getter or cancellation + /// function from appearing correct merely because opaque handles coincide. + @Test func cancellationUsesTheActiveNativeHandleKind() throws { + let stub = NativeStub() + let runner = BenchmarkRunner(native: stub.api) + runner.cancel() + var cancelled = false + #expect(try runner.runAll { _, _ in + if !cancelled { cancelled = true; runner.cancel() } + }.results.count == 2) + runner.cancel() + #expect(stub.suiteCancels == 1 && stub.cancels == 0) + #expect(stub.suiteCreates == 1 && stub.suiteFrees == 1) + cancelled = false + #expect(try runner.run(.singleCore) { _ in + if !cancelled { cancelled = true; runner.cancel() } + }.success) + runner.cancel() + #expect(stub.suiteCancels == 1 && stub.cancels == 1) + #expect(stub.creates == 1 && stub.frees == 1) + } + + /// Keep the suite reservation across native benchmark boundaries. + /// A contender queued between results must never run between the owner's two /// benchmarks; both another suite and a standalone run obey that reservation. /// Repeat queued contenders to exercise scheduler handoffs without requiring /// either thread to acquire the lock first in any particular iteration. @@ -146,22 +382,17 @@ struct BenchsetTests { let secondStarted = LockedBox(false) let intruded = LockedBox(false) let owner = Thread.current - var native = stub.api - let free = native.free - native.free = { handle in - free(handle) - if stub.frees == 1 { - freeing.signal() - #expect(attempting.wait(timeout: .now() + 5) == .success) - usleep(1000) - } + stub.betweenBenchmarks = { + freeing.signal() + #expect(attempting.wait(timeout: .now() + 5) == .success) + usleep(1000) } stub.action = { stub, _, _ in if Thread.current === owner { if stub.configs.last?.benchmark == Benchmark.multiCore.rawValue { secondStarted.value = true } } else if !secondStarted.value { intruded.value = true } } - let runner = BenchmarkRunner(native: native) + let runner = BenchmarkRunner(native: stub.api) DispatchQueue.global().async { defer { finished.signal() } guard freeing.wait(timeout: .now() + 5) == .success else { return } @@ -226,7 +457,7 @@ struct BenchsetTests { #expect(cancelDone.wait(timeout: .now() + 5) == .success) #expect(outcomes.value.filter { $0 == "owner" }.count == 1) #expect(outcomes.value.filter { $0 == "busy" }.count == 7) - #expect(stub.creates == 2 && stub.frees == 2) + #expect(stub.suiteCreates == 1 && stub.suiteFrees == 1) } /// Keeps progress on the caller thread, full sample payloads and one result per benchmark. @@ -278,7 +509,7 @@ struct BenchsetTests { #expect(stub.configs.allSatisfy { $0.warmup_seconds == 0.25 && $0.measurement_seconds == 0.75 }) #expect(benchmarks == Array(repeating: .singleCore, count: stub.phases.count) + Array(repeating: .multiCore, count: stub.phases.count)) - #expect(stub.creates == 2 && stub.frees == 2) + #expect(stub.suiteCreates == 1 && stub.suiteFrees == 1) #expect(result.results.allSatisfy { $0.unstable }) #expect(SuiteResult(results: []).aggregatedScores.piEquivalent == nil) #expect(SuiteResult(results: [result.results[0]]).aggregatedScores.piEquivalent == nil) @@ -302,7 +533,7 @@ struct BenchsetTests { #expect(result.results.count == stop) #expect(result.results.last?.status == status) #expect(result.aggregatedScores.piEquivalent == nil) - #expect(stub.frees == stop) + #expect(stub.suiteFrees == 1) } } let stub = NativeStub() @@ -410,12 +641,12 @@ struct BenchsetTests { stub.action = { stub, _, _ in if stub.configs.count == failingRun { stub.resultAvailable = false } } - let freesBefore = stub.frees + let freesBefore = stub.suiteFrees var published: SuiteResult? #expect(throws: BenchsetError.resultUnavailable) { published = try runner.runAll() } #expect(published == nil) #expect(stub.configs.count == failingRun) - #expect(stub.frees == freesBefore + 2) + #expect(stub.suiteFrees == freesBefore + 1) runner.cancel() #expect(stub.cancels == 0) stub.resultAvailable = true diff --git a/tests/test_aggregate_exponential.py b/tests/test_aggregate_exponential.py new file mode 100644 index 0000000..59d475b --- /dev/null +++ b/tests/test_aggregate_exponential.py @@ -0,0 +1,278 @@ +"""Preserve Python's exponential overflow boundary without moving arithmetic out of Rust.""" + +import ctypes +import gc +import math +import os +import shutil +import subprocess +import sys +import weakref +from contextlib import contextmanager +from types import SimpleNamespace +from typing import Any, Callable, Iterator, List, Optional, Tuple, Type, cast + +import pytest + +from benchset import BenchmarkResult, SuiteResult, _native +from tests.test_aggregate_getters import _ORDER, _result + + +@pytest.fixture(scope='module') +def floating_environment(tmp_path_factory: pytest.TempPathFactory) -> Any: + """Expose platform fenv constants and save/restore whole environments in C.""" + if os.name == 'nt': + pytest.skip('POSIX shared-library fenv probe; MSVC floating environment differs') + compiler = shutil.which('cc') or shutil.which('clang') or shutil.which('gcc') + if compiler is None: + pytest.skip('No C compiler for the floating-environment probe') + directory = tmp_path_factory.mktemp('aggregate-exponential') + source = directory / 'fenv.c' + source.write_text('''#include +#include +static fenv_t saved; +static int modes[] = {FE_TONEAREST, FE_DOWNWARD, FE_UPWARD, FE_TOWARDZERO}; +int begin(unsigned index) { + if (index >= 4 || fegetenv(&saved)) return -1; + if (fesetround(modes[index]) || feclearexcept(FE_ALL_EXCEPT)) return -1; + return feraiseexcept(FE_INVALID); +} +int expected_mode(unsigned index) { return index < 4 ? modes[index] : -1; } +int rounding(void) { return fegetround(); } +int flags(void) { return fetestexcept(FE_ALL_EXCEPT); } +int end(void) { return fesetenv(&saved); } +double geometric(double first, double second) { + return exp((log(first) + log(second)) / 2.0); +} +''') + library = directory / 'fenv.so' + built = subprocess.run([compiler, '-shared', '-fPIC', '-fno-builtin', str(source), '-lm', '-o', str(library)], + capture_output=True, text=True, check=False) + assert built.returncode == 0, built.stderr + probe = ctypes.CDLL(str(library)) + for name in ('begin', 'expected_mode'): + getattr(probe, name).argtypes = [ctypes.c_uint] + getattr(probe, name).restype = ctypes.c_int + for name in ('rounding', 'flags', 'end'): + getattr(probe, name).argtypes = [] + getattr(probe, name).restype = ctypes.c_int + probe.geometric.argtypes = [ctypes.c_double, ctypes.c_double] + probe.geometric.restype = ctypes.c_double + return probe + + +@contextmanager +def rounding(probe: Any, mode: int) -> Iterator[None]: + # CPython 3.15 GC timing statistics can set FE_INEXACT. Keep unrelated + # automatic collection outside the measured numeric operation. + collecting = gc.isenabled() + gc.disable() + before = probe.rounding(), probe.flags() + try: + assert probe.begin(mode) == 0 + yield + finally: + try: + assert probe.end() == 0 + assert (probe.rounding(), probe.flags()) == before + finally: + if collecting: + gc.enable() + + +def prepared_native(monkeypatch: pytest.MonkeyPatch) -> _native.Native: + native = _native.Native() + monkeypatch.setattr(_native, 'Native', lambda: native) + return native + + +def mean(results: Tuple[BenchmarkResult, ...]) -> Optional[float]: + return SuiteResult.from_results(results).aggregated_scores.pi_equivalent + + +def endpoint_results(number: float) -> Tuple[BenchmarkResult, ...]: + return tuple(BenchmarkResult(name, status='completed', pi_equivalent=number) + for name in ('single_core', 'multi_core')) + + +def observe(operation: Callable[[], Optional[float]]) -> Tuple[Optional[float], Optional[str]]: + try: + return operation(), None + except OverflowError as error: + return None, str(error) + + +@pytest.mark.parametrize('mode', range(4), ids=['nearest', 'downward', 'upward', 'towardzero']) +@pytest.mark.parametrize('number', [sys.float_info.max, float.fromhex('0x0.0000000000001p-1022')], + ids=['maximum', 'minimum-subnormal']) +def test_finite_endpoints_preserve_python_exp_overflow_and_caller_rounding( + floating_environment: Any, monkeypatch: pytest.MonkeyPatch, mode: int, number: float, +) -> None: + """A finite exponent may overflow under upward rounding and must raise as before. + + Derive the old Python outcome and floating exception flags under the same + mode, then invoke the real Rust calculator. Other modes protect finite + endpoints against clamping, direct products and unconditional overflow errors. + Minimum subnormals also preserve exp underflow as a value rather than an error. + The context manager restores every flag and rounding mode even on failure. + """ + prepared_native(monkeypatch) + results = endpoint_results(number) + with rounding(floating_environment, mode): + expected = observe(lambda: math.exp(sum(math.log(number) for _index in range(2)) / 2)) + expected_flags = floating_environment.flags() + with rounding(floating_environment, mode): + actual = observe(lambda: mean(results)) + actual_flags = floating_environment.flags() + actual_mode = floating_environment.rounding() + assert actual == expected + if expected[0] is not None: + assert float.hex(cast(float, actual[0])) == float.hex(expected[0]) + assert actual_flags == expected_flags + assert actual_mode == floating_environment.expected_mode(mode) + + +@pytest.mark.parametrize('mode', range(4)) +@pytest.mark.parametrize('log_value', [math.inf, math.nan], ids=['infinite', 'nan']) +def test_fresh_nonfinite_log_values_are_not_mistaken_for_finite_exp_overflow( + floating_environment: Any, monkeypatch: pytest.MonkeyPatch, mode: int, log_value: float, +) -> None: + """A late getter may yield infinity/NaN after finite validation without an exp error. + + math.exp(+inf) returns +inf; only a finite overflowing exponent raises. + Preserve the original six reads and skip a blanket rejection of nonfinite + aggregates, while the shared calculator still controls the complete order. + """ + prepared_native(monkeypatch) + events: List[str] = [] + results = (_result('single_core', 'first', (1.0,) * 5 + (log_value,), events), + _result('multi_core', 'second', (1.0,) * 6, events)) + with rounding(floating_environment, mode): + value = mean(results) + actual_mode = floating_environment.rounding() + assert value is not None + assert math.isnan(value) if math.isnan(log_value) else value == math.inf + assert tuple(events) == _ORDER + assert actual_mode == floating_environment.expected_mode(mode) + + +@pytest.mark.parametrize('mode', range(4)) +@pytest.mark.parametrize('position', [0, 1]) +def test_zero_skips_the_entire_overflowing_log_and_exponential_path( + floating_environment: Any, monkeypatch: pytest.MonkeyPatch, mode: int, position: int, +) -> None: + """A zero suppresses every later getter and exp operation without new fenv flags. + + The other value is the maximum finite float, so evaluating logs eagerly + could overflow under upward rounding. Four-value getters reject unused + fifth/sixth reads and preserve short-circuiting at either zero position. + """ + prepared_native(monkeypatch) + events: List[str] = [] + values = [(sys.float_info.max,) * 4, (sys.float_info.max,) * 4] + values[position] = (*values[position][:3], -0.0) + results = (_result('single_core', 'first', values[0], events), + _result('multi_core', 'second', values[1], events)) + with rounding(floating_environment, mode): + initial_flags = floating_environment.flags() + value = mean(results) + flags = floating_environment.flags() + actual_mode = floating_environment.rounding() + assert value == 0.0 + assert math.copysign(1.0, value) == 1.0 + assert tuple(events) == _ORDER[:9 + position] + assert flags == initial_flags + assert actual_mode == floating_environment.expected_mode(mode) + + +@pytest.mark.parametrize('mode', range(4)) +def test_raw_native_snapshot_aggregation_retains_c_exponential_semantics( + floating_environment: Any, monkeypatch: pytest.MonkeyPatch, mode: int, +) -> None: + """The Python-only overflow translation must not alter raw C/Rust infinity results.""" + from benchset import _aggregate # noqa: I001, PLC0415 — Only the new raw protocol control needs its layout. + + class Input(ctypes.Structure): + _fields_ = [('benchmark', ctypes.c_uint32), ('status', ctypes.c_uint32), # noqa: RUF012 + ('pi_equivalent', ctypes.c_double), ('has_pi_equivalent', ctypes.c_uint32)] + + native = prepared_native(monkeypatch) + inputs = (Input * 2)(Input(1, 0, sys.float_info.max, 1), Input(2, 0, sys.float_info.max, 1)) + scores = _aggregate._Scores() + with rounding(floating_environment, mode): + expected = floating_environment.geometric(sys.float_info.max, sys.float_info.max) + assert native.lib.benchset_aggregate(inputs, 2, ctypes.byref(scores)) == 1 + actual_mode = floating_environment.rounding() + assert scores.has_pi_equivalent == 1 + assert scores.pi_equivalent == expected + assert actual_mode == floating_environment.expected_mode(mode) + + +def request_exponential(monkeypatch: pytest.MonkeyPatch) -> None: + """Drive the real Python reader directly, isolating its new exp transport field.""" + from benchset import _aggregate # noqa: PLC0415 + + def calculate(_names: object, _count: int, reader: Any, user: object, _out: object, _host: int) -> int: + request = _aggregate._Input() + request.pi_equivalent = 1500.0 + assert reader(0, 7, ctypes.byref(request), user) == 0 + return 1 + + native = _native.Native.__new__(_native.Native) + cast(Any, native).lib = SimpleNamespace(benchset_aggregate_read=calculate) + monkeypatch.setattr(_native, 'Native', lambda: native) + mean((BenchmarkResult('single_core'),)) + + +@pytest.mark.parametrize('error_type', [OverflowError, RuntimeError, KeyboardInterrupt, SystemExit, GeneratorExit]) +def test_exponential_host_errors_preserve_exact_identity_and_original_context( + monkeypatch: pytest.MonkeyPatch, error_type: Type[BaseException], +) -> None: + """The exp primitive transports every BaseException unchanged across the C callback.""" + original = error_type('host exponential failed') + + def exponentiate(value: float) -> float: + assert value == 1500.0 + raise original + + monkeypatch.setattr(math, 'exp', exponentiate) + try: + raise LookupError('caller context') + except LookupError as context: + with pytest.raises(error_type, match='host exponential failed') as caught: + request_exponential(monkeypatch) + assert caught.value is original + assert caught.value.__context__ is context # noqa: PT017 — Preserve the active caller exception state. + assert caught.value.__cause__ is None + + +def test_exponential_error_does_not_retain_its_own_callback_traceback_without_gc( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Dropping an exp failure must release its callback frames without cyclic collection.""" + class Error(RuntimeError): + pass + + errors: List[weakref.ReferenceType[Error]] = [] + + def make_error() -> Error: + error = Error('unretained exp failure') + errors.append(weakref.ref(error)) + return error + + def exponentiate(_value: float) -> float: + raise make_error() + + monkeypatch.setattr(math, 'exp', exponentiate) + enabled = gc.isenabled() + gc.disable() + try: + try: + request_exponential(monkeypatch) + except Error: + pass + assert len(errors) == 1 + assert errors[0]() is None + finally: + if enabled: + gc.enable() diff --git a/tests/test_aggregate_getters.py b/tests/test_aggregate_getters.py new file mode 100644 index 0000000..a60b0a3 --- /dev/null +++ b/tests/test_aggregate_getters.py @@ -0,0 +1,151 @@ +"""Preserve staged public ratio reads instead of caching caller-owned fields.""" + +import math +from typing import Any, List, Optional, Tuple, Type, Union + +import pytest + +from benchset import BenchmarkResult, SuiteResult + +_Read = Union[float, None, BaseException] +_STAGES = ('presence', 'finite', 'ge', 'zero', 'log-guard', 'log-value') +_ORDER = ( + 'first:success', 'first:presence', 'first:finite', 'first:ge', + 'second:success', 'second:presence', 'second:finite', 'second:ge', + 'first:zero', 'second:zero', 'first:log-guard', 'first:log-value', + 'second:log-guard', 'second:log-value', +) + + +def _result( + name: str, label: str, values: Tuple[_Read, ...], events: List[str], *, status: str = 'completed', +) -> BenchmarkResult: + reads = 0 + + class Result(BenchmarkResult): + @property + def success(self) -> bool: + events.append(label + ':success') + return self.status == 'completed' + + def __getattribute__(self, attribute: str) -> Any: + nonlocal reads + if attribute == 'pi_equivalent': + assert reads < len(values), 'an unused ratio getter was evaluated' + events.append(label + ':' + _STAGES[reads]) + value = values[reads] + reads += 1 + if isinstance(value, BaseException): + raise value + return value + return super().__getattribute__(attribute) + + return Result(name, status=status, pi_equivalent=1.0) + + +def _aggregate(first: BenchmarkResult, second: BenchmarkResult) -> Optional[float]: + return SuiteResult.from_results((first, second)).aggregated_scores.pi_equivalent + + +def test_ratio_getters_keep_validation_zero_and_logarithm_stages_in_order() -> None: + """Each public read remains observable and only the final value enters log. + + HEAD reads presence, finiteness and >= separately for each result, scans + every zero before logging, then independently reads the None guard and log + operand. Distinct ordinary floats expose any cached-value substitution. + """ + events: List[str] = [] + first = _result('single_core', 'first', (11.0, 17.0, 19.0, 23.0, 29.0, 4.0), events) + second = _result('multi_core', 'second', (31.0, 37.0, 41.0, 43.0, 47.0, 9.0), events) + assert _aggregate(first, second) == pytest.approx(6.0) + assert tuple(events) == _ORDER + + +@pytest.mark.parametrize('position', [0, 1]) +def test_later_zero_getter_controls_the_aggregate_and_hides_all_logs(position: int) -> None: + """A zero discovered by the later zero scan overrides earlier positive reads.""" + events: List[str] = [] + values = [(1.0, 1.0, 1.0, 1.0), (4.0, 4.0, 4.0, 4.0)] + values[position] = (*values[position][:3], 0.0) + first = _result('single_core', 'first', values[0], events) + second = _result('multi_core', 'second', values[1], events) + assert _aggregate(first, second) == 0.0 + assert tuple(events) == _ORDER[:9 + position] + + +@pytest.mark.parametrize('skip', [(0,), (1,), (0, 1)]) +def test_none_log_guard_skips_only_its_own_log_value(skip: Tuple[int, ...]) -> None: + """A field may become None after validation and zero checks without a sixth read. + + The original log generator skips that term while retaining the full suite + denominator. Both absent guards therefore produce exp(0 / 2), or one. + """ + events: List[str] = [] + values: List[Tuple[_Read, ...]] = [(4.0,) * 6, (9.0,) * 6] + for position in skip: + values[position] = (*values[position][:4], None) + first = _result('single_core', 'first', values[0], events) + second = _result('multi_core', 'second', values[1], events) + expected = math.exp(sum(math.log(number) for position, number in enumerate((4.0, 9.0)) + if position not in skip) / 2) + assert _aggregate(first, second) == pytest.approx(expected) + omitted = {('first' if position == 0 else 'second') + ':log-value' for position in skip} + assert events == [event for event in _ORDER if event not in omitted] + + +@pytest.mark.parametrize('position', [0, 1]) +@pytest.mark.parametrize('stage', _STAGES) +@pytest.mark.parametrize('error_type', [ValueError, RuntimeError, OverflowError, OSError, + KeyboardInterrupt, SystemExit, GeneratorExit]) +def test_ratio_getter_exceptions_keep_identity_at_every_original_read( + position: int, stage: str, error_type: Type[BaseException], +) -> None: + """Every staged getter can raise; no cache may move, hide or replace its error.""" + events: List[str] = [] + original = error_type('ratio getter failed at ' + stage) + values: List[Tuple[_Read, ...]] = [(2.0,) * 6, (2.0,) * 6] + changed = list(values[position]) + changed[_STAGES.index(stage)] = original + values[position] = tuple(changed) + first = _result('single_core', 'first', values[0], events) + second = _result('multi_core', 'second', values[1], events) + with pytest.raises(error_type, match='ratio getter failed at ' + stage) as caught: + _aggregate(first, second) + assert caught.value is original + boundary = ('first' if position == 0 else 'second') + ':' + stage + assert tuple(events) == _ORDER[:_ORDER.index(boundary) + 1] + + +@pytest.mark.parametrize(('case', 'expected_events'), [ + ('incomplete', ()), + ('reordered', ()), + ('failed-first', _ORDER[:1]), + ('failed-second', _ORDER[:5]), + ('missing-first', _ORDER[:2]), + ('missing-second', _ORDER[:6]), + ('nonfinite-first', _ORDER[:3]), + ('nonfinite-second', _ORDER[:7]), + ('negative-first', _ORDER[:4]), + ('negative-second', _ORDER[:8]), +]) +def test_rejection_hides_every_unused_ratio_getter(case: str, expected_events: Tuple[str, ...]) -> None: + """Structural/status/presence/finite/sign rejection stops at its original read. + + Each result supplies only the values HEAD actually needs; an additional + getter raises immediately instead of silently returning a convenient value. + """ + events: List[str] = [] + values: List[Tuple[_Read, ...]] = [(1.0, 1.0, 1.0), (4.0, 4.0, 4.0)] + if case.startswith('missing'): + values[0 if case.endswith('first') else 1] = (None,) + elif case.startswith('nonfinite'): + values[0 if case.endswith('first') else 1] = (1.0, math.nan) + elif case.startswith('negative'): + values[0 if case.endswith('first') else 1] = (1.0, 1.0, -1.0) + first = _result('single_core', 'first', values[0], events, + status='failed' if case == 'failed-first' else 'completed') + second = _result('multi_core', 'second', values[1], events, + status='failed' if case == 'failed-second' else 'completed') + results = (first,) if case == 'incomplete' else ((second, first) if case == 'reordered' else (first, second)) + assert SuiteResult.from_results(results).aggregated_scores.pi_equivalent is None + assert tuple(events) == expected_events diff --git a/tests/test_aggregate_memory.py b/tests/test_aggregate_memory.py new file mode 100644 index 0000000..003dc0a --- /dev/null +++ b/tests/test_aggregate_memory.py @@ -0,0 +1,51 @@ +"""Bound the aggregation bridge's scratch storage without cyclic collection.""" + +import weakref +from typing import Any, Callable, List, Optional, Type + +import pytest + +from benchset import BenchmarkResult, SuiteResult, _native +from tests.test_resources import ( + without_cyclic_gc, # noqa: F401 — Shared pytest fixture. +) + + +@pytest.mark.usefixtures('without_cyclic_gc') +def test_aggregate_scratch_type_does_not_grow_with_suite_length(monkeypatch: pytest.MonkeyPatch) -> None: + """Different incomplete tuple lengths must not leave a distinct cyclic ctypes class per call. + + Reuse one prepared Native to isolate scratch types from DLL wrappers. The + borrowed pointer may use a permanent scalar or fixed array type; only the + number of distinct surviving types must stay bounded. HEAD uses no bridge. + """ + native = _native.Native() + calculate = native.lib.benchset_aggregate_read + observed: List[Callable[[], Optional[Type[Any]]]] = [] + + def inspect( # noqa: PLR0913 — Mirrors the native reader entry point. + names: Any, count: int, reader: Any, user: Any, out: Any, host_names: int, + ) -> int: + observed.append(weakref.ref(type(names))) + return int(calculate(names, count, reader, user, out, host_names)) + + monkeypatch.setattr(native.lib, 'benchset_aggregate_read', inspect) + monkeypatch.setattr(_native, 'Native', lambda: native) + first = BenchmarkResult('single_core', status='completed', pi_equivalent=1) + second = BenchmarkResult('multi_core', status='completed', pi_equivalent=4) + for count in range(3, 43): + results = (first, second) + (first,) * (count - 2) + assert SuiteResult.from_results(results).aggregated_scores.pi_equivalent is None + retained_types = {reference() for reference in observed if reference() is not None} + assert len(retained_types) <= 1, 'scratch types must not accumulate for each distinct input length' + + +@pytest.mark.parametrize('count', [0, 1, 2, 3, 8, 64]) +def test_aggregate_scratch_capacity_does_not_replace_result_count(count: int) -> None: + """A fixed native pointer must still carry the actual tuple count, including empty and complete suites.""" + first = BenchmarkResult('single_core', status='completed', pi_equivalent=1) + second = BenchmarkResult('multi_core', status='completed', pi_equivalent=4) + results = (first, second)[:count] if count <= 2 else (first, second) + (first,) * (count - 2) + suite = SuiteResult.from_results(results) + assert suite.results is results + assert suite.aggregated_scores.pi_equivalent == (2 if count == 2 else None) diff --git a/tests/test_aggregate_protocols.py b/tests/test_aggregate_protocols.py new file mode 100644 index 0000000..0f30c34 --- /dev/null +++ b/tests/test_aggregate_protocols.py @@ -0,0 +1,321 @@ +"""Characterize float-subclass operators at the public aggregate boundary.""" + +import math +from typing import Any, Callable, List, Optional, Tuple, Type + +import pytest + +from benchset import BenchmarkResult, BenchmarkRunner, SuiteResult, _native + + +def ratio( + number: float, name: str, calls: List[str], *, + ge: Optional[Callable[[], bool]] = None, eq: Optional[Callable[[], bool]] = None, +) -> float: + class Ratio(float): + __slots__ = () + __hash__ = float.__hash__ + + def __ge__(self, other: float) -> bool: + assert type(other) is int + assert other == 0 + calls.append(name + ':ge') + return float.__ge__(self, other) if ge is None else ge() + + def __eq__(self, other: object) -> bool: + assert type(other) is int + assert other == 0 + calls.append(name + ':eq') + return float.__eq__(self, other) if eq is None else eq() + + return Ratio(number) + + +def aggregate(first: float, second: float) -> Optional[float]: + values = (BenchmarkResult('single_core', status='completed', pi_equivalent=first), + BenchmarkResult('multi_core', status='completed', pi_equivalent=second)) + return SuiteResult.from_results(values).aggregated_scores.pi_equivalent + + +@pytest.mark.parametrize(('ge', 'eq', 'expected'), [(False, False, None), (True, True, 0.0)]) +def test_float_operators_control_validation_and_zero(ge: bool, eq: bool, expected: Optional[float]) -> None: + """The original >= and == operators, with integer zero, decide rejection and zero.""" + calls: List[str] = [] + first = ratio(1, 'first', calls, ge=lambda: ge, eq=lambda: eq) + assert aggregate(first, 4) == expected + assert calls == (['first:ge', 'first:eq'] if ge else ['first:ge']) + + +@pytest.mark.parametrize('number', [math.nan, math.inf, -math.inf]) +def test_nonfinite_float_skips_overridden_operators(number: float) -> None: + """math.isfinite rejects a nonfinite float before either overridden operator runs.""" + calls: List[str] = [] + assert aggregate(ratio(number, 'first', calls), 4) is None + assert calls == [] + + +@pytest.mark.parametrize(('stage', 'error_type'), [ + (stage, error_type) for stage in ('ge', 'eq') + for error_type in (ValueError, RuntimeError, OverflowError, KeyboardInterrupt, SystemExit, GeneratorExit) +]) +def test_float_operator_exceptions_keep_identity(stage: str, error_type: Type[BaseException]) -> None: + """Operator exceptions return from the C callback as the exact original host object.""" + original = error_type('custom float operator failed') + calls: List[str] = [] + + def fail() -> bool: + raise original + + value = ratio(1, 'first', calls, **{stage: fail}) + with pytest.raises(error_type, match='custom float operator failed') as caught: + aggregate(value, 4) + assert caught.value is original + assert calls == (['first:ge'] if stage == 'ge' else ['first:ge', 'first:eq']) + + +@pytest.mark.parametrize('second_valid', [False, True]) +def test_zero_operators_follow_complete_validation_and_short_circuit(second_valid: bool) -> None: + """Validate both coefficients before zero checks, then stop after the first true equality.""" + calls: List[str] = [] + + def forbidden() -> bool: + raise AssertionError('later zero operator must not run') + + first = ratio(1, 'first', calls, eq=lambda: True) + second = ratio(4, 'second', calls, ge=lambda: second_valid, eq=forbidden) + assert aggregate(first, second) == (0 if second_valid else None) + assert calls == ['first:ge', 'second:ge'] + (['first:eq'] if second_valid else []) + + +@pytest.mark.parametrize('number', [-1.0, 0.0, -0.0]) +def test_custom_operators_preserve_nonpositive_log_domain_errors(number: float) -> None: + """A nonpositive number accepted by custom operators still raises math.log's ValueError.""" + calls: List[str] = [] + with pytest.raises(ValueError, match=r'.+') as expected: + math.log(number) + first = ratio(number, 'first', calls, ge=lambda: True, eq=lambda: False) + with pytest.raises(ValueError, match=r'.+') as caught: + aggregate(first, ratio(4, 'second', calls)) + assert type(caught.value) is type(expected.value) + assert str(caught.value) == str(expected.value) + assert calls == ['first:ge', 'second:ge', 'first:eq', 'second:eq'] + + +def test_later_custom_zero_skips_an_earlier_nonpositive_log() -> None: + """The complete zero scan hides an earlier invalid logarithm when a later equality is true.""" + calls: List[str] = [] + first = ratio(-1, 'first', calls, ge=lambda: True, eq=lambda: False) + second = ratio(4, 'second', calls, eq=lambda: True) + assert aggregate(first, second) == 0 + assert calls == ['first:ge', 'second:ge', 'first:eq', 'second:eq'] + + +@pytest.mark.parametrize(('matches', 'count'), [(False, 2), (True, 2), (True, 1), (True, 3)]) +def test_aggregate_names_compare_lazily_at_their_positions(matches: bool, count: int) -> None: + """Tuple equality compares the shared prefix before rejecting a different length.""" + calls: List[str] = [] + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + calls.append(str(other)) + return matches + + names = [Name('single_core'), Name('multi_core'), Name('extra')][:count] + values = tuple(BenchmarkResult(name, status='completed', pi_equivalent=2) for name in names) + value = SuiteResult.from_results(values).aggregated_scores.pi_equivalent + assert value == (2 if count == 2 and matches else None) + assert calls == (['single_core', 'multi_core'][:min(count, 2)] if matches else ['single_core']) + + +def test_aggregate_names_can_match_both_positions_without_matching_their_bytes() -> None: + """A global first matching ID cannot replace positional tuple equality for subclass names.""" + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + return True + + values = (BenchmarkResult(Name('anything'), status='completed', pi_equivalent=1), + BenchmarkResult(Name('anything'), status='completed', pi_equivalent=4)) + assert SuiteResult.from_results(values).aggregated_scores.pi_equivalent == 2 + + +@pytest.mark.parametrize('error_type', [ValueError, RuntimeError, OverflowError, OSError, + KeyboardInterrupt, SystemExit, GeneratorExit]) +@pytest.mark.parametrize('factory', ['aggregate', 'run']) +def test_name_comparison_exceptions_keep_identity(error_type: Type[BaseException], factory: str) -> None: + """Name validation preserves the original comparison exception, including runner validation.""" + original = error_type('custom name comparison failed') + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + raise original + + def invoke() -> None: + if factory == 'aggregate': + values = (BenchmarkResult(Name('single_core'), status='completed', pi_equivalent=1), + BenchmarkResult('multi_core', status='completed', pi_equivalent=4)) + SuiteResult.from_results(values) + else: + BenchmarkRunner().run(Name('single_core'), warmup_seconds=0, measurement_seconds=0.01) + + with pytest.raises(error_type, match='custom name comparison failed') as caught: + invoke() + assert caught.value is original + + +@pytest.mark.parametrize('matches', [False, True]) +def test_runner_name_comparisons_keep_validation_and_execution_order(matches: bool) -> None: + """Membership validates first; execution compares single_core once and never hashes the name.""" + calls: List[str] = [] + + class Name(str): + __slots__ = () + + def __eq__(self, other: object) -> bool: + calls.append(str(other)) + return matches + + def __hash__(self) -> int: + raise AssertionError('benchmark lookup must not hash caller names') + + runner = BenchmarkRunner() + result = runner.run(Name('multi_core'), warmup_seconds=0, measurement_seconds=0.01, + on_progress=lambda _progress: runner.cancel()) + assert calls == (['single_core', 'single_core'] if matches else ['single_core', 'multi_core']) + assert result.status == ('cancelled' if matches else 'failed') + if matches: + assert result.workers == 1 + else: + assert result.reason == 'unknown benchmark: multi_core' + + +def test_runner_execution_compares_once_before_multicore_fallback() -> None: + """A stateful name accepted in validation reaches the old single comparison/fallback execution path.""" + calls: List[str] = [] + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + calls.append(str(other)) + if len(calls) > 2: + raise AssertionError('execution must not compare a second benchmark') + return len(calls) == 1 + + runner = BenchmarkRunner() + result = runner.run(Name('multi_core'), warmup_seconds=0, measurement_seconds=0.01, + on_progress=lambda _progress: runner.cancel()) + assert calls == ['single_core', 'single_core'] + assert result.status == 'cancelled' + + +@pytest.mark.parametrize('error_type', [RuntimeError, OverflowError, OSError, ValueError]) +def test_execution_name_errors_keep_runner_failure_handling(error_type: Type[Exception]) -> None: + """Execution comparisons retain the runner's existing caught/propagated exception boundary.""" + original = error_type('execution name failed') + calls: List[str] = [] + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + calls.append(str(other)) + if len(calls) == 1: + return True + raise original + + runner = BenchmarkRunner() + if error_type is ValueError: + with pytest.raises(ValueError, match='execution name failed') as caught: + runner.run(Name('single_core'), warmup_seconds=0, measurement_seconds=0.01) + assert caught.value is original + else: + result = runner.run(Name('single_core'), warmup_seconds=0, measurement_seconds=0.01) + assert result.status == 'failed' + assert result.reason == 'execution name failed' + assert calls == ['single_core', 'single_core'] + + +@pytest.mark.parametrize('case', ['complete', 'extra', 'first-mismatch', 'comparison-error']) +def test_aggregate_materializes_all_name_attributes_before_comparing(case: str) -> None: + """The factory snapshots every name before tuple equality can reject or raise.""" + calls: List[str] = [] + original = ValueError('name comparison failed') + + def name(text: str, label: str) -> str: + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + calls.append('eq:' + label) + if case == 'comparison-error': + raise original + return False if case == 'first-mismatch' else str.__eq__(self, other) + + return Name(text) + + class Result(BenchmarkResult): + def __getattribute__(self, attribute: str) -> Any: + if attribute == 'benchmark': + calls.append('get:' + object.__getattribute__(self, 'reason')) + return object.__getattribute__(self, attribute) + + values: Tuple[BenchmarkResult, ...] = ( + Result(name('single_core', 'first'), reason='first', status='completed', pi_equivalent=2), + Result(name('multi_core', 'second'), reason='second', status='completed', pi_equivalent=2), + ) + if case == 'extra': + values += (Result(name('extra', 'third'), reason='third', status='completed', pi_equivalent=2),) + if case == 'comparison-error': + with pytest.raises(ValueError, match='name comparison failed') as caught: + SuiteResult.from_results(values) + assert caught.value is original + else: + suite = SuiteResult.from_results(values) + assert suite.aggregated_scores.pi_equivalent == (2 if case == 'complete' else None) + expected = ['get:first', 'get:second'] + (['get:third'] if case == 'extra' else []) + expected += ['eq:first'] if case in ('first-mismatch', 'comparison-error') else ['eq:first', 'eq:second'] + assert calls == expected + + +@pytest.mark.parametrize('position', [0, 1]) +@pytest.mark.parametrize('error_type', [ValueError, RuntimeError, OverflowError, + KeyboardInterrupt, SystemExit, GeneratorExit]) +def test_aggregate_name_getter_errors_precede_native_loading( + monkeypatch: pytest.MonkeyPatch, position: int, error_type: Type[BaseException], +) -> None: + """Name snapshots preserve getter exception identity before native availability matters.""" + calls: List[str] = [] + original = error_type('name getter failed') + + class Result(BenchmarkResult): + def __getattribute__(self, attribute: str) -> Any: + if attribute == 'benchmark': + label = object.__getattribute__(self, 'reason') + calls.append('get:' + label) + if label == str(position): + raise original + return object.__getattribute__(self, attribute) + + def unavailable() -> None: + calls.append('native') + raise OSError('native loader failed') + + monkeypatch.setattr(_native, 'Native', unavailable) + values = (Result('single_core', reason='0', status='completed', pi_equivalent=1), + Result('multi_core', reason='1', status='completed', pi_equivalent=4)) + with pytest.raises(error_type, match='name getter failed') as caught: + SuiteResult.from_results(values) + assert caught.value is original + assert calls == ['get:' + str(index) for index in range(position + 1)] diff --git a/tests/test_aggregate_tuple_inputs.py b/tests/test_aggregate_tuple_inputs.py new file mode 100644 index 0000000..6e958d0 --- /dev/null +++ b/tests/test_aggregate_tuple_inputs.py @@ -0,0 +1,116 @@ +"""Aggregate actual tuple contents independently of overridden tuple protocols.""" + +import sys +from typing import Any, List, Tuple + +import pytest + +from benchset import BenchmarkResult, SuiteResult + + +def _valid_results() -> Tuple[BenchmarkResult, ...]: + return (BenchmarkResult('single_core', status='completed', pi_equivalent=4.0), + BenchmarkResult('multi_core', status='completed', pi_equivalent=9.0)) + + +@pytest.mark.parametrize('protocol', [ + 'getitem-error', 'getitem-lie', 'len-error', 'len-zero', 'len-one', 'len-large', + 'iter-error', 'iter-empty', 'iter-wrong', 'iter-changing', +]) +def test_tuple_subclass_protocols_cannot_change_its_actual_aggregate(protocol: str) -> None: + """Use actual tuple elements and length, as explicitly chosen by the caller. + + Tuple subclasses can override iteration, indexing and length independently. + None of those hooks may run during aggregation: lying values must not alter + a genuine complete suite, and raising hooks must not replace its score. + Keep the caller's original object in SuiteResult.results. + """ + calls: List[str] = [] + fake = (BenchmarkResult('wrong', status='completed', pi_equivalent=100.0),) + + class Results(tuple): + __slots__ = () + + def __getitem__(self, index: Any) -> Any: + if protocol.startswith('getitem'): + calls.append('getitem') + if protocol == 'getitem-error': + raise ValueError('overridden tuple indexing must not run') + return fake[0] + return tuple.__getitem__(self, index) + + def __len__(self) -> int: + if protocol.startswith('len'): + calls.append('len') + if protocol == 'len-error': + raise ValueError('overridden tuple length must not run') + return {'len-zero': 0, 'len-one': 1, 'len-large': sys.maxsize}[protocol] + return tuple.__len__(self) + + def __iter__(self) -> Any: + if protocol.startswith('iter'): + calls.append('iter') + if protocol == 'iter-error': + raise ValueError('overridden tuple iteration must not run') + if protocol == 'iter-empty': + return iter(()) + if protocol == 'iter-changing' and len(calls) > 1: + return tuple.__iter__(self) + return iter(fake) + return tuple.__iter__(self) + + results = Results(_valid_results()) + suite = SuiteResult.from_results(results) + assert suite.results is results + assert suite.aggregated_scores.pi_equivalent == 6.0 + assert calls == [] + + +@pytest.mark.parametrize('contents', ['empty', 'short', 'extra', 'reversed', 'wrong-name', 'failed', 'missing-ratio']) +def test_tuple_subclass_cannot_hide_invalid_actual_contents_with_valid_protocols(contents: str) -> None: + """An invalid actual tuple remains unavailable even when all hooks fake a suite. + + Reusing an overridden iterator's snapshot would silently validate fictional + results. Rust must receive the actual empty, incomplete, reordered, failed + or coefficient-less tuple while retaining the original public container. + """ + valid = _valid_results() + actual = { + 'empty': (), + 'short': valid[:1], + 'extra': (*valid, valid[0]), + 'reversed': valid[::-1], + 'wrong-name': (BenchmarkResult('wrong', status='completed', pi_equivalent=4.0), valid[1]), + 'failed': (BenchmarkResult('single_core', status='failed', pi_equivalent=4.0), valid[1]), + 'missing-ratio': (BenchmarkResult('single_core', status='completed'), valid[1]), + }[contents] + calls: List[str] = [] + + class Results(tuple): + __slots__ = () + + def __getitem__(self, index: Any) -> Any: + calls.append('getitem') + return valid[index] + + def __len__(self) -> int: + calls.append('len') + return len(valid) + + def __iter__(self) -> Any: + calls.append('iter') + return iter(valid) + + results = Results(actual) + suite = SuiteResult.from_results(results) + assert suite.results is results + assert suite.aggregated_scores.pi_equivalent is None + assert calls == [] + + +def test_normalizing_tuple_inputs_does_not_reject_existing_list_inputs() -> None: + """Restrict the new protocol bypass to tuples without narrowing list behavior.""" + results = list(_valid_results()) + suite = SuiteResult.from_results(results) # type: ignore[arg-type] + assert suite.results is results + assert suite.aggregated_scores.pi_equivalent == 6.0 diff --git a/tests/test_aggregation_boundary.py b/tests/test_aggregation_boundary.py new file mode 100644 index 0000000..5b343dc --- /dev/null +++ b/tests/test_aggregation_boundary.py @@ -0,0 +1,161 @@ +"""Field-preserving transport for caller-built full-suite snapshots.""" + +import ctypes +import gc +import os +import shutil +import subprocess +from dataclasses import replace +from pathlib import Path +from typing import Any, List, Tuple + +import pytest + +from benchset import AggregatedScores, BenchmarkResult, SuiteResult, _aggregate, _native + + +@pytest.mark.parametrize('position', [0, 1]) +@pytest.mark.parametrize('name', ['', 'single_core\0', 'single_core\0ignored', 'много_ядер', '\ud800', 'single_core\udfff']) +def test_aggregate_does_not_alias_unknown_benchmark_names(position: int, name: str) -> None: + """Unknown names remain invalid, including UTF-8, NUL and lone surrogates. + + Length-delimited transport must never truncate an unknown string into a + valid benchmark name or silently substitute an expected suite member. + """ + results = [BenchmarkResult('single_core', status='completed', pi_equivalent=0.5), + BenchmarkResult('multi_core', status='completed', pi_equivalent=2)] + results[position] = replace(results[position], benchmark=name) + assert SuiteResult.from_results(tuple(results)).aggregated_scores.pi_equivalent is None + + +@pytest.mark.parametrize('presence', [0, 1, 7, 2**32 - 1]) +@pytest.mark.parametrize('ratio', [0.0, 8.0]) +def test_aggregate_boundary_preserves_inputs_and_native_output( + monkeypatch: pytest.MonkeyPatch, presence: int, ratio: float, +) -> None: + """Read requested public success/ratio fields and preserve the native result without recalculation. + + Presence and numeric reads stay separate, so an absent value needs no + conversion. A deliberately unrelated output and a nonstandard presence flag + expose Python-side aggregation, truthiness mistakes for zero and lossy coefficients. + """ + native = _native.Native() + received: List[Tuple[int, int, float, int]] = [] + + def calculate( # noqa: PLR0913 — Mirrors the native reader entry point. + names: Any, count: int, reader: Any, user: Any, out: Any, host_names: int, + ) -> int: + assert count == 3 + assert host_names == 1 + for index in range(count): + value = _aggregate._Input() + assert reader(index, 0, ctypes.byref(value), user) == 1 + assert reader(index, 1, ctypes.byref(value), user) == 1 + assert value.pi_equivalent == 0.0 + if value.has_pi_equivalent: + assert reader(index, 6, ctypes.byref(value), user) == 1 + received.append((names[0], value.status, value.pi_equivalent, value.has_pi_equivalent)) + scores = ctypes.cast(out, ctypes.POINTER(_aggregate._Scores)).contents + scores.pi_equivalent = 0.0 + scores.has_pi_equivalent = presence + return 1 + + monkeypatch.setattr(native.lib, 'benchset_aggregate_read', calculate) + monkeypatch.setattr(_native, 'Native', lambda: native) + values = (BenchmarkResult('single_core', status='failed', pi_equivalent=0.12345678901234567), + BenchmarkResult('multi_core', status='cancelled', pi_equivalent=None), + BenchmarkResult('unknown', status='unknown', pi_equivalent=ratio)) + suite = SuiteResult.from_results(values) + assert suite.results is values + assert received == [(0, 2, 0.12345678901234567, 1), (0, 2, 0.0, 0), (0, 2, ratio, 1)] + assert suite.aggregated_scores == AggregatedScores(0.0 if presence else None) + + +def test_aggregate_boundary_reports_unavailable_native_output(monkeypatch: pytest.MonkeyPatch) -> None: + """A rejected native getter cannot silently manufacture an absent aggregate.""" + native = _native.Native() + monkeypatch.setattr(native.lib, 'benchset_aggregate_read', lambda *_args: 0) + monkeypatch.setattr(_native, 'Native', lambda: native) + with pytest.raises(RuntimeError, match='native aggregate is unavailable'): + SuiteResult.from_results(()) + + +@pytest.mark.parametrize('symbol', ['benchset_aggregate', 'benchset_aggregate_read', 'benchset_copy_double']) +def test_aggregate_is_a_required_native_symbol_before_allocation(monkeypatch: pytest.MonkeyPatch, symbol: str) -> None: + """A same-version older library must report its missing aggregate symbol before allocating handles.""" + library = _native.Native().lib + + class OlderLibrary: + def __getattr__(self, name: str) -> Any: + if name == symbol: + raise AttributeError(name) + if name == 'benchset_create': + return forbidden + return getattr(library, name) + + def forbidden() -> None: + pytest.fail('a missing symbol must be detected before allocation') + + monkeypatch.setattr(ctypes, 'CDLL', lambda _path: OlderLibrary()) + with pytest.raises(RuntimeError, match='missing required symbol: ' + symbol): + _native.Native() + + +@pytest.mark.skipif(os.name == 'nt', reason='POSIX shared-library fenv probe; MSVC floating environment differs') +@pytest.mark.parametrize('position', [0, 1]) +@pytest.mark.parametrize('other', [0.0, 2.0]) +def test_zero_aggregate_preserves_floating_environment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, position: int, other: float, +) -> None: + """Any zero skips every logarithm, preserving caller floating-point exception flags. + + A tiny C probe supplies platform fenv macros and saves/restores the complete + environment in a finally block. An existing FE_INVALID flag must survive, + while neither log(0)'s divide-by-zero nor log(other)'s inexact flag may appear. + Native bindings are prepared first to keep loader activity outside the probe. + Disable automatic GC during measurement: CPython 3.15 collection statistics + independently set FE_INEXACT and must not be attributed to the calculator. + """ + compiler = shutil.which('cc') or shutil.which('clang') or shutil.which('gcc') + if compiler is None: + pytest.skip('No C compiler for the floating-environment probe') + source = tmp_path / 'fenv.c' + source.write_text('''#include +static fenv_t saved; +int begin(void) { + if (fegetenv(&saved) || feclearexcept(FE_ALL_EXCEPT)) return -1; + if (feraiseexcept(FE_INVALID)) return -1; + return fetestexcept(FE_ALL_EXCEPT); +} +int flags(void) { return fetestexcept(FE_ALL_EXCEPT); } +int end(void) { return fesetenv(&saved); } +''') + library = tmp_path / 'fenv.so' + built = subprocess.run([compiler, '-shared', '-fPIC', str(source), '-lm', '-o', str(library)], + capture_output=True, text=True, check=False) + assert built.returncode == 0, built.stderr + probe = ctypes.CDLL(str(library)) + for name in ('begin', 'flags', 'end'): + getattr(probe, name).argtypes = [] + getattr(probe, name).restype = ctypes.c_int + native = _native.Native() + monkeypatch.setattr(_native, 'Native', lambda: native) + results = [BenchmarkResult('single_core', status='completed', pi_equivalent=other), + BenchmarkResult('multi_core', status='completed', pi_equivalent=other)] + results[position] = replace(results[position], pi_equivalent=-0.0) + collecting = gc.isenabled() + gc.disable() + try: + initial = probe.begin() + try: + value = SuiteResult.from_results(tuple(results)).aggregated_scores.pi_equivalent + observed = probe.flags() + finally: + restored = probe.end() + finally: + if collecting: + gc.enable() + assert initial >= 0 + assert restored == 0 + assert value == 0.0 + assert observed == initial diff --git a/tests/test_approved_python_contracts.py b/tests/test_approved_python_contracts.py new file mode 100644 index 0000000..a2bc927 --- /dev/null +++ b/tests/test_approved_python_contracts.py @@ -0,0 +1,311 @@ +"""Pin explicitly chosen host conversion, comparison and diagnostic contracts.""" + +from typing import Any, List, Tuple, Type + +import pytest + +from benchset import BENCHMARKS, BenchmarkResult, BenchmarkRunner, Progress, SuiteResult +from benchset._native import Native + + +class _StringLookalike: + def __getattribute__(self, name: str) -> Any: + if name == '__class__': + return str + return super().__getattribute__(name) + + +class _Diagnostic(str): + __slots__ = () + + def __str__(self) -> str: + raise AssertionError('a valid diagnostic must not be coerced') + + +_Probe = Tuple[BenchmarkRunner, List[Progress], List[int], List[int]] + + +@pytest.fixture +def probe(monkeypatch: pytest.MonkeyPatch) -> _Probe: + """Observe real native allocation/release without changing validation behavior.""" + runner = BenchmarkRunner() + progress: List[Progress] = [] + created: List[int] = [] + released: List[int] = [] + original_create, original_free = Native.create, Native.free + + def create(native: Native, suite: bool = False) -> int: + handle = original_create(native, True) if suite else original_create(native) + created.append(handle) + return handle + + def free(native: Native, handle: int, suite: bool = False) -> None: + original_free(native, handle, True) if suite else original_free(native, handle) + released.append(handle) + + monkeypatch.setattr(Native, 'create', create) + monkeypatch.setattr(Native, 'free', free) + return runner, progress, created, released + + +def _assert_no_work_and_reuse(probe: _Probe) -> None: + runner, progress, created, released = probe + assert not progress + assert not created + assert not released + assert runner._active is None + assert runner._native is None + assert runner._handle is None + result = runner.run('single_core', warmup_seconds=0, measurement_seconds=.01, + on_progress=lambda _value: runner.cancel()) + assert result.status == 'cancelled' + assert len(created) == 1 + assert released == created + assert runner._active is None + + +@pytest.mark.parametrize('diagnostic', [ + pytest.param(None, id='none'), pytest.param(False, id='false'), + pytest.param(0, id='zero'), pytest.param(4.0, id='float'), + pytest.param(b'text', id='bytes'), pytest.param(['text'], id='list'), + pytest.param(_StringLookalike(), id='spoofed-str-class'), +]) +@pytest.mark.parametrize(('warmup', 'duration'), [(0.0, .01), (-1.0, 0.0)]) +def test_nonstring_diagnostic_results_raise_before_native_work( + probe: _Probe, diagnostic: object, warmup: float, duration: float, +) -> None: + """A malformed reflected formatter cannot validate or execute an unknown name. + + The chosen contract requires an actual str or str subclass, including when + a non-string spoofs __class__. Unknown-name formatting precedes duration + validation, so invalid windows cannot hide its TypeError. No handle or + progress may be created; the same runner remains usable afterward. + """ + runner, progress, _created, _released = probe + prefixes: List[str] = [] + + class Name(str): + __slots__ = () + + def __radd__(self, prefix: str) -> Any: + prefixes.append(prefix) + return diagnostic + + with pytest.raises(TypeError, match=r'^benchmark diagnostic must be a string$'): + runner.run(Name('unknown'), warmup_seconds=warmup, measurement_seconds=duration, + on_progress=progress.append) + assert prefixes == ['unknown benchmark: '] + _assert_no_work_and_reuse(probe) + + +@pytest.mark.parametrize('diagnostic', [ + pytest.param('caller diagnostic', id='str'), pytest.param('', id='empty-str'), + pytest.param(_Diagnostic('caller diagnostic'), id='str-subclass'), + pytest.param(_Diagnostic(''), id='empty-str-subclass'), +]) +@pytest.mark.parametrize(('warmup', 'duration'), [(0.0, .01), (-1.0, 0.0)]) +def test_real_string_diagnostics_keep_identity_without_coercion( + probe: _Probe, diagnostic: str, warmup: float, duration: float, +) -> None: + """Actual strings, including empty/subclass values, remain failed diagnostics. + + Type validation must neither call str(value), reject falsy strings, nor + replace a caller-owned diagnostic object. Invalid windows keep the original + unknown-name precedence, with no native work and complete runner reuse. + """ + runner, progress, _created, _released = probe + prefixes: List[str] = [] + + class Name(str): + __slots__ = () + + def __radd__(self, prefix: str) -> str: + prefixes.append(prefix) + return diagnostic + + name = Name('unknown') + result = runner.run(name, warmup_seconds=warmup, measurement_seconds=duration, + on_progress=progress.append) + assert result.status == 'failed' + assert result.benchmark is name + assert result.reason is diagnostic + assert prefixes == ['unknown benchmark: '] + _assert_no_work_and_reuse(probe) + + +@pytest.mark.parametrize('error_type', [TypeError, RuntimeError, OSError, KeyboardInterrupt, SystemExit]) +@pytest.mark.parametrize(('warmup', 'duration'), [(0.0, .01), (-1.0, 0.0)]) +def test_formatter_errors_escape_unchanged_before_duration_checks( + probe: _Probe, error_type: Type[BaseException], warmup: float, duration: float, +) -> None: + """A formatter's own error retains identity, cause and context through Rust. + + The new return-type guard must not reclassify genuine formatter exceptions, + even backend-looking errors or control exceptions. Name formatting still + takes precedence over invalid windows and leaves no native activity. + """ + runner, progress, _created, _released = probe + original = error_type('formatter failed') + context = LookupError('formatter cause') + prefixes: List[str] = [] + + class Name(str): + __slots__ = () + + def __radd__(self, prefix: str) -> str: + prefixes.append(prefix) + try: + raise context + except LookupError: + raise original from context + + with pytest.raises(error_type) as caught: + runner.run(Name('unknown'), warmup_seconds=warmup, measurement_seconds=duration, + on_progress=progress.append) + assert caught.value is original + assert original.__context__ is context + assert original.__cause__ is context + assert original.__suppress_context__ + assert prefixes == ['unknown benchmark: '] + _assert_no_work_and_reuse(probe) + + +@pytest.mark.parametrize('position', [0, 1]) +@pytest.mark.parametrize(('finite', 'log_value', 'expected'), [(4.0, 4.0, 2.0), (9.0, 4.0, 2.0), (4.0, 9.0, 3.0)]) +def test_integer_subclasses_use_each_fresh_float_conversion_for_aggregation( + position: int, finite: float, log_value: float, expected: float, +) -> None: + """The approved numeric contract uses converted values rather than int payloads. + + An int payload of 16 converts to four and therefore contributes log(4), + producing two with a unit peer. A changed second conversion proves that + the logarithm reads fresh host numeric data, independently of validation. + """ + conversions: List[float] = [] + events: List[str] = [] + + class Ratio(int): + __hash__ = int.__hash__ + + def __float__(self) -> float: + assert len(conversions) < 2 + events.append('float:' + str(len(conversions))) + value = (finite, log_value)[len(conversions)] + conversions.append(value) + return value + + def __ge__(self, other: Any) -> bool: + events.append('ge') + return int.__ge__(self, other) + + def __eq__(self, other: object) -> bool: + events.append('eq') + return int.__eq__(self, other) + + values: List[float] = [1.0, 1.0] + values[position] = Ratio(16) + results = tuple(BenchmarkResult(name, status='completed', pi_equivalent=value) + for name, value in zip(BENCHMARKS, values)) + assert SuiteResult.from_results(results).aggregated_scores.pi_equivalent == pytest.approx(expected) + assert conversions == [finite, log_value] + assert events == ['float:0', 'ge', 'eq', 'float:1'] + + +@pytest.mark.parametrize('position', [0, 1]) +@pytest.mark.parametrize('conversion', [0, 1], ids=['validation', 'logarithm']) +@pytest.mark.parametrize('error_type', [RuntimeError, KeyboardInterrupt]) +def test_integer_float_conversion_errors_keep_identity_at_both_stages( + position: int, conversion: int, error_type: Type[BaseException], +) -> None: + """Both required integer conversions remain fallible host primitives. + + Preserve exact errors rather than caching the validation value, suppressing + a logarithm conversion failure, or using the integer payload instead. + """ + original = error_type('integer conversion failed') + calls: List[int] = [] + events: List[str] = [] + + class Ratio(int): + __hash__ = int.__hash__ + + def __float__(self) -> float: + current = len(calls) + calls.append(current) + events.append('float:' + str(current)) + if current == conversion: + raise original + return 4.0 + + def __ge__(self, other: Any) -> bool: + events.append('ge') + return int.__ge__(self, other) + + def __eq__(self, other: object) -> bool: + events.append('eq') + return int.__eq__(self, other) + + values: List[float] = [1.0, 1.0] + values[position] = Ratio(16) + results = tuple(BenchmarkResult(name, status='completed', pi_equivalent=value) + for name, value in zip(BENCHMARKS, values)) + with pytest.raises(error_type) as caught: + SuiteResult.from_results(results) + assert caught.value is original + assert calls == list(range(conversion + 1)) + assert events == (['float:0'] if conversion == 0 else ['float:0', 'ge', 'eq', 'float:1']) + + +@pytest.mark.parametrize('source', ['property', 'status-comparison']) +@pytest.mark.parametrize('position', [0, 1]) +@pytest.mark.parametrize('outcome', [True, False, ValueError, KeyboardInterrupt]) +def test_success_truth_is_evaluated_once( + source: str, position: int, outcome: Any, +) -> None: + """Convert each requested success value to bool exactly once. + + Both an overridden success property and an overridden status comparison + can return a stateful truth object. Do not retry false values, bypass its + bool conversion, duplicate the property read, or swallow its exception. + """ + reads: List[str] = [] + calls: List[str] = [] + error = outcome('truth failed') if isinstance(outcome, type) else None + + class Truth: + def __bool__(self) -> bool: + calls.append('bool') + assert len(calls) == 1 + if error is not None: + raise error + return bool(outcome) + + class Result(BenchmarkResult): + @property + def success(self) -> Any: + reads.append('success') + return Truth() + + class Status(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> Any: + assert other == 'completed' + reads.append('success') + return Truth() + + results = [BenchmarkResult(name, status='completed', pi_equivalent=value) + for name, value in zip(BENCHMARKS, (4.0, 9.0))] + results[position] = (Result if source == 'property' else BenchmarkResult)( + BENCHMARKS[position], status=Status('completed'), pi_equivalent=(4.0, 9.0)[position], + ) + if error is not None: + with pytest.raises(type(error)) as caught: + SuiteResult.from_results(tuple(results)) + assert caught.value is error + else: + score = SuiteResult.from_results(tuple(results)).aggregated_scores.pi_equivalent + assert score == (pytest.approx(6.0) if outcome else None) + assert reads == ['success'] + assert calls == ['bool'] diff --git a/tests/test_build_native.py b/tests/test_build_native.py index 8705d67..323de82 100644 --- a/tests/test_build_native.py +++ b/tests/test_build_native.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json +import os import runpy import shutil import subprocess @@ -28,6 +30,7 @@ def test_developer_build_copies_the_fresh_host_library_after_successful_cargo( during copying must still see the previous library, never partial bytes; interrupted copies must leave that library intact and remove staging files. Custom Cargo output directories must supply fresh bytes, never stale target/. + The matching Rust-generated metadata must also reach the source package. The copied entry point keeps its real relative-path calculations intact. """ system, filename = configuration @@ -46,13 +49,17 @@ def test_developer_build_copies_the_fresh_host_library_after_successful_cargo( monkeypatch.setattr(sys, 'platform', system) def compile_(arguments: list[str], **options: Any) -> subprocess.CompletedProcess[str]: - assert arguments == ['cargo', 'build', '--release', '--locked'] - assert options == {'cwd': tmp_path, 'check': True} + assert arguments == ['cargo', 'build', '--release', '--locked', '--message-format=json-render-diagnostics'] + assert options == {'cwd': tmp_path, 'check': True, 'env': None, 'stdout': subprocess.PIPE, 'encoding': 'utf-8'} assert synchronized[-1] == tmp_path library = output / 'release' / filename library.parent.mkdir(parents=True, exist_ok=True) library.write_bytes(b'fresh native library') - return subprocess.CompletedProcess(arguments, 0) + metadata = output / 'reported-build-script-output/metadata.json' + metadata.parent.mkdir(parents=True, exist_ok=True) + metadata.write_text('{"fresh": true}\n') + message = {'reason': 'build-script-executed', 'env': [['BENCHSET_METADATA_PATH_HEX', os.fsencode(metadata).hex()]]} + return subprocess.CompletedProcess(arguments, 0, stdout=json.dumps(message) + '\n') monkeypatch.setattr(subprocess, 'run', compile_) copy = shutil.copy2 @@ -71,6 +78,7 @@ def copy_with_reader(source: Path, destination: Path) -> Path: previous = installed.read_bytes() runpy.run_path(str(script), run_name='__main__') assert installed.read_bytes() == b'fresh native library' + assert (tmp_path / 'benchset/_metadata.json').read_text() == '{"fresh": true}\n' copy_fails = True with pytest.raises(OSError, match='copy interrupted'): runpy.run_path(str(script), run_name='__main__') diff --git a/tests/test_cli.py b/tests/test_cli.py index efc88d1..a5712d1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,6 +4,7 @@ import io import locale import os +import select import signal import subprocess import sys @@ -25,6 +26,138 @@ __version__, cli, ) +from tests.helpers import mock_suite + + +@pytest.mark.skipif(os.name == 'nt', reason='POSIX pipe backpressure and subprocess SIGINT') +@pytest.mark.parametrize(('delivery', 'operation'), [ + ('process', 'render'), ('process', 'flush'), ('process', 'separator'), ('process', 'pending'), + ('eligible-worker', 'render'), ('eligible-worker', 'flush'), ('eligible-worker', 'separator'), +]) +def test_sigint_interrupts_blocked_intermediate_output(operation: str, delivery: str) -> None: + """Ctrl+C interrupts every intermediate output operation before native is acknowledged. + + Fill a real pipe and keep its reader open without draining it. A proxy routes + exactly the selected output operation into that pipe; stderr synchronizes + SIGINT delivery. Unbuffered child output isolates this from shutdown flushes. + A signal pending before handler restoration must bypass stdout altogether. + The adversarial route selects the waiting worker when its mask permits it, + which POSIX also allows for process-directed SIGINT. A helper waits for the + main thread to enter write before delivery. It observes eligibility both + before publication and while waiting, detecting a late block or early + restore without depending on the kernel choosing a particular thread. + """ + reader, writer = os.pipe() + command_reader, command_writer = os.pipe() + process = None + try: + os.set_blocking(writer, False) + try: + while True: + os.write(writer, b'x' * 4096) + except BlockingIOError: + pass + os.set_blocking(writer, True) + program = r''' +import signal +import sys +from benchset import cli + +output = sys.stdout +operation = sys.argv[1] +def blocked_write(): + sys.stderr.write('blocked\n') + sys.stderr.flush() + output.write('x') + +class Output: + encoding = 'utf-8' + def isatty(self): return False + def write(self, text): + if operation == 'render' or (operation in ('separator', 'pending') and text == '\n'): + blocked_write() + return len(text) + def flush(self): + if operation == 'flush': blocked_write() + +previous = signal.getsignal(signal.SIGINT) +if operation == 'pending': + install = signal.signal + fired = [] + def deliver_before_restore(signum, handler): + if (handler is previous or getattr(handler, '__name__', '') == 'interrupt_output') and not fired: + fired.append(True) + signal.raise_signal(signal.SIGINT) + return install(signum, handler) + signal.signal = deliver_before_restore +sys.stdout = Output() +try: + status = cli.main(['--warmup=0', '--duration=.1']) +finally: + sys.stdout = output +assert signal.getsignal(signal.SIGINT) is previous +raise SystemExit(status) +''' + if delivery == 'eligible-worker': + routing = r''' +import os, queue, threading, time +parked, waiting = threading.Event(), [] +class Messages(queue.Queue): + def put(self, *args, **kwargs): + waiting[:] = [threading.get_ident(), signal.SIGINT not in signal.pthread_sigmask(signal.SIG_BLOCK, [])] + return super().put(*args, **kwargs) +class Acknowledgement(threading.Event): + def wait(self, timeout=None): + waiting[1] |= signal.SIGINT not in signal.pthread_sigmask(signal.SIG_BLOCK, []) + parked.set() + return super().wait(timeout) +cli.Queue, cli.ResultEvent = Messages, Acknowledgement +def deliver(): + signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGINT]) + if not os.read(int(sys.argv[2]), 1): return + assert parked.wait(5), 'observer never waited for acknowledgement' + main = threading.main_thread().ident + deadline = time.monotonic() + 5 + while True: + frame = sys._current_frames().get(main) + if frame and frame.f_code is blocked_write.__code__ and frame.f_lineno == blocked_write.__code__.co_firstlineno + 3: + break + assert time.monotonic() < deadline, 'main thread never entered blocked write' + time.sleep(.001) + signal.pthread_kill(waiting[0] if waiting[1] else main, signal.SIGINT) +threading.Thread(target=deliver, daemon=True).start() +''' + program = program.replace('output = sys.stdout', routing + '\noutput = sys.stdout', 1) + process = subprocess.Popen([sys.executable, '-u', '-c', program, operation, str(command_reader)], + pass_fds=(command_reader,), stdout=writer, stderr=subprocess.PIPE, text=True) + os.close(writer) + writer = -1 + assert process.stderr is not None + if operation == 'pending': + assert process.wait(timeout=CLI_PROCESS_TIMEOUT) == 130 + diagnostics = process.stderr.read() + assert 'single_core' in diagnostics + assert 'Full Pi equivalent :' in diagnostics + assert 'blocked' not in diagnostics + return + assert select.select([process.stderr], [], [], CLI_PROCESS_TIMEOUT)[0], 'output operation was never reached' + assert process.stderr.readline() == 'blocked\n' + if delivery == 'eligible-worker': + os.write(command_writer, b'!') + else: + process.send_signal(signal.SIGINT) + assert process.wait(timeout=5) == 130 + assert not process.stderr.read() + finally: + if process is not None: + if process.poll() is None: + process.kill() + process.communicate() + if writer >= 0: + os.close(writer) + os.close(reader) + os.close(command_reader) + os.close(command_writer) # Content checks allow slow interpreter startup on shared runners. Cancellation # and terminal-disconnect responsiveness retain their separate 5-second bounds. @@ -260,6 +393,7 @@ def execute(_self: BenchmarkRunner, benchmark: str, *, warmup_seconds: float, return BenchmarkResult(benchmark, status='completed', score=1, pi_equivalent=1) monkeypatch.setattr(BenchmarkRunner, 'run', execute) + mock_suite(monkeypatch) monkeypatch.setenv('BENCHSET_WARMUP', '0') monkeypatch.setenv('BENCHSET_DURATION', '0') assert cli.main([]) == 0 @@ -339,6 +473,7 @@ def fail(_runner: BenchmarkRunner, name: str, **_options: object) -> BenchmarkRe raise RuntimeError('backend unavailable') monkeypatch.setattr(BenchmarkRunner, 'run', fail) + mock_suite(monkeypatch) assert cli.main(['--bench', benchmark]) == 1 captured = capsys.readouterr() assert not captured.out @@ -437,6 +572,115 @@ def test_render_handles_real_stream_encodings_without_losing_values(encoding: st stream.detach() +@pytest.mark.parametrize('ignored', [False, True]) +def test_intermediate_output_preserves_caller_sigint_policy( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ignored: bool, +) -> None: + """Custom and ignored Ctrl+C during output keep the suite running and restore the caller's policy.""" + mock_suite(monkeypatch) + monkeypatch.setattr(BenchmarkRunner, 'run', lambda _self, name, **_options: + BenchmarkResult(name, status='completed', score=1, pi_equivalent=1)) + calls: List[int] = [] + handler = signal.SIG_IGN if ignored else lambda signum, _frame: calls.append(signum) + previous = signal.signal(signal.SIGINT, handler) + render = cli.render + + def interrupted_render(result: BenchmarkResult, stream: TextIO) -> None: + if result.benchmark == 'single_core': + signal.raise_signal(signal.SIGINT) + render(result, stream) + + monkeypatch.setattr(cli, 'render', interrupted_render) + try: + assert cli.main(['--warmup=0', '--duration=.01']) == 0 + assert calls == ([] if ignored else [signal.SIGINT]) + assert signal.getsignal(signal.SIGINT) is handler + finally: + signal.signal(signal.SIGINT, previous) + captured = capsys.readouterr() + assert 'single_core' in captured.out + assert 'multi_core' in captured.out + assert 'Full Pi equivalent' in captured.out + assert not captured.err + + +@pytest.mark.parametrize('boundary', ['handler', 'render', 'restore']) +def test_repeated_sigint_at_output_handoff_waits_for_native_cleanup(boundary: str) -> None: + """A second Ctrl+C cannot escape cleanup after the output handler takes over. + + Interrupt after installing the caller's handler, while rendering, or before + reinstating the CLI handler, then interrupt join with native cleanup blocked. + An independent worker completion event detects premature return even if an + interrupted Thread.join incorrectly reports the thread as stopped. + """ + program = 'BOUNDARY = ' + repr(boundary) + ''' +import signal +from threading import Event, Thread +from benchset import BenchmarkRunner, cli +from benchset._native import Native + +entered, release, finished = (Event() for _ in range(3)) +runner, workers = BenchmarkRunner(), [] +free, install = Native.free, signal.signal +previous = signal.getsignal(signal.SIGINT) +armed = True +outputting = False +def interrupted_install(signum, handler): + global armed, outputting + if BOUNDARY == 'restore' and outputting and handler is not previous and armed: + armed = False + signal.raise_signal(signal.SIGINT) + old = install(signum, handler) + is_output = handler is previous or getattr(handler, '__name__', '') == 'interrupt_output' + if is_output: outputting = True + if BOUNDARY == 'handler' and is_output and armed: + armed = False + signal.raise_signal(signal.SIGINT) + return old +def blocked_free(self, *args, **kwargs): + free(self, *args, **kwargs) + entered.set() + assert release.wait(5), 'cleanup was never released' +def interrupted_render(*args): + signal.raise_signal(signal.SIGINT) +class Worker(Thread): + armed = True + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + workers.append(self) + def run(self): + try: + super().run() + finally: + finished.set() + def join(self, *args, **kwargs): + if self.armed: + self.armed = False + assert entered.wait(5), 'native cleanup was never reached' + signal.raise_signal(signal.SIGINT) + release.set() + return super().join(*args, **kwargs) +Native.free = blocked_free +signal.signal = interrupted_install +cli.Thread = Worker +cli.BenchmarkRunner = lambda: runner +if BOUNDARY == 'render': cli.render = interrupted_render +try: + code = cli.main(['--warmup=0', '--duration=.1']) + assert finished.is_set(), 'CLI returned before native cleanup' + assert runner._active is None, 'CLI returned with runner ownership still claimed' + assert signal.getsignal(signal.SIGINT) is previous + assert code == 130 +finally: + release.set() + for worker in workers: Thread.join(worker, timeout=5) +''' + result = subprocess.run([sys.executable, '-c', program], capture_output=True, + text=True, check=False, timeout=10) + assert result.returncode == 0, result.stdout + result.stderr + assert not result.stderr + + @pytest.mark.parametrize('stage', ['start', 'join']) def test_sigint_at_worker_lifecycle_boundaries_waits_for_teardown(stage: str) -> None: """Defer Ctrl+C until the CLI's worker has actually stopped, then restore SIGINT. @@ -577,6 +821,7 @@ def execute(_self: BenchmarkRunner, name: str, **_kwargs: object) -> BenchmarkRe return BenchmarkResult(name, status='completed', score=1, pi_equivalent=1) monkeypatch.setattr(BenchmarkRunner, 'run', execute) + mock_suite(monkeypatch) assert cli.main(['--warmup=0', '--duration=.03']) == 0 assert 'multi_core' in output.published @@ -606,6 +851,7 @@ def execute(_self: BenchmarkRunner, benchmark: str, *, return BenchmarkResult(benchmark, status='completed', score=1, pi_equivalent=1) monkeypatch.setattr(BenchmarkRunner, 'run', execute) + mock_suite(monkeypatch) assert cli.main(['--bench', 'single_core']) == 0 progress, result = terminal.getvalue().split('Benchmark', 1) fragments = [part for part in progress.split('\r') if part] @@ -645,6 +891,7 @@ def execute(_self: BenchmarkRunner, benchmark: str, *, return BenchmarkResult(benchmark, status='completed', score=1, pi_equivalent=1) monkeypatch.setattr(BenchmarkRunner, 'run', execute) + mock_suite(monkeypatch) assert cli.main(['--bench=single_core']) == 0 progress = terminal.getvalue().split('Benchmark', 1)[0] frames = [part for part in progress.split('\r') if part] @@ -875,6 +1122,7 @@ def run(_runner: BenchmarkRunner, name: str, **_options: object) -> BenchmarkRes output = io.StringIO() monkeypatch.setattr(BenchmarkRunner, 'run', run) + mock_suite(monkeypatch) monkeypatch.setattr(cli.sys, 'stdout', output) monkeypatch.setattr(cli.sys, 'stderr', None) assert cli.main(['--warmup=0', '--duration=.03']) == expected @@ -951,11 +1199,18 @@ def fileno(self) -> int: def test_real_ctrl_c_cancels_each_phase_and_stops_the_suite( invocation: Tuple[List[str], Dict[str, str]], phase: str, ) -> None: + """Real terminal progress remains cancellable in either active phase. + + Keep the five-second cancellation bound. A timeout requests a fatal Python + stack dump from the disposable child, exposing a blocked I/O or cleanup + thread instead of reporting an opaque process timeout on shared runners. + """ import pty # noqa: PLC0415 import select # noqa: PLC0415 import time # noqa: PLC0415 command, environment = invocation + environment = dict(environment, PYTHONFAULTHANDLER='1') master, slave = pty.openpty() process = subprocess.Popen([*command, '--warmup', '20' if phase == 'warmup' else '0', '--duration', '20'], env=environment, stdout=slave, @@ -970,7 +1225,12 @@ def test_real_ctrl_c_cancels_each_phase_and_stops_the_suite( observed += os.read(master, 4096) assert phase.encode() in observed process.send_signal(signal.SIGINT) - _output, errors = process.communicate(timeout=5) + try: + _output, errors = process.communicate(timeout=5) + except subprocess.TimeoutExpired: + process.send_signal(signal.SIGABRT) + _output, diagnostic = process.communicate(timeout=5) + pytest.fail('CLI did not finish cancellation within five seconds:\n' + diagnostic) assert process.returncode == 130 assert 'cancelled' in errors assert 'Traceback' not in errors diff --git a/tests/test_cli_active_signal_policy.py b/tests/test_cli_active_signal_policy.py new file mode 100644 index 0000000..14253c9 --- /dev/null +++ b/tests/test_cli_active_signal_policy.py @@ -0,0 +1,140 @@ +"""Restore the CLI entry signal policy after changes during native work.""" + +import signal +import sys +from threading import Event, Thread +from types import FrameType +from typing import Any, Callable, List, Optional, TextIO, Tuple, Union + +import pytest + +from benchset import BenchmarkResult, BenchmarkRunner, Progress, cli +from tests.helpers import assert_runner_reusable + +_Handler = Union[int, Callable[[int, Optional[FrameType]], None], None] +_POLICIES = ('default', 'custom', 'ignored') + + +def _handler(policy: str, custom: Callable[[int, Optional[FrameType]], None]) -> _Handler: + return signal.default_int_handler if policy == 'default' else signal.SIG_IGN if policy == 'ignored' else custom + + +@pytest.mark.skipif(sys.platform == 'win32', reason='SIGUSR1 coordinates caller policy changes on the main thread') +@pytest.mark.parametrize(('benchmark', 'changed_child'), [ + ('single_core', 'single_core'), ('all', 'single_core'), ('all', 'multi_core'), +]) +@pytest.mark.parametrize('initial', _POLICIES) +@pytest.mark.parametrize('selected', _POLICIES) +@pytest.mark.parametrize('caller_error', [False, True]) +def test_cli_restores_its_entry_policy_after_a_caller_changes_sigint_during_native_work( # noqa: PLR0913, PLR0915 — One owner spans policy changes, cleanup and reuse. + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], + benchmark: str, changed_child: str, initial: str, selected: str, caller_error: bool, +) -> None: + """Native-work changes do not become output-phase caller policy selections. + + A real progress callback requests SIGUSR1 and waits until its main-thread + handler selects another SIGINT policy. Deliver it after the wait condition + has released its lock, so CPython wait internals cannot replace the caller + exception while reacquiring that lock. No SIGINT or output callback changes + that policy. HEAD restored its entry handler before rendering each result; + the shared-suite cleanup must retain that scope while preserving the + separate, approved policy changes made during intermediate output. A caller + error after the selection also exercises cleanup while an error unwinds. + """ + runner = BenchmarkRunner() + changed, requested = Event(), Event() + sent = False + signals: List[str] = [] + rendering: List[Tuple[str, _Handler]] = [] + workers: List[Thread] = [] + original = RuntimeError('caller control signal failed') + + def entry_custom(_signum: int, _frame: Optional[FrameType]) -> None: + signals.append('entry') + + def selected_custom(_signum: int, _frame: Optional[FrameType]) -> None: + signals.append('selected') + + entry_handler = _handler(initial, entry_custom) + selected_handler = _handler(selected, selected_custom) + + def select_policy(_signum: int, _frame: Optional[FrameType]) -> None: + signal.signal(signal.SIGINT, selected_handler) + changed.set() + if caller_error: + raise original + + class Observer(cli._Progress): + def __call__(self, value: Progress) -> None: + nonlocal sent + if value.benchmark == changed_child and not sent: + sent = True + requested.set() + assert changed.wait(5), 'main thread did not select the caller policy' + super().__call__(value) + + class MainWait(Event): + def wait(self, timeout: Optional[float] = None) -> bool: + value = super().wait(timeout) + if requested.is_set(): + requested.clear() + signal.raise_signal(signal.SIGUSR1) + return value + + class Worker(Thread): + def __init__(self, *arguments: Any, **options: Any) -> None: + super().__init__(*arguments, **options) + workers.append(self) + + render = cli.render + + def observe_render(result: BenchmarkResult, stream: TextIO) -> None: + rendering.append((result.benchmark, signal.getsignal(signal.SIGINT))) + render(result, stream) + + monkeypatch.setattr(cli, 'BenchmarkRunner', lambda: runner) + monkeypatch.setattr(cli, '_Progress', Observer) + monkeypatch.setattr(cli, 'Thread', Worker) + monkeypatch.setattr(cli, 'Event', MainWait) + monkeypatch.setattr(cli, 'render', observe_render) + previous = signal.signal(signal.SIGINT, entry_handler) + previous_usr = signal.signal(signal.SIGUSR1, select_policy) + try: + arguments = ['--bench=' + benchmark, '--warmup=.01', '--duration=.1'] + if caller_error: + with pytest.raises(RuntimeError, match='caller control signal failed') as caught: + cli.main(arguments) + assert caught.value is original + else: + assert cli.main(arguments) == 0 + output, errors = capsys.readouterr() + assert not errors + assert changed.is_set() + assert sent + assert not signals + assert signal.getsignal(signal.SIGINT) is entry_handler + expected = ['single_core', 'multi_core'] if benchmark == 'all' else ['single_core'] + if caller_error: + expected = ['single_core'] if changed_child == 'multi_core' else [] + assert [name for name, _handler in rendering] == expected + for index, (_name, actual) in enumerate(rendering): + if initial == 'default' and benchmark == 'all' and index == 0: + # The temporary default-output guard is an approved CLI policy. + assert actual is signal.default_int_handler or getattr(actual, '__name__', '') == 'interrupt_output' + else: + assert actual is entry_handler + assert output.count('Status : completed') == len(rendering) + if not caller_error: + assert_runner_reusable(runner) + finally: + # Old CLI could return a non-SIGINT error while its worker was finishing. + # This policy test joins it explicitly rather than pinning that old bug. + runner.cancel() + for worker in workers: + if worker.ident is not None: + worker.join(timeout=5) + assert not worker.is_alive() + signal.signal(signal.SIGUSR1, previous_usr) + signal.signal(signal.SIGINT, previous) + if caller_error: + assert_runner_reusable(runner) diff --git a/tests/test_cli_cleanup_signals.py b/tests/test_cli_cleanup_signals.py new file mode 100644 index 0000000..cfca523 --- /dev/null +++ b/tests/test_cli_cleanup_signals.py @@ -0,0 +1,328 @@ +"""Keep native ownership safe through caller signal errors and nested output calls.""" + +import io +import json +import os +import signal +import subprocess +import sys +from pathlib import Path +from threading import enumerate as threads +from typing import Any, List + +import pytest + +from benchset import BenchmarkRunner, cli, run, run_all +from benchset._native import Native +from tests.helpers import assert_runner_reusable + +_ROOT = Path(__file__).resolve().parents[1] +_CLEANUP = r''' +import builtins +import gc +import hashlib +import io +import json +from pathlib import Path +import signal +import sys +import weakref +from threading import Event, Thread +from benchset import BenchmarkRunner, cli +from benchset._native import Native + +primary_type, secondary_type, inherited = sys.argv[1:4] +normal = len(sys.argv) > 4 and sys.argv[4] == 'normal' +immediate = len(sys.argv) > 4 and sys.argv[4] == 'immediate' +normal_joined = False +class WeakError(RuntimeError): + pass + +primary = (WeakError if primary_type == 'WeakError' else getattr(builtins, primary_type))('primary caller signal') +if primary_type == 'WeakError': + gc.disable() +secondary = getattr(builtins, secondary_type)('secondary caller signal') +entered, release, finished = (Event() for _ in range(3)) +runner, workers, acknowledgements, live = BenchmarkRunner(), [], [], set() +install, create, free = signal.signal, Native.create, Native.free +outputting = tearing_down = False +remaining = 3 +calls = 0 + +def custom(_signum, _frame): + global calls + calls += 1 + # Exercise both immediate replacement and a repeated first handler error. + raise primary if calls == 1 or (calls == 2 and not immediate and not normal) else secondary + +previous = install(signal.SIGINT, custom) + +def intercepted(signum, handler): + global remaining + if (normal_joined if normal else outputting) and getattr(handler, '__name__', '') == 'interrupt' and remaining: + remaining -= 1 + signal.raise_signal(signal.SIGINT) + return install(signum, handler) + +def allocate(native, *arguments): + handle = create(native, *arguments) + live.add((id(native), handle, arguments)) + return handle + +def blocked_free(native, handle, *arguments): + free(native, handle, *arguments) + live.remove((id(native), handle, arguments)) + if outputting and not normal: + entered.set() + assert release.wait(5), 'native cleanup was never released' + +def render(*_arguments): + global outputting + outputting = True + if not normal: + signal.raise_signal(signal.SIGINT) + +class Worker(Thread): + armed = True + + def __init__(self, *arguments, **options): + super().__init__(*arguments, **options) + workers.append(self) + + def run(self): + try: + super().run() + finally: + finished.set() + + def join(self, *arguments, **options): + global normal_joined + if self.armed and outputting and not normal: + self.armed = False + assert entered.wait(5), 'native cleanup was never entered' + signal.raise_signal(signal.SIGINT) + release.set() + value = super().join(*arguments, **options) + if normal: + install(signal.SIGINT, custom) + normal_joined = True + return value + +class Acknowledgement(Event): + def wait(self, timeout=None): + return True if tearing_down else super().wait(timeout) + +def acknowledgement(): + value = Acknowledgement() + acknowledgements.append(value) + return value + +Native.create, Native.free = allocate, blocked_free +signal.signal = intercepted +cli.Thread, cli.ResultEvent = Worker, acknowledgement +cli.BenchmarkRunner, cli.render = lambda: runner, render +stdout, stderr = sys.stdout, sys.stderr +sys.stdout, sys.stderr = io.StringIO(), io.StringIO() +result = {} + +def invoke(): + try: + result['code'] = cli.main(['--warmup=0', '--duration=.05']) + except BaseException as error: + result['exception'] = type(error).__name__ + result['original'] = error is primary + +try: + if inherited == 'yes': + try: + raise LookupError('unrelated caller context') + except LookupError: + invoke() + else: + invoke() + result.update(finished=finished.is_set(), active=bool(runner._active), + native=runner._native is not None, handle=runner._handle, + live=len(live), restored=signal.getsignal(signal.SIGINT) is custom, + workers=[worker.is_alive() for worker in workers], + cli_path=str(Path(cli.__file__).resolve()), + cli_sha256=hashlib.sha256(Path(cli.__file__).read_bytes()).hexdigest()) +finally: + # Failed implementations must also leave this subprocess able to exit. + # The assertions below use witnesses captured before this fallback cleanup. + tearing_down = True + signal.signal = install + install(signal.SIGINT, previous) + release.set() + runner.cancel() + for value in acknowledgements: + value.set() + for worker in workers: + Thread.join(worker, timeout=5) + sys.stdout, sys.stderr = stdout, stderr +if primary_type == 'WeakError': + reference = weakref.ref(primary) + primary = secondary = None + result['error_collected'] = reference() is None + gc.enable() +print(json.dumps(result)) +''' + + +@pytest.mark.parametrize('primary', ['RuntimeError', 'ValueError', 'KeyboardInterrupt', 'SystemExit', 'GeneratorExit']) +@pytest.mark.parametrize('secondary', ['RuntimeError', 'KeyboardInterrupt', 'SystemExit']) +@pytest.mark.parametrize('inherited', [False, True]) +@pytest.mark.parametrize('immediate', [False, True]) +def test_cli_cleanup_keeps_the_first_custom_signal_error_and_joins_native_workers( + primary: str, secondary: str, inherited: bool, immediate: bool, +) -> None: + """Repeated caller signal errors cannot escape a live native cleanup boundary. + + An event blocks real native release, and another signal interrupts handoff + and join, including a distinct error on the first restoration attempt. + Secondary errors must preserve the first escaping error; + KeyboardInterrupt retains CLI exit 130. A separate completion witness catches + premature return even if interrupted Thread.join corrupts its own bookkeeping. + Calling from an unrelated except block prevents caller context from being + mistaken for this CLI's pending exception. Fallback teardown cannot satisfy + the witnesses, because they are copied before fallback begins. + """ + environment = os.environ.copy() + environment.pop('PYTHONPATH', None) + process = subprocess.run( + [sys.executable, '-c', _CLEANUP, primary, secondary, 'yes' if inherited else 'no', + 'immediate' if immediate else 'repeated'], + cwd=_ROOT, env=environment, capture_output=True, text=True, check=False, timeout=15, + ) + assert process.returncode == 0, process.stdout + process.stderr + assert not process.stderr + result = json.loads(process.stdout) + assert result['cli_path'] == str((_ROOT / 'benchset/cli.py').resolve()) + assert result['finished'], 'CLI returned before its worker finished' + assert not result['active'] + assert not result['native'] + assert result['handle'] is None + assert result['live'] == 0 + assert not any(result['workers']) + assert result['restored'] + if primary == 'KeyboardInterrupt': + assert result['code'] == 130 + else: + assert result['exception'] == primary + assert result['original'], 'a later signal replaced the first caller error' + + +@pytest.mark.parametrize('primary', ['RuntimeError', 'KeyboardInterrupt', 'SystemExit', 'WeakError']) +@pytest.mark.parametrize('inherited', [False, True]) +def test_cli_keeps_a_new_cleanup_guard_error_after_a_successful_join( + primary: str, inherited: bool, +) -> None: + """A fresh signal error after successful join must escape rather than disappear. + + The worker installs a caller handler immediately after its normal join; + real signals then interrupt cleanup-guard installation. Preserve the first + new error after teardown even when an unrelated caller exception is active. + This distinguishes a body error being unwound from inherited except context. + Weak errors must disappear with cyclic GC disabled once the caller drops them, + protecting the temporary error holder from retaining its own traceback. + """ + environment = os.environ.copy() + environment.pop('PYTHONPATH', None) + process = subprocess.run( + [sys.executable, '-c', _CLEANUP, primary, 'GeneratorExit', 'yes' if inherited else 'no', 'normal'], + cwd=_ROOT, env=environment, capture_output=True, text=True, check=False, timeout=15, + ) + assert process.returncode == 0, process.stdout + process.stderr + assert not process.stderr + result = json.loads(process.stdout) + assert result['finished'] + assert not result['active'] + assert not result['native'] + assert result['handle'] is None + assert result['live'] == 0 + assert not any(result['workers']) + assert result['restored'] + if primary == 'KeyboardInterrupt': + assert result['code'] == 130 + else: + assert result['exception'] == primary + assert result['original'] + if primary == 'WeakError': + assert result['error_collected'], 'the guard retained its own exception traceback' + + +@pytest.mark.parametrize('nested', ['run', 'run_all']) +@pytest.mark.parametrize('repeated', [False, True]) +def test_cli_nested_output_sigint_cancels_the_outer_cli_after_complete_native_cleanup( + monkeypatch: pytest.MonkeyPatch, nested: str, repeated: bool, +) -> None: + """Preserve the approved whole-CLI cancellation policy for nested output work. + + The caller's output sink handles any nested KeyboardInterrupt itself. SIGINT + still cancels the enclosing CLI with 130; making the output guard look like + the ordinary default handler must not silently isolate the nested call. + Once cancelled, another signal must let the output sink finish normally. + Audit all real native handles and worker completion before caller reuse. + """ + live = set() + allocated: List[Any] = [] + released: List[Any] = [] + create, free = Native.create, Native.free + entered = False + nested_finished = False + + def allocate(native: Native, *arguments: Any) -> int: + handle = create(native, *arguments) + identity = id(native), handle, arguments + live.add(identity) + allocated.append(identity) + return handle + + def release(native: Native, handle: int, *arguments: Any) -> None: + free(native, handle, *arguments) + identity = id(native), handle, arguments + live.remove(identity) + released.append(identity) + + class Output(io.StringIO): + def write(self, text: str) -> int: + nonlocal entered, nested_finished + if not entered and text.startswith('Benchmark'): + entered = True + emitted = False + + def progress(_value: Any) -> None: + nonlocal emitted + if not emitted: + emitted = True + signal.raise_signal(signal.SIGINT) + + try: + if nested == 'run': + run('single_core', warmup_seconds=0, measurement_seconds=.01, on_progress=progress) + else: + run_all(warmup_seconds=0, measurement_seconds=.01, on_progress=progress) + except KeyboardInterrupt: + pass + if repeated: + signal.raise_signal(signal.SIGINT) + nested_finished = True + return super().write(text) + + output, errors = Output(), io.StringIO() + monkeypatch.setattr(Native, 'create', allocate) + monkeypatch.setattr(Native, 'free', release) + monkeypatch.setattr(cli.sys, 'stdout', output) + monkeypatch.setattr(cli.sys, 'stderr', errors) + previous = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + assert cli.main(['--warmup=0', '--duration=.05']) == 130 + assert entered + assert nested_finished + assert not live + assert len(allocated) == len(released) == 2 + assert not any(worker.name == 'benchset-cli' for worker in threads()) + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + assert_runner_reusable(BenchmarkRunner()) + assert not live + finally: + signal.signal(signal.SIGINT, previous) diff --git a/tests/test_cli_embedded_signals.py b/tests/test_cli_embedded_signals.py new file mode 100644 index 0000000..45e850d --- /dev/null +++ b/tests/test_cli_embedded_signals.py @@ -0,0 +1,269 @@ +"""Preserve bounded CLI failure for signal handlers owned by a native host.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +_ROOT = Path(__file__).resolve().parents[1] +_PROGRAM = r''' +import hashlib +import io +import json +import os +from pathlib import Path +import signal +import sys +from threading import Event, Thread +sys.path.insert(0, str(Path.cwd())) +from benchset import BenchmarkRunner, cli +from benchset._native import Native + +benchmark = sys.argv[1] +window = next((value for value in sys.argv[2:] if value in ('warmup', 'duration', 'syntax')), 'valid') +arguments = ['--bench=' + benchmark, '--warmup=0', '--duration=.01'] +if window == 'warmup': + arguments[1] = '--warmup=-1' +elif window == 'duration': + arguments[2] = '--duration=0' +elif window == 'syntax': + arguments[2] = '--duration=not-a-number' +runner = BenchmarkRunner() +workers, live, allocated, released, invalid_restores = [], [], [], [], [] +getsignal, install = signal.getsignal, signal.signal +initialize, create, free = Native.__init__, Native.create, Native.free +native_initializations = 0 +initial = getsignal(signal.SIGINT) +captured = False +installed_guard = None + +# A native embedding host with install_signal_handlers=0 can legitimately +# produce this state. Subsequent reads observe the actual Python CLI guard. +def embedded_handler(signum): + global captured + if signum == signal.SIGINT and not captured: + captured = True + return None + return getsignal(signum) + +expected = None +if window != 'syntax': + try: + install(signal.SIGINT, None) + except TypeError as error: + expected = str(error) + +def restore(signum, handler): + global installed_guard + try: + value = install(signum, handler) + if signum == signal.SIGINT and installed_guard is None: + installed_guard = handler + return value + except TypeError as error: + if signum == signal.SIGINT and handler is None: + invalid_restores.append(error) + if len(invalid_restores) > 3: + # Make an unbounded retry an immediate ordinary test failure, + # rather than depending on a subprocess timeout or Ctrl+C. + print(json.dumps({'unbounded_retry': True}), file=stdout, flush=True) + os._exit(90) + raise + +def construct(native): + global native_initializations + native_initializations += 1 + initialize(native) + +def allocate(native, *arguments): + handle = create(native, *arguments) + identity = id(native), handle, arguments + live.append(identity) + allocated.append(identity) + return handle + +def release(native, handle, *arguments): + free(native, handle, *arguments) + identity = id(native), handle, arguments + live.remove(identity) + released.append(identity) + +class Worker(Thread): + def __init__(self, *arguments, **options): + super().__init__(*arguments, **options) + self.finished = Event() + self.allow_finish = Event() + workers.append(self) + + def run(self): + try: + super().run() + finally: + assert self.allow_finish.wait(5), 'CLI did not join its worker' + self.finished.set() + + def join(self, *arguments, **options): + self.allow_finish.set() + return super().join(*arguments, **options) + +Native.__init__, Native.create, Native.free = construct, allocate, release +cli.BenchmarkRunner = lambda: runner +cli.Thread = Worker +signal.getsignal, signal.signal = embedded_handler, restore +output, errors = io.StringIO(), io.StringIO() +stdout, stderr = sys.stdout, sys.stderr +try: + sys.stdout, sys.stderr = output, errors + observed = {} + try: + observed['code'] = cli.main(arguments) + except TypeError as error: + observed.update(error=str(error), original=bool(invalid_restores) and error is invalid_restores[0]) + observed.update( + captured=captured, allocated=len(allocated), released=allocated == released, + native_initializations=native_initializations, live=len(live), + active=runner._active is not None, native=runner._native is not None, handle=runner._handle, + workers_finished=bool(workers) and all(worker.finished.is_set() for worker in workers), + workers_alive=any(worker.is_alive() for worker in workers), workers_count=len(workers), + output=output.getvalue(), errors=errors.getvalue(), expected=expected, + guard_installed=installed_guard is not None, + guard_retained=getsignal(signal.SIGINT) is installed_guard, + ) +finally: + sys.stdout, sys.stderr = stdout, stderr + signal.getsignal, signal.signal = getsignal, install + install(signal.SIGINT, initial) + for worker in workers: + worker.join(timeout=5) + +result = runner.run('single_core', warmup_seconds=10, measurement_seconds=10, + on_progress=lambda _progress: runner.cancel()) +observed['reuse_status'] = result.status +observed['reuse_clean'] = not live and runner._active is None and runner._handle is None +observed['cli_path'] = str(Path(cli.__file__).resolve()) +observed['cli_sha256'] = hashlib.sha256(Path(cli.__file__).read_bytes()).hexdigest() +print(json.dumps(observed)) +''' + + +@pytest.mark.parametrize('benchmark', ['single_core', 'all']) +@pytest.mark.parametrize('window', ['valid', 'warmup', 'duration']) +def test_cli_native_owned_sigint_state_raises_original_type_error_after_cleanup(benchmark: str, window: str) -> None: + """A native-owned SIGINT handler must not make final restoration retry forever. + + Real C embedding with PyConfig.install_signal_handlers=0 established that + getsignal(SIGINT) may return None. Python cannot reinstall that opaque C + handler: preserve the original one-shot TypeError instead of substituting + SIG_DFL. Model only that initial backend state, run real native benchmarks, + and audit resources before reusing the runner. A completion event requires + an actual worker join. A bounded invalid-call witness makes the previous + infinite retry fail promptly without timing assumptions. + + The user explicitly approved this TypeError preceding numeric range errors + after successful syntax parsing. Invalid windows therefore allocate no native + handle, while the parsing-only syntax failure is covered separately below. + """ + environment = os.environ.copy() + environment.pop('PYTHONPATH', None) + result = subprocess.run([sys.executable, '-c', _PROGRAM, benchmark, window], cwd=_ROOT, + env=environment, capture_output=True, text=True, check=False, timeout=15) + assert result.returncode == 0, result.stdout + result.stderr + assert not result.stderr + observed = json.loads(result.stdout) + assert observed['cli_path'] == str((_ROOT / 'benchset/cli.py').resolve()) + assert observed['captured'] + assert 'error' in observed, observed + assert observed['error'] == observed['expected'] + assert observed['original'] + assert observed['guard_retained'], 'an opaque native handler must not be silently replaced with SIG_DFL' + assert observed['native_initializations'] == 1 + assert observed['allocated'] == (1 if window == 'valid' else 0) + assert observed['released'] + assert observed['live'] == 0 + assert not observed['active'] + assert not observed['native'] + assert observed['handle'] is None + assert observed['workers_finished'] + assert not observed['workers_alive'] + assert not observed['output'] + assert not observed['errors'] + assert observed['reuse_status'] == 'cancelled' + assert observed['reuse_clean'] + + +def test_cli_syntax_errors_precede_native_signal_ownership_without_loading_rust() -> None: + """Malformed argument syntax remains argparse exit 2 in an opaque native host. + + Unlike the explicitly approved numeric-range priority, parsing must finish + before signal ownership, Rust construction or worker startup. Snapshot all + three before the shared fixture restores its environment and reuses the + runner, so its later successful work cannot mask eager initialization. + """ + environment = os.environ.copy() + environment.pop('PYTHONPATH', None) + result = subprocess.run([sys.executable, '-c', _PROGRAM, 'single_core', 'syntax'], cwd=_ROOT, + env=environment, capture_output=True, text=True, check=False, timeout=15) + assert result.returncode == 0, result.stdout + result.stderr + assert not result.stderr + observed = json.loads(result.stdout) + assert observed['cli_path'] == str((_ROOT / 'benchset/cli.py').resolve()) + assert observed['code'] == 2 + assert not observed['guard_installed'] + assert observed['native_initializations'] == 0 + assert observed['allocated'] == observed['live'] == observed['workers_count'] == 0 + assert not observed['active'] + assert not observed['native'] + assert observed['handle'] is None + assert not observed['output'] + assert 'usage: benchset' in observed['errors'] + assert 'must be a number of seconds' in observed['errors'] + assert 'Traceback' not in observed['errors'] + assert observed['reuse_status'] == 'cancelled' + assert observed['reuse_clean'] + + +@pytest.mark.parametrize('caller', ['callable', 'default']) +def test_valid_final_sigint_policy_still_retries_a_caller_type_error(caller: str) -> None: + """Only an opaque None policy makes TypeError a permanent input failure. + + A caller's SIGTERM handler can instead raise TypeError just before a valid + SIGINT policy is reinstalled. Retry that interrupted operation and preserve + its first error after complete cleanup. SIG_DFL is valid despite being zero, + so a truthiness check must not confuse it with the unsupported None state. + """ + from tests.test_cli_final_signal_restore import _program # noqa: PLC0415 + + # Python 3.8's metadata finder does not treat an empty sys.path entry as + # the current directory; keep this checkout ahead of any older install. + program = 'import sys\nfrom pathlib import Path\nsys.path.insert(0, str(Path.cwd()))\n' + _program('normal', 'before', False) + if caller == 'default': + replacements = { + 'previous = install(signal.SIGINT, custom)': + 'previous = install(signal.SIGINT, signal.SIG_DFL)', + 'and handler is custom and not final_restore_fired)': + 'and handler is signal.SIG_DFL and not final_restore_fired)', + '(replacement if latest and final_restore_fired else custom)': + '(replacement if latest and final_restore_fired else signal.SIG_DFL)', + } + for old, new in replacements.items(): + assert program.count(old) == 1 + program = program.replace(old, new) + environment = os.environ.copy() + environment.pop('PYTHONPATH', None) + result = subprocess.run([sys.executable, '-c', program, 'TypeError', 'GeneratorExit', 'no', 'normal'], + cwd=_ROOT, env=environment, capture_output=True, text=True, check=False, timeout=15) + assert result.returncode == 0, result.stdout + result.stderr + assert not result.stderr + observed = json.loads(result.stdout) + assert observed['exception'] == 'TypeError' + assert observed['original'] + assert observed['finished'] + assert not observed['active'] + assert not observed['native'] + assert observed['handle'] is None + assert observed['live'] == 0 + assert not any(observed['workers']) + assert observed['restored'] diff --git a/tests/test_cli_final_signal_restore.py b/tests/test_cli_final_signal_restore.py new file mode 100644 index 0000000..fd322ad --- /dev/null +++ b/tests/test_cli_final_signal_restore.py @@ -0,0 +1,191 @@ +"""Final caller-handler restoration must not replace or retain an earlier error.""" + +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Dict, List + +import pytest + +from tests.test_cli_cleanup_signals import _CLEANUP + +_ROOT = Path(__file__).resolve().parents[1] + + +def _program(stage: str, boundary: str, latest: bool, owned: bool = False) -> str: + """Extend the real native cleanup fixture with a final restoration signal. + + SIGTERM models a caller exception before installing SIGINT's old policy; + SIGINT exercises the boundary just after that policy has been installed. + The fixture records native ownership before its independent fallback teardown. + """ + program = _CLEANUP + replacements = { + 'remaining = 3\ncalls = 0': + 'remaining = ' + ('0' if stage == 'normal' else '3') + '\ncalls = 0\n' + 'final_restore_fired = False\n' + 'latest = ' + repr(latest) + '\n' + 'def replacement(_signum, _frame):\n pass', + ' calls += 1\n': + ' calls += 1\n' + ' if latest and final_restore_fired:\n' + ' install(signal.SIGINT, replacement)\n', + 'previous = install(signal.SIGINT, custom)\n': + 'previous = install(signal.SIGINT, custom)\n' + 'previous_term = install(signal.SIGTERM, custom)\n', + ' global remaining\n': ' global remaining, final_restore_fired\n', + ' return install(signum, handler)\n\ndef allocate': + ' final = ((normal_joined if normal else outputting)\n' + ' and handler is custom and not final_restore_fired)\n' + ' if final:\n' + ' final_restore_fired = True\n' + + (' signal.raise_signal(signal.SIGTERM)\n' if boundary == 'before' else '') + + ' value = install(signum, handler)\n' + + (' if final:\n signal.raise_signal(signal.SIGINT)\n' if boundary == 'after' else '') + + ' return value\n\ndef allocate', + 'restored=signal.getsignal(signal.SIGINT) is custom,': + 'restored=signal.getsignal(signal.SIGINT) is ' + '(replacement if latest and final_restore_fired else custom),', + ' install(signal.SIGINT, previous)\n': + ' install(signal.SIGINT, previous)\n' + ' install(signal.SIGTERM, previous_term)\n', + ' result.update(finished=finished.is_set(),': + ' result.update(diagnostics=sys.stderr.getvalue(), finished=finished.is_set(),', + } + for old, new in replacements.items(): + assert program.count(old) == 1 + program = program.replace(old, new) + if owned: + replacements = { + 'previous_term = install(signal.SIGTERM, custom)': + 'policy_selected = Event()\n' + 'def select_policy(_signum, _frame):\n' + ' install(signal.SIGINT, replacement)\n' + ' policy_selected.set()\n' + 'previous_term = install(signal.SIGTERM, select_policy)', + ' entered.set()\n': + ' signal.raise_signal(signal.SIGTERM)\n' + ' entered.set()\n', + " assert entered.wait(5), 'native cleanup was never entered'\n": + " assert entered.wait(5), 'native cleanup was never entered'\n" + " assert policy_selected.wait(5), 'caller policy was never selected'\n", + } + for old, new in replacements.items(): + assert program.count(old) == 1 + program = program.replace(old, new) + return program + + +def _invoke( # noqa: PLR0913 — Independent caller-policy and exception-state controls. + stage: str, boundary: str, primary: str, *, latest: bool = False, owned: bool = False, inherited: bool = False, +) -> Dict[str, object]: + environment = os.environ.copy() + environment.pop('PYTHONPATH', None) + arguments: List[str] = [primary, 'GeneratorExit', 'yes' if inherited else 'no', + 'immediate' if stage == 'body' else 'normal'] + process = subprocess.run( + [sys.executable, '-c', _program(stage, boundary, latest, owned), *arguments], + cwd=_ROOT, env=environment, capture_output=True, text=True, check=False, timeout=15, + ) + assert process.returncode == 0, process.stdout + process.stderr + assert not process.stderr + result = json.loads(process.stdout) + assert result['cli_path'] == str((_ROOT / 'benchset/cli.py').resolve()) + assert result['finished'] + assert not result['active'] + assert not result['native'] + assert result['handle'] is None + assert result['live'] == 0 + assert not any(result['workers']) + assert result['restored'] + return result + + +@pytest.mark.parametrize('stage', ['body', 'guard', 'normal']) +@pytest.mark.parametrize('boundary', ['before', 'after']) +@pytest.mark.parametrize('primary', ['RuntimeError', 'KeyboardInterrupt', 'OSError']) +def test_final_caller_handler_restoration_preserves_the_first_error( + stage: str, boundary: str, primary: str, +) -> None: + """A later restoration signal cannot replace a body or saved guard error. + + Also cover the first error arising only at normal final restoration: it + must propagate after restoring caller policy, rather than being swallowed. + No native handle, active runner or CLI worker may survive any exit. + """ + result = _invoke(stage, boundary, primary) + if primary == 'KeyboardInterrupt': + assert result['code'] == 130 + elif primary == 'OSError': + assert result['code'] == 1 + assert 'primary caller signal' in str(result['diagnostics']) + assert 'secondary caller signal' not in str(result['diagnostics']) + else: + assert result['exception'] == primary + assert result['original'] + + +@pytest.mark.parametrize('stage', ['body', 'guard', 'normal']) +@pytest.mark.parametrize('boundary', ['before', 'after']) +def test_final_caller_handler_restoration_does_not_retain_its_saved_error( + stage: str, boundary: str, +) -> None: + """Clear saved errors on every restoration exit with cyclic GC disabled. + + A saved error's traceback contains the CLI frame and its error list. Leaving + that error in the list after a secondary exception creates a self-retaining + cycle, even though all native handles were already freed. + """ + result = _invoke(stage, boundary, 'WeakError') + assert result['exception'] == 'WeakError' + assert result['original'] + assert result['error_collected'] + + +@pytest.mark.parametrize('stage', ['body', 'guard', 'normal']) +@pytest.mark.parametrize('boundary', ['before', 'after']) +def test_final_caller_handler_restoration_preserves_a_policy_selected_by_the_caller( + stage: str, boundary: str, +) -> None: + """A caller handler can select a new SIGINT policy before raising its error. + + Retrying an interrupted final installation must retain that latest caller + choice instead of overwriting it with the policy saved on CLI entry. + """ + result = _invoke(stage, boundary, 'RuntimeError', latest=True) + assert result['exception'] == 'RuntimeError' + assert result['original'] + + +def test_final_restoration_does_not_adopt_a_policy_changed_during_owned_native_cleanup() -> None: + """Only a released caller policy can replace the saved CLI entry policy. + + SIGTERM selects another SIGINT handler while real native release is blocked + and the CLI still owns cleanup. The first final installation must restore + the saved caller handler; only retries of an interrupted final installation + may retain a subsequent caller choice. + """ + result = _invoke('body', 'owned', 'RuntimeError', owned=True) + assert result['exception'] == 'RuntimeError' + assert result['original'] + + +@pytest.mark.parametrize('boundary', ['before', 'after']) +@pytest.mark.parametrize('primary', ['RuntimeError', 'KeyboardInterrupt']) +def test_final_restoration_keeps_a_new_error_when_the_caller_is_in_an_unrelated_except_block( + boundary: str, primary: str, +) -> None: + """Inherited except context is not an error currently unwinding this CLI. + + The first error occurs only during final restoration after a normal join. + Looking at sys.exc_info() instead of explicit completion state would confuse + the caller's handled LookupError with a pending CLI failure and hide it. + """ + result = _invoke('normal', boundary, primary, inherited=True) + if primary == 'KeyboardInterrupt': + assert result['code'] == 130 + else: + assert result['exception'] == 'RuntimeError' + assert result['original'] diff --git a/tests/test_cli_loader_validation.py b/tests/test_cli_loader_validation.py new file mode 100644 index 0000000..ab1cb9e --- /dev/null +++ b/tests/test_cli_loader_validation.py @@ -0,0 +1,88 @@ +"""Keep native inspection failures inside the CLI's ordinary failed-result path.""" + +import ctypes +from threading import current_thread +from typing import Any, Callable, List + +import pytest + +from benchset import __version__, cli +from benchset._native import ABI_VERSION + + +class _BrokenLibrary: + def __init__(self, failure: str) -> None: + self.failure = failure + + def __getattr__(self, name: str) -> Callable[[], Any]: + if name == 'benchset_abi_version': + if self.failure == 'missing-abi': + raise AttributeError('missing initial symbol: benchset_abi_version') + + def abi() -> int: + if self.failure == 'abi-error': + raise ValueError('ABI getter exploded') + return ABI_VERSION + + return abi + if name == 'benchset_version': + if self.failure == 'missing-version': + raise AttributeError('missing initial symbol: benchset_version') + + def version() -> Any: + if self.failure == 'invalid-utf8': + return b'\xff' + if self.failure == 'none-version': + return None + if self.failure == 'text-version': + return __version__ + if self.failure == 'version-error': + raise ValueError('version getter exploded') + return __version__.encode('utf-8') + + return version + raise AssertionError('unexpected native lookup after failed initial validation: ' + name) + + +@pytest.mark.parametrize(('failure', 'diagnostic'), [ + ('missing-abi', 'benchset_abi_version'), + ('missing-version', 'benchset_version'), + ('invalid-utf8', 'utf-8'), + ('none-version', 'NoneType'), + ('text-version', "'str' object"), + ('version-error', 'version getter exploded'), + ('abi-error', 'ABI getter exploded'), +]) +@pytest.mark.parametrize('benchmark', ['single_core', 'all']) +@pytest.mark.parametrize('duration', ['.01', '-1']) +def test_cli_defers_initial_native_validation_errors_to_failed_results( # noqa: PLR0913 — Independent loader/CLI input axes. + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], + failure: str, diagnostic: str, benchmark: str, duration: str, +) -> None: + """Run real settings and worker ownership around a successfully opened bad image. + + The pre-migration CLI inspected native symbols only in its worker: missing + ABI/version symbols, invalid version bytes and getter exceptions became a + failed result with exit 1. Eager Rust duration validation now reaches those + same exceptions in settings before the worker starts. Preserve the old CLI + result boundary without changing Native/run's existing raw exception API. + An invalid duration must keep the accepted native-load-error precedence. + """ + attempts: List[str] = [] + + def load(_path: str) -> _BrokenLibrary: + attempts.append(current_thread().name) + return _BrokenLibrary(failure) + + monkeypatch.setattr(ctypes, 'CDLL', load) + assert cli.main(['--bench=' + benchmark, '--warmup=0', '--duration=' + duration]) == 1 + output, errors = capsys.readouterr() + assert not output + assert attempts[-1] == 'benchset-cli', 'settings must defer the failure to worker reporting' + assert 'Benchmark : single_core' in errors + assert 'Status : failed' in errors + assert diagnostic in errors + assert 'Traceback' not in errors + assert 'usage:' not in errors + assert '--duration must be finite and positive' not in errors + assert errors.count('Full Pi equivalent :') == (1 if benchmark == 'all' else 0) diff --git a/tests/test_cli_loading_signals.py b/tests/test_cli_loading_signals.py new file mode 100644 index 0000000..09712e6 --- /dev/null +++ b/tests/test_cli_loading_signals.py @@ -0,0 +1,343 @@ +"""Native initialization remains inside the CLI's deferred cancellation boundary.""" + +import signal +from threading import Event, Thread, current_thread, main_thread +from typing import Any, List, Optional, Tuple, Type + +import pytest + +from benchset import BenchmarkRunner, cli +from benchset._native import Native +from tests.helpers import assert_runner_reusable + + +@pytest.mark.parametrize('benchmark', ['single_core', 'all']) +@pytest.mark.parametrize('stage', ['before', 'after']) +@pytest.mark.parametrize('policy', ['default', 'custom', 'ignored']) +def test_cli_sigint_during_native_loading_cancels_reports_and_releases_before_reuse( # noqa: PLR0915 — One scope owns actual signal restoration and all worker/handle witnesses. + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], + benchmark: str, stage: str, policy: str, +) -> None: + """Defer startup SIGINT, render its native cancellation, and retain caller policy. + + The original CLI first constructed Native in its owned worker. Even a + caller's custom/ignored SIGINT policy was temporarily deferred there, so + Ctrl+C cancelled initialization and printed the cancelled result with 130. + Eager main-thread argument validation must not bypass that boundary. Emit + real signals immediately before/after real Native initialization; an Event + makes a worker-side loader wait for observed cancellation without sleeps. + Two calls on the same runner check restoration, joined workers and exact + native handle release independently of Thread's own completion bookkeeping. + """ + runner = BenchmarkRunner() + cancelled = Event() + caller_signals: List[int] = [] + workers: List[Tuple[Thread, Event]] = [] + live: List[Tuple[Native, int, Tuple[Any, ...]]] = [] + allocated: List[Tuple[Native, int, Tuple[Any, ...]]] = [] + released: List[Tuple[Native, int, Tuple[Any, ...]]] = [] + armed = False + emitted = 0 + constructions = 0 + original_init, original_create, original_free = Native.__init__, Native.create, Native.free + original_cancel = runner.cancel + + def custom(signum: int, _frame: object) -> None: + caller_signals.append(signum) + + handler = (signal.default_int_handler if policy == 'default' else + signal.SIG_IGN if policy == 'ignored' else custom) + + def emit() -> None: + nonlocal armed, emitted + if armed: + armed = False + emitted += 1 + signal.raise_signal(signal.SIGINT) + if current_thread() is not main_thread(): + assert cancelled.wait(5), 'CLI did not cancel its initializing worker' + + def initialize(native: Native) -> None: + nonlocal constructions + constructions += 1 + if stage == 'before': + emit() + original_init(native) + if stage == 'after': + emit() + + def create(native: Native, *arguments: Any) -> int: + handle = original_create(native, *arguments) + identity = native, handle, arguments + assert not live, 'the CLI overlapped native owners' + live.append(identity) + allocated.append(identity) + return handle + + def free(native: Native, handle: int, *arguments: Any) -> None: + identity = native, handle, arguments + assert live.pop() == identity, 'a native handle was freed twice or with the wrong owner/mode' + released.append(identity) + original_free(native, handle, *arguments) + + def cancel() -> None: + cancelled.set() + original_cancel() + + class Worker(Thread): + def __init__(self, *arguments: Any, **options: Any) -> None: + super().__init__(*arguments, **options) + self.finished = Event() + self.finish_allowed = Event() + workers.append((self, self.finished)) + + def run(self) -> None: + try: + super().run() + finally: + assert self.finish_allowed.wait(5), 'CLI did not join its initialized worker' + self.finished.set() + + def join(self, timeout: Optional[float] = None) -> None: + self.finish_allowed.set() + super().join(timeout) + + monkeypatch.setattr(Native, '__init__', initialize) + monkeypatch.setattr(Native, 'create', create) + monkeypatch.setattr(Native, 'free', free) + monkeypatch.setattr(runner, 'cancel', cancel) + monkeypatch.setattr(cli, 'BenchmarkRunner', lambda: runner) + monkeypatch.setattr(cli, 'Thread', Worker) + previous = signal.signal(signal.SIGINT, handler) + try: + for attempt in range(2): + cancelled.clear() + armed = True + assert cli.main(['--bench=' + benchmark, '--warmup=0', '--duration=.01']) == 130 + output, errors = capsys.readouterr() + assert not output + assert 'Benchmark : single_core' in errors + assert 'Status : cancelled' in errors + assert 'Reason : cancelled before starting' in errors + assert 'Traceback' not in errors + assert errors.count('Full Pi equivalent :') == (1 if benchmark == 'all' else 0) + assert signal.getsignal(signal.SIGINT) is handler + assert not caller_signals, 'startup SIGINT escaped the CLI cancellation policy' + assert emitted == attempt + 1 + assert constructions == attempt + 1, 'the observer must reuse the runner-owned Native' + assert cancelled.is_set() + assert runner._active is None + assert runner._native is None + assert runner._handle is None + assert not live + assert released == allocated + assert len(allocated) == attempt + 1 + assert workers + assert all(done.is_set() and not worker.is_alive() for worker, done in workers) + finally: + armed = False + cancelled.set() + for worker, _done in workers: + worker.join(timeout=5) + signal.signal(signal.SIGINT, previous) + + +@pytest.mark.parametrize('benchmark', ['single_core', 'all']) +@pytest.mark.parametrize('window', ['warmup', 'duration']) +def test_cli_initial_load_error_precedes_invalid_arguments_even_if_a_retry_could_succeed( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], benchmark: str, window: str, +) -> None: + """A later successful open must not reclassify an initial load failure as usage. + + Loading errors have the explicitly accepted CLI precedence and exit 1. + Before the startup fix settings swallowed its first OSError; the worker's + later ordinary invalid-result path still returned 1. A post-result native + validation retry must not turn that load-failure path into argparse exit 2. + """ + original_init = Native.__init__ + attempts = 0 + + def initialize(native: Native) -> None: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise OSError('first native load failed') + original_init(native) + + monkeypatch.setattr(Native, '__init__', initialize) + arguments = ['--bench=' + benchmark, '--warmup=0', '--duration=.01'] + arguments[1 if window == 'warmup' else 2] = '--' + window + '=-1' + assert cli.main(arguments) == 1 + output, errors = capsys.readouterr() + assert not output + assert 'Status : failed' in errors + assert 'usage:' not in errors + assert 'Traceback' not in errors + assert attempts >= 1 + + +@pytest.mark.parametrize('benchmark', ['single_core', 'all']) +@pytest.mark.parametrize('invalid', [False, True]) +@pytest.mark.parametrize('error_type', [ValueError, AttributeError, RuntimeError, OSError, OverflowError]) +def test_cli_optional_duration_inspection_errors_still_defer_to_the_native_run( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], + benchmark: str, invalid: bool, error_type: Type[Exception], +) -> None: + """Optional CLI duration inspection cannot replace the API worker's result. + + settings previously swallowed ordinary duration_error failures, leaving + execution to report its actual validation or cancellation result. The new + observer must retain that fallback without loading another Native object. + Real native cancellation makes valid-input outcomes deterministic, while + invalid input still returns a normal failed result rather than usage exit 2. + """ + runner = BenchmarkRunner() + inspections = 0 + original = error_type('optional duration inspection failed') + + def inspect(_native: Native, _warmup: float, _measurement: float) -> str: + nonlocal inspections + inspections += 1 + raise original + + class Progress(cli._Progress): + def __call__(self, value: Any) -> None: + super().__call__(value) + runner.cancel() + + monkeypatch.setattr(Native, 'duration_error', inspect, raising=False) + monkeypatch.setattr(cli, 'BenchmarkRunner', lambda: runner) + monkeypatch.setattr(cli, '_Progress', Progress) + assert cli.main(['--bench=' + benchmark, '--warmup=0', '--duration=' + ('-1' if invalid else '.1')]) == ( + 1 if invalid else 130 + ) + output, errors = capsys.readouterr() + assert not output + assert 'Status : ' + ('failed' if invalid else 'cancelled') in errors + assert 'optional duration inspection failed' not in errors + assert 'Traceback' not in errors + assert 'usage:' not in errors + assert inspections == 1 + assert runner._active is None + assert runner._native is None + assert runner._handle is None + + +@pytest.mark.parametrize(('arguments', 'reason'), [ + (['--warmup=-1'], '--warmup must be finite and nonnegative'), + (['--warmup=nan'], '--warmup must be finite and nonnegative'), + (['--warmup=inf'], '--warmup must be finite and nonnegative'), + (['--duration=0'], '--duration must be finite and positive'), + (['--duration=-1'], '--duration must be finite and positive'), + (['--duration=nan'], '--duration must be finite and positive'), + (['--duration=inf'], '--duration must be finite and positive'), +]) +def test_standalone_settings_still_rejects_invalid_windows_with_the_rust_cli_diagnostic( + capsys: pytest.CaptureFixture[str], arguments: List[str], reason: str, +) -> None: + """Keep direct settings validation after execution switches to parse-only setup. + + Public settings(argv) must still load Rust, select its CLI spelling and + raise argparse's SystemExit(2) immediately; execution's deferred observer + must not silently turn this standalone helper into a parsing-only API. + """ + with pytest.raises(SystemExit) as caught: + cli.settings(arguments) + assert caught.value.code == 2 + output, errors = capsys.readouterr() + assert not output + assert errors.endswith('benchset: error: ' + reason + '\n') + assert 'usage: benchset' in errors + + +@pytest.mark.parametrize('stage', ['loading', 'inspection']) +@pytest.mark.parametrize('error_type', [ValueError, AttributeError, RuntimeError, OSError, OverflowError]) +def test_standalone_settings_retains_the_optional_loading_and_inspection_error_fallback( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], + stage: str, error_type: Type[Exception], +) -> None: + """The safe helper's implicit-Native path retains ordinary prevalidation fallback. + + A direct settings call still returns parsed arguments when native loading + or optional duration inspection fails, even if duration is invalid. Those + failures belong to the eventual execution boundary; parser errors must not + mask them. This complements the provided-Native observer cases above. + """ + original_init = Native.__init__ + calls: List[str] = [] + + def initialize(native: Native) -> None: + calls.append('loading') + if stage == 'loading': + raise error_type('optional loading failed') + original_init(native) + + def inspect(_native: Native, _warmup: float, _measurement: float) -> str: + calls.append('inspection') + raise error_type('optional inspection failed') + + monkeypatch.setattr(Native, '__init__', initialize) + monkeypatch.setattr(Native, 'duration_error', inspect) + config = cli.settings(['--bench=single_core', '--warmup=0', '--duration=-1']) + assert config.bench == 'single_core' + assert config.warmup == 0.0 + assert config.duration == -1.0 + assert calls == (['loading'] if stage == 'loading' else ['loading', 'inspection']) + assert capsys.readouterr() == ('', '') + + +def test_cli_defers_intermediate_output_if_cancelled_while_releasing_its_signal_handler( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], +) -> None: + """Cancellation just before output handoff must defer rendering until join. + + Invoke the still-owned handler synchronously inside its replacement call. + This selects the narrow pre-installation window on Windows as well as POSIX, + without relying on OS signal delivery or scheduler timing. Real native work + must finish cleanup before any result is rendered, and the runner stays reusable. + """ + runner = BenchmarkRunner() + joined = fired = False + rendered: List[bool] = [] + install, render = signal.signal, cli.render + + def replace(signum: int, handler: Any) -> Any: + nonlocal fired + if not fired and getattr(handler, '__name__', '') == 'interrupt_output': + fired = True + previous = signal.getsignal(signum) + assert callable(previous) + previous(signum, None) + return install(signum, handler) + + class Worker(Thread): + def join(self, timeout: Optional[float] = None) -> None: + nonlocal joined + super().join(timeout) + joined = True + + def show(*arguments: Any) -> None: + rendered.append(joined) + render(*arguments) + + monkeypatch.setattr(cli, 'BenchmarkRunner', lambda: runner) + monkeypatch.setattr(cli, 'Thread', Worker) + monkeypatch.setattr(cli, 'render', show) + previous = install(signal.SIGINT, signal.default_int_handler) + monkeypatch.setattr(signal, 'signal', replace) + try: + assert cli.main(['--warmup=0', '--duration=.02']) == 130 + output, errors = capsys.readouterr() + assert fired + assert joined + assert rendered + assert all(rendered), 'intermediate output escaped before native cleanup' + assert not output + assert 'Full Pi equivalent :' in errors + assert runner._active is None + assert runner._native is None + assert runner._handle is None + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + finally: + install(signal.SIGINT, previous) + assert_runner_reusable(runner) diff --git a/tests/test_cli_observer_signals.py b/tests/test_cli_observer_signals.py new file mode 100644 index 0000000..cd2e3d3 --- /dev/null +++ b/tests/test_cli_observer_signals.py @@ -0,0 +1,111 @@ +"""Keep process-directed SIGINT eligible for the CLI's blocked output thread.""" + +import io +import queue +import signal +import threading +from typing import Callable, List, Optional, Set + +import pytest + +from benchset import ( + AggregatedScores, + BenchmarkResult, + BenchmarkRunner, + Progress, + SuiteResult, + cli, +) + + +@pytest.mark.parametrize('available', [True, False]) +@pytest.mark.parametrize('already_blocked', [True, False]) +@pytest.mark.parametrize('fail_at', ['none', 'queue', 'wait', 'clear']) +def test_cli_acknowledgement_preserves_worker_signal_mask_on_every_exit( # noqa: PLR0915 — Keep worker-mask and fault witnesses within one lifetime. + monkeypatch: pytest.MonkeyPatch, available: bool, already_blocked: bool, fail_at: str, +) -> None: + """Only the parked observer blocks SIGINT, retaining its complete prior mask. + + A fake suite publishes a cancelled snapshot without timed benchmark work. + Publication, waiting or acknowledgement clearing may raise once; the suite + records the mask again when publication returns or raises. Emulated OS + masks exercise both POSIX and platforms without pthread_sigmask on every + host, including a caller that already blocked SIGINT and another signal. + Real CLI threads and acknowledgement events retain the output/join boundary. + """ + initial = {int(signal.SIGTERM)} | ({int(signal.SIGINT)} if already_blocked else set()) + current = initial.copy() + parked: List[Set[int]] = [] + after: List[Set[int]] = [] + published: List[BenchmarkResult] = [] + thread_errors: List[BaseException] = [] + failed = False + block, restore = getattr(signal, 'SIG_BLOCK', 0), getattr(signal, 'SIG_SETMASK', 2) + monkeypatch.setattr(signal, 'SIG_BLOCK', block, raising=False) + monkeypatch.setattr(signal, 'SIG_SETMASK', restore, raising=False) + + def mask(how: int, selected: Set[int]) -> Set[int]: + nonlocal current + assert threading.current_thread() is not threading.main_thread() + previous = current.copy() + current = current | set(selected) if how == block else set(selected) + return previous + + if available: + monkeypatch.setattr(signal, 'pthread_sigmask', mask, raising=False) + else: + monkeypatch.delattr(signal, 'pthread_sigmask', raising=False) + + def checkpoint(stage: str) -> None: + nonlocal failed + if fail_at == stage and not failed: + failed = True + raise RuntimeError('acknowledgement failed') + + class Messages(queue.Queue): + def put(self, item: object, block: bool = True, timeout: Optional[float] = None) -> None: + assert isinstance(item, BenchmarkResult) + published.append(item) + checkpoint('queue') + super().put(item, block, timeout) + + class Acknowledgement(threading.Event): + def wait(self, timeout: Optional[float] = None) -> bool: + parked.append(current.copy()) + checkpoint('wait') + return super().wait(timeout) + + def clear(self) -> None: + checkpoint('clear') + super().clear() + + def suite( + _runner: BenchmarkRunner, _warmup: float, _measurement: float, + _progress: Optional[Callable[[Progress], None]], + on_result: Optional[Callable[[BenchmarkResult], bool]] = None, + ) -> SuiteResult: + result = BenchmarkResult('single_core', status='cancelled') + assert on_result is not None + try: + on_result(result) + finally: + after.append(current.copy()) + return SuiteResult((result,), AggregatedScores()) + + monkeypatch.setattr(BenchmarkRunner, '_run_all', suite) + monkeypatch.setattr(cli, 'ResultEvent', Acknowledgement) + monkeypatch.setattr(cli, 'Queue', Messages) + monkeypatch.setattr(threading, 'excepthook', lambda event: thread_errors.append(event.exc_value)) + output, errors = io.StringIO(), io.StringIO() + monkeypatch.setattr(cli.sys, 'stdout', output) + monkeypatch.setattr(cli.sys, 'stderr', errors) + assert cli.main(['--warmup=0', '--duration=.1']) in (1, 130) + assert not thread_errors + assert published[-1].reason == ('acknowledgement failed' if fail_at != 'none' else None) + assert not output.getvalue() + assert len(parked) == (2 if fail_at in ('wait', 'clear') else 1) + expected = initial | {int(signal.SIGINT)} if available else initial + assert all(value == expected for value in parked) + assert after == [initial] + assert current == initial + assert not any(worker.name == 'benchset-cli' for worker in threading.enumerate()) diff --git a/tests/test_cli_progress_streams.py b/tests/test_cli_progress_streams.py new file mode 100644 index 0000000..5f24dac --- /dev/null +++ b/tests/test_cli_progress_streams.py @@ -0,0 +1,217 @@ +"""Keep terminal presentation local to each CLI child while streaming one suite.""" + +import io +import os +from threading import enumerate as threads +from typing import Callable, List, Optional, Tuple, Type + +import pytest + +from benchset import BenchmarkRunner, Progress, cli +from benchset._native import Native +from tests.helpers import assert_runner_reusable + + +class _Output(io.StringIO): + def __init__(self, terminal: bool) -> None: + super().__init__() + self.terminal = terminal + self.writes: List[str] = [] + self.boundary = 'write' + self.on_result: Optional[Callable[[], None]] = None + self.first_result = False + + def isatty(self) -> bool: + return self.terminal + + def _select(self) -> None: + operation, self.on_result = self.on_result, None + if operation is not None: + operation() + + def write(self, text: str) -> int: + self.writes.append(text) + if text.startswith('Benchmark') and 'single_core' in text: + self.first_result = True + if self.boundary == 'write': + self._select() + if self.first_result and self.boundary == 'separator' and text == '\n': + self._select() + return super().write(text) + + def flush(self) -> None: + if self.first_result and self.boundary == 'flush': + self._select() + super().flush() + + +@pytest.mark.parametrize('initial_terminal', [False, True]) +@pytest.mark.parametrize('replace_stream', [False, True]) +@pytest.mark.parametrize('boundary', ['write', 'flush', 'separator']) +@pytest.mark.parametrize('warmup', [0.0, 0.1]) +def test_cli_uses_the_next_childs_current_output_stream_for_terminal_presentation( + monkeypatch: pytest.MonkeyPatch, initial_terminal: bool, replace_stream: bool, boundary: str, warmup: float, +) -> None: + """Recapture terminal status after result write/flush and separator output. + + Real native execution keeps the first child paused at its observer while + the caller replaces stdout or changes that stream's isatty result. The next + child must emit progress only for its own terminal, and a text destination + must receive neither progress frames nor an earlier child's erase sequence. + Inspect policy at the first callback, regardless of its phase: zero warmup + or a descheduled coordinator can legitimately start with measurement. + """ + initial = _Output(initial_terminal) + destination = _Output(not initial_terminal) if replace_stream else initial + initial.boundary = boundary + selected = False + first_callback: List[Tuple[Progress, bool, bool, int]] = [] + present = cli._Progress.__call__ + + def observe(self: cli._Progress, value: Progress) -> None: + if value.benchmark == 'multi_core' and not first_callback: + first_callback.append((value, self.terminal, cli.sys.stdout is destination, self.width)) + present(self, value) + + def select() -> None: + nonlocal selected + selected = True + if replace_stream: + cli.sys.stdout = destination + else: + destination.terminal = not initial_terminal + + initial.on_result = select + errors = io.StringIO() + monkeypatch.setattr(cli.sys, 'stdout', initial) + monkeypatch.setattr(cli.sys, 'stderr', errors) + monkeypatch.setattr(cli.shutil, 'get_terminal_size', lambda *_args, **_kwargs: os.terminal_size((120, 24))) + monkeypatch.setattr(cli._Progress, '__call__', observe) + assert cli.main([f'--warmup={warmup}', '--duration=.2']) == 0, errors.getvalue() + assert selected + assert not errors.getvalue() + value, terminal, current_stream, width = first_callback[0] + assert terminal is not initial_terminal + assert current_stream + assert width == 0 + text = destination.getvalue() + if not replace_stream: + text = text[text.index('Benchmark'):] + assert 'multi_core' in text + if initial_terminal: + assert '\r' not in text + else: + assert '\rmulti_core: ' in text + first = next(frame for frame in destination.writes if frame.startswith('\rmulti_core: ')) + assert f': {value.phase} ' in first + assert not any(thread.name == 'benchset-cli' for thread in threads()) + assert_runner_reusable(BenchmarkRunner()) + + +def test_cli_next_child_has_no_previous_child_erase_width(monkeypatch: pytest.MonkeyPatch) -> None: + """A fresh child's first frame must not pad using its predecessor's width. + + Observe width before rendering the next child's first callback, even if + scheduling skips its warmup. This catches stale state when measurement's + wider label would otherwise conceal the unwanted erase width. + """ + terminal = _Output(True) + monkeypatch.setattr(cli.sys, 'stdout', terminal) + monkeypatch.setattr(cli.sys, 'stderr', io.StringIO()) + monkeypatch.setattr(cli.shutil, 'get_terminal_size', lambda *_args, **_kwargs: os.terminal_size((120, 24))) + widths: List[int] = [] + present = cli._Progress.__call__ + + def observe(self: cli._Progress, value: Progress) -> None: + if value.benchmark == 'multi_core' and not widths: + widths.append(self.width) + present(self, value) + + monkeypatch.setattr(cli._Progress, '__call__', observe) + assert cli.main(['--warmup=0', '--duration=.2']) == 0 + assert widths == [0] + first = next(frame for frame in terminal.writes if frame.startswith('\rmulti_core: ')) + assert first.endswith('%'), 'a new child must not erase a previous child frame' + + +def test_native_child_keeps_reporting_a_disconnect_after_isatty_changes(monkeypatch: pytest.MonkeyPatch) -> None: + """Only child boundaries recapture isatty; a disconnected child must still fail. + + After the first real progress write, the terminal reports False from isatty + and rejects subsequent progress writes. Rechecking on every callback would + hide that I/O failure and let a disconnected benchmark report success. + """ + class Terminal(_Output): + progress_attempts = 0 + + def write(self, text: str) -> int: + if text.startswith('\r') and not text.strip('\r').isspace(): + self.progress_attempts += 1 + if not self.terminal: + raise OSError('terminal disconnected after first progress') + self.terminal = False + return super().write(text) + + terminal, errors = Terminal(True), io.StringIO() + monkeypatch.setattr(cli.sys, 'stdout', terminal) + monkeypatch.setattr(cli.sys, 'stderr', errors) + assert cli.main(['--bench=single_core', '--warmup=.1', '--duration=.1']) == 1 + assert terminal.progress_attempts >= 2 + assert 'terminal disconnected after first progress' in errors.getvalue() + assert not any(thread.name == 'benchset-cli' for thread in threads()) + assert_runner_reusable(BenchmarkRunner()) + + +@pytest.mark.parametrize('error_type', [ValueError, OSError, KeyboardInterrupt]) +def test_next_child_terminal_inspection_failure_releases_native_state( + monkeypatch: pytest.MonkeyPatch, error_type: Type[BaseException], +) -> None: + """A failing next-child isatty inspection must unblock and free the shared suite. + + HEAD had already freed the first child before inspecting the next output. + The shared Rust observer is still waiting for acknowledgement at this point; + preserve ValueError identity and the existing I/O/interrupt exit codes while + leaving no native handle or CLI worker behind. + """ + original = error_type('next output cannot report its terminal status') + + class Output(_Output): + def isatty(self) -> bool: + if self.first_result: + raise original + return False + + handles = set() + create, free = Native.create, Native.free + present = cli._Progress.__call__ + benchmarks: List[str] = [] + + def record_progress(self: cli._Progress, value: Progress) -> None: + benchmarks.append(value.benchmark) + present(self, value) + + def track_create(self: Native, suite: bool = False) -> int: + handle = create(self, True) if suite else create(self) + handles.add((id(self), handle, suite)) + return handle + + def track_free(self: Native, handle: int, suite: bool = False) -> None: + free(self, handle, True) if suite else free(self, handle) + handles.remove((id(self), handle, suite)) + + monkeypatch.setattr(Native, 'create', track_create) + monkeypatch.setattr(Native, 'free', track_free) + monkeypatch.setattr(cli._Progress, '__call__', record_progress) + monkeypatch.setattr(cli.sys, 'stdout', Output(False)) + monkeypatch.setattr(cli.sys, 'stderr', io.StringIO()) + if error_type is ValueError: + with pytest.raises(ValueError, match='next output cannot report') as caught: + cli.main(['--warmup=.1', '--duration=.1']) + assert caught.value is original + else: + assert cli.main(['--warmup=.1', '--duration=.1']) == (1 if error_type is OSError else 130) + assert not handles + assert 'multi_core' not in benchmarks + assert not any(thread.name == 'benchset-cli' for thread in threads()) + assert_runner_reusable(BenchmarkRunner()) + assert not handles diff --git a/tests/test_cli_signal_policy.py b/tests/test_cli_signal_policy.py new file mode 100644 index 0000000..faec17d --- /dev/null +++ b/tests/test_cli_signal_policy.py @@ -0,0 +1,144 @@ +"""Preserve caller-selected SIGINT policies across streamed CLI suite output.""" + +import io +import signal +from threading import enumerate as threads +from types import FrameType +from typing import Callable, List, Optional, Union + +import pytest + +from benchset import BenchmarkResult, BenchmarkRunner, cli +from benchset._native import Native +from tests.helpers import assert_runner_reusable, mock_suite + +_Handler = Union[int, Callable[[int, Optional[FrameType]], None], None] + + +def _policy(name: str, custom: Callable[[int, Optional[FrameType]], None]) -> _Handler: + if name == 'custom': + return custom + return signal.SIG_IGN if name == 'ignored' else signal.default_int_handler + + +class _Output(io.StringIO): + def __init__(self, handler: _Handler, boundary: str, error: Optional[Exception] = None) -> None: + super().__init__() + self.handler, self.boundary, self.error = handler, boundary, error + self.first_result = self.selected = False + + def _select(self) -> None: + self.selected = True + signal.signal(signal.SIGINT, self.handler) + if self.error is not None: + raise self.error + + def write(self, value: str) -> int: + if value.startswith('Benchmark') and 'single_core' in value: + self.first_result = True + if self.boundary == 'write': + self._select() + return super().write(value) + + def flush(self) -> None: + if self.first_result and not self.selected and self.boundary == 'flush': + self._select() + super().flush() + + +@pytest.mark.parametrize('initial', ['default', 'custom', 'ignored']) +@pytest.mark.parametrize('selected', ['default', 'custom', 'ignored']) +@pytest.mark.parametrize('boundary', ['write', 'flush']) +def test_cli_retains_a_policy_selected_during_intermediate_result_output( + monkeypatch: pytest.MonkeyPatch, initial: str, selected: str, boundary: str, +) -> None: + """A caller may replace SIGINT while writing or flushing the first result. + + The old CLI restored its handler before rendering and captured the caller's + new policy for the next child. Streaming through one Rust suite must retain + that selected policy after its temporary cleanup guard has been released. + Emit another SIGINT after return to verify the handler's actual behavior, + including custom and ignored policies, rather than only its identity. + """ + mock_suite(monkeypatch) + monkeypatch.setattr(BenchmarkRunner, 'run', lambda _self, name, **_options: + BenchmarkResult(name, status='completed', score=1, pi_equivalent=1)) + calls: List[str] = [] + + def initial_custom(_signum: int, _frame: Optional[FrameType]) -> None: + calls.append('initial') + + def selected_custom(_signum: int, _frame: Optional[FrameType]) -> None: + calls.append('selected') + + handler = _policy(selected, selected_custom) + output = _Output(handler, boundary) + monkeypatch.setattr(cli.sys, 'stdout', output) + monkeypatch.setattr(cli.sys, 'stderr', io.StringIO()) + previous = signal.signal(signal.SIGINT, _policy(initial, initial_custom)) + try: + assert cli.main(['--warmup=0', '--duration=.01']) == 0 + assert output.selected + assert 'multi_core' in output.getvalue() + assert signal.getsignal(signal.SIGINT) is handler + if selected == 'default': + with pytest.raises(KeyboardInterrupt): + signal.raise_signal(signal.SIGINT) + else: + signal.raise_signal(signal.SIGINT) + assert calls == (['selected'] if selected == 'custom' else []) + finally: + signal.signal(signal.SIGINT, previous) + + +@pytest.mark.parametrize('selected', ['default', 'custom', 'ignored']) +@pytest.mark.parametrize('boundary', ['write', 'flush']) +def test_cli_output_error_restores_the_selected_policy_after_native_cleanup( + monkeypatch: pytest.MonkeyPatch, selected: str, boundary: str, +) -> None: + """An output error must retain its exact object and the newly selected policy. + + Use real native execution and track every handle through free. The first + intermediate output fails while the shared suite's observer is waiting for + acknowledgement; teardown must unblock and join it before returning the + caller's exception and restoring the caller-selected signal handler. + """ + calls: List[int] = [] + + def custom(signum: int, _frame: Optional[FrameType]) -> None: + calls.append(signum) + + handler = _policy(selected, custom) + original = ValueError('first result output failed') + output = _Output(handler, boundary, original) + monkeypatch.setattr(cli.sys, 'stdout', output) + monkeypatch.setattr(cli.sys, 'stderr', io.StringIO()) + handles = set() + create, free = Native.create, Native.free + + def track_create(self: Native, suite: bool = False) -> int: + handle = create(self, suite) + handles.add((id(self), handle, suite)) + return handle + + def track_free(self: Native, handle: int, suite: bool = False) -> None: + free(self, handle, suite) + handles.remove((id(self), handle, suite)) + + monkeypatch.setattr(Native, 'create', track_create) + monkeypatch.setattr(Native, 'free', track_free) + previous = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + with pytest.raises(ValueError, match='first result output failed') as caught: + cli.main(['--warmup=0', '--duration=.05']) + assert caught.value is original + assert output.selected + assert not handles + assert not any(thread.name == 'benchset-cli' for thread in threads()) + assert signal.getsignal(signal.SIGINT) is handler + assert not calls + assert_runner_reusable(BenchmarkRunner()) + assert not handles + assert signal.getsignal(signal.SIGINT) is handler + finally: + signal.signal(signal.SIGINT, previous) diff --git a/tests/test_cli_terminal_errors.py b/tests/test_cli_terminal_errors.py new file mode 100644 index 0000000..4e9aaf9 --- /dev/null +++ b/tests/test_cli_terminal_errors.py @@ -0,0 +1,134 @@ +"""Keep argument diagnostics ahead of terminal-query errors in the CLI.""" + +import io +import signal +from threading import enumerate as threads +from typing import Any, List, Tuple, Type + +import pytest + +from benchset import BenchmarkRunner, cli +from benchset._native import Native +from tests.helpers import assert_runner_reusable + +_ERRORS = (OSError, ValueError, RuntimeError, KeyboardInterrupt, SystemExit, GeneratorExit) + + +class _Output(io.StringIO): + def __init__(self, error: BaseException) -> None: + super().__init__() + self.error = error + + def isatty(self) -> bool: + raise self.error + + +def _audit_handles(monkeypatch: pytest.MonkeyPatch) -> Tuple[List[Tuple[int, Tuple[Any, ...]]], + List[Tuple[int, Tuple[Any, ...]]]]: + created: List[Tuple[int, Tuple[Any, ...]]] = [] + freed: List[Tuple[int, Tuple[Any, ...]]] = [] + original_create, original_free = Native.create, Native.free + + def create(native: Native, *arguments: Any) -> int: + handle = original_create(native, *arguments) + created.append((handle, arguments)) + return handle + + def free(native: Native, handle: int, *arguments: Any) -> None: + freed.append((handle, arguments)) + original_free(native, handle, *arguments) + + monkeypatch.setattr(Native, 'create', create) + monkeypatch.setattr(Native, 'free', free) + return created, freed + + +@pytest.mark.parametrize('benchmark', ['single_core', 'all']) +@pytest.mark.parametrize(('window', 'value', 'diagnostic'), [ + ('warmup', '-1', '--warmup must be finite and nonnegative'), + ('duration', '0', '--duration must be finite and positive'), +]) +@pytest.mark.parametrize('error_type', _ERRORS) +def test_invalid_cli_ranges_keep_priority_over_terminal_query_failures( # noqa: PLR0913 — Independent range, selection, and stream-failure axes. + monkeypatch: pytest.MonkeyPatch, benchmark: str, window: str, value: str, + diagnostic: str, error_type: Type[BaseException], +) -> None: + """A bad duration must retain argparse exit 2 when Rust is available. + + The original CLI rejected these ranges before asking stdout whether it was + a terminal. Moving Rust validation into the worker must not let an ordinary + terminal-query failure or a stream-raised control exception replace that + diagnostic. Native is real; handle auditing and runner reuse check that the + rejection leaves neither an active worker nor a partially initialized run. + """ + # New argparse versions probe stdout while deciding whether to use colors; + # isolate the later progress-query boundary this test exercises. + monkeypatch.setenv('NO_COLOR', '1') + runner = BenchmarkRunner() + created, freed = _audit_handles(monkeypatch) + output, errors = _Output(error_type('terminal query failed')), io.StringIO() + monkeypatch.setattr(cli, 'BenchmarkRunner', lambda: runner) + previous = signal.getsignal(signal.SIGINT) + arguments = ['--bench=' + benchmark, '--warmup=0', '--duration=.01'] + arguments[1 if window == 'warmup' else 2] = '--' + window + '=' + value + with monkeypatch.context() as streams: + streams.setattr(cli.sys, 'stdout', output) + streams.setattr(cli.sys, 'stderr', errors) + assert cli.main(arguments) == 2 + assert not output.getvalue() + assert errors.getvalue().startswith('usage: benchset ') + assert errors.getvalue().splitlines()[-1] == 'benchset: error: ' + diagnostic + assert 'terminal query failed' not in errors.getvalue() + assert 'Traceback' not in errors.getvalue() + assert signal.getsignal(signal.SIGINT) is previous + assert not any(worker.name == 'benchset-cli' for worker in threads()) + assert not runner._active + assert runner._native is None + assert runner._handle is None + assert created == freed == [] + assert_runner_reusable(runner) + assert created == freed + + +@pytest.mark.parametrize('benchmark', ['single_core', 'all']) +@pytest.mark.parametrize('error_type', _ERRORS) +def test_valid_cli_ranges_preserve_the_original_terminal_query_error_boundary( + monkeypatch: pytest.MonkeyPatch, benchmark: str, error_type: Type[BaseException], +) -> None: + """Validation fallback must not swallow a real stream error for valid inputs. + + OSError remains a CLI diagnostic with exit 1; KeyboardInterrupt remains exit + 130. Other ordinary/control exceptions retain their exact original object. + No benchmark may start just to decide which error should be reported. + """ + # New argparse versions probe stdout while deciding whether to use colors; + # isolate the later progress-query boundary this test exercises. + monkeypatch.setenv('NO_COLOR', '1') + runner = BenchmarkRunner() + created, freed = _audit_handles(monkeypatch) + original = error_type('terminal query failed') + output, errors = _Output(original), io.StringIO() + monkeypatch.setattr(cli, 'BenchmarkRunner', lambda: runner) + previous = signal.getsignal(signal.SIGINT) + with monkeypatch.context() as streams: + streams.setattr(cli.sys, 'stdout', output) + streams.setattr(cli.sys, 'stderr', errors) + arguments = ['--bench=' + benchmark, '--warmup=0', '--duration=.01'] + if error_type is OSError: + assert cli.main(arguments) == 1 + elif error_type is KeyboardInterrupt: + assert cli.main(arguments) == 130 + else: + with pytest.raises(error_type, match='terminal query failed') as caught: + cli.main(arguments) + assert caught.value is original + assert not output.getvalue() + assert errors.getvalue() == ('benchset: error: terminal query failed\n' if error_type is OSError else '') + assert signal.getsignal(signal.SIGINT) is previous + assert not any(worker.name == 'benchset-cli' for worker in threads()) + assert not runner._active + assert runner._native is None + assert runner._handle is None + assert created == freed == [] + assert_runner_reusable(runner) + assert created == freed diff --git a/tests/test_core_contract.py b/tests/test_core_contract.py new file mode 100644 index 0000000..e1cf330 --- /dev/null +++ b/tests/test_core_contract.py @@ -0,0 +1,45 @@ +"""Observable contracts that must survive moving wrapper decisions into Rust.""" + +import math +from typing import List + +import pytest + +from benchset import BenchmarkRunner, Progress, run + + +@pytest.mark.parametrize(('name', 'warmup', 'measurement', 'reason'), [ + ('unknown', -1, 0, 'unknown benchmark: unknown'), + ('', 0, 1, 'unknown benchmark: '), + ('неизвестный\x00benchmark', 0, 1, 'unknown benchmark: неизвестный\x00benchmark'), + ('\ud800', 0, 1, 'unknown benchmark: \ud800'), + ('single_core\udfff', 0, 1, 'unknown benchmark: single_core\udfff'), + ('single_core', -1, 0, 'warmup_seconds must be finite and nonnegative'), + ('multi_core', math.nan, math.inf, 'warmup_seconds must be finite and nonnegative'), + ('single_core', 0, -0.0, 'measurement_seconds must be finite and positive'), + ('multi_core', -0.0, math.nan, 'measurement_seconds must be finite and positive'), +]) +@pytest.mark.parametrize('reusable', [False, True]) +def test_validation_preserves_exact_errors_precedence_and_requested_values( + name: str, warmup: float, measurement: float, reason: str, reusable: bool, +) -> None: + """Keep unknown-name/warmup/measurement precedence and lossless diagnostics. + + Embedded NUL, UTF-8 and lone surrogates guard the string ABI against + truncation or encoding failures. Rejected options have no callbacks, and + requested NaN/signed zero survive. + """ + events: List[Progress] = [] + execute = BenchmarkRunner().run if reusable else run + result = execute(name, warmup_seconds=warmup, measurement_seconds=measurement, on_progress=events.append) + assert result.benchmark == name + assert result.status == 'failed' + assert result.reason == reason + assert result.score is result.pi_equivalent is None + assert not events + for actual, requested in [(result.warmup_seconds, warmup), (result.measurement_seconds, measurement)]: + if math.isnan(requested): + assert math.isnan(actual) + else: + assert actual == requested + assert math.copysign(1, actual) == math.copysign(1, requested) diff --git a/tests/test_errors.py b/tests/test_errors.py index 6c34770..36573c8 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -8,7 +8,7 @@ from benchset import BenchmarkResult, BenchmarkRunner, api, cli, run, run_all from benchset._native import Native -from tests.helpers import assert_result_references +from tests.helpers import assert_result_references, mock_suite def test_missing_library_is_a_failed_result(monkeypatch: pytest.MonkeyPatch) -> None: @@ -96,12 +96,28 @@ def isatty(self) -> bool: return True -def test_terminal_progress(monkeypatch: pytest.MonkeyPatch) -> None: - terminal = Terminal() +def _assert_completed_or_no_work(code: int, output: str, errors: str) -> None: + """Permit the documented short-window outcome while rejecting every other failure.""" + if code == 0: + assert 'completed' in output + assert not errors + else: + assert code == 1 + assert 'failed' in errors + reasons = [line.split(':', 1)[1].strip() for line in errors.splitlines() if line.startswith('Reason ')] + assert reasons == ['measurement window was too short to complete any work'] + + +@pytest.mark.parametrize('duration', ['0.02', '0.000000001']) +def test_terminal_progress(monkeypatch: pytest.MonkeyPatch, duration: str) -> None: + """Terminal measurement frames reach 100% even when no work fits in the window.""" + terminal, errors = Terminal(), io.StringIO() monkeypatch.setenv('COLUMNS', '80') monkeypatch.setattr(cli.sys, 'stdout', terminal) - assert cli.main(['--bench', 'single_core', '--warmup', '0', '--duration', '0.02']) == 0 - assert 'measurement' in terminal.getvalue() + monkeypatch.setattr(cli.sys, 'stderr', errors) + code = cli.main(['--bench', 'single_core', '--warmup', '0', '--duration', duration]) + _assert_completed_or_no_work(code, terminal.getvalue(), errors.getvalue()) + assert '\rsingle_core: measurement ' in terminal.getvalue() assert '100.0%' in terminal.getvalue() @@ -109,6 +125,7 @@ def test_background_failure_reaches_stderr(monkeypatch: pytest.MonkeyPatch, caps def broken(*_arguments: object, **_keywords: object) -> BenchmarkResult: raise RuntimeError('background failure') monkeypatch.setattr(BenchmarkRunner, 'run', broken) + mock_suite(monkeypatch) assert cli.main(['--bench', 'single_core', '--warmup', '0', '--duration', '0.02']) == 1 captured = capsys.readouterr() assert not captured.out @@ -168,7 +185,8 @@ def join(self, timeout: Optional[float] = None) -> None: monkeypatch.setattr(cli, 'BenchmarkRunner', lambda: runner) try: try: - status = cli.main(['--warmup', '0', '--duration', '0.05']) + selection = ['--bench', 'single_core'] if 'join' in boundary else [] + status = cli.main([*selection, '--warmup', '0', '--duration', '0.05']) except KeyboardInterrupt: pytest.fail('SIGINT escaped the CLI during ' + boundary) assert status == 130 @@ -179,7 +197,7 @@ def join(self, timeout: Optional[float] = None) -> None: Thread.join(worker, 5) assert workers assert all(not worker.is_alive() for worker in workers) - assert len(workers) == 1 # An interrupted suite must not start multi_core. + assert len(workers) == 1 # Each command owns one worker, including native suites. assert signal.getsignal(signal.SIGINT) is original_handler captured = capsys.readouterr() assert not captured.out @@ -199,15 +217,35 @@ def start(self) -> None: assert signal.getsignal(signal.SIGINT) is original_handler -def test_cli_can_run_outside_main_thread(capsys: pytest.CaptureFixture[str]) -> None: +@pytest.mark.parametrize('selection', ['single_core', 'all']) +@pytest.mark.parametrize('duration', ['0.02', '0.000000001']) +def test_cli_can_run_outside_main_thread( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], selection: str, duration: str, +) -> None: + """Single and streaming suites never attempt signal installation off the main thread. + + Observe handler changes directly, independent of benchmark scheduling. The + nanosecond control exercises legitimate no-work output and code 1; no other + failure may be accepted. The signal spy records without changing process + state so a broken cleanup loop cannot hang while retrying a forbidden call. + """ results: List[int] = [] + signal_calls: List[int] = [] + original_handler = signal.getsignal(signal.SIGINT) + + def record_signal(signum: int, _handler: object) -> object: + signal_calls.append(signum) + return original_handler + + monkeypatch.setattr(signal, 'signal', record_signal) worker = Thread(target=lambda: results.append(cli.main([ - '--bench', 'single_core', '--warmup', '0', '--duration', '0.02', + '--bench', selection, '--warmup', '0', '--duration', duration, ]))) worker.start() worker.join(5) assert not worker.is_alive() - assert results == [0] + assert len(results) == 1 captured = capsys.readouterr() - assert 'completed' in captured.out - assert not captured.err + _assert_completed_or_no_work(results[0], captured.out, captured.err) + assert not signal_calls + assert signal.getsignal(signal.SIGINT) is original_handler diff --git a/tests/test_exception_contexts.py b/tests/test_exception_contexts.py new file mode 100644 index 0000000..aca0c95 --- /dev/null +++ b/tests/test_exception_contexts.py @@ -0,0 +1,180 @@ +"""Preserve caller exception graphs when a native callback transports an existing error.""" + +from typing import Any, Callable, Dict, NoReturn, Optional, Type + +import pytest + +from benchset import BenchmarkResult, BenchmarkRunner, SuiteResult, api, run_all +from tests.helpers import assert_runner_reusable + + +class _GuardExceptionAttributes: + def __getattribute__(self, name: str) -> Any: + if name in ('__context__', '__cause__', '__suppress_context__'): + raise AssertionError('exception attribute reader must not be called') + return super().__getattribute__(name) + + def __setattr__(self, name: str, value: Any) -> None: + if name in ('__context__', '__cause__', '__suppress_context__'): + raise AssertionError('exception attribute writer must not be called') + super().__setattr__(name, value) + + +class _GuardedValueError(_GuardExceptionAttributes, ValueError): + pass + + +class _GuardedRuntimeError(_GuardExceptionAttributes, RuntimeError): + pass + + +class _GuardedKeyboardInterrupt(_GuardExceptionAttributes, KeyboardInterrupt): + pass + + +_GUARDED: Dict[Type[BaseException], Type[BaseException]] = { + ValueError: _GuardedValueError, + RuntimeError: _GuardedRuntimeError, + KeyboardInterrupt: _GuardedKeyboardInterrupt, +} + + +def _field(error: BaseException, name: str) -> Any: + return BaseException.__dict__[name].__get__(error) + + +def _check_exception_graph( + operation: Callable[[Callable[[], NoReturn]], object], + error_type: Type[BaseException], chain: str, guarded: bool, +) -> None: + outer = LookupError('outer') + inner = ValueError('inner') + cause = TypeError('explicit cause') + error_class = _GUARDED[error_type] if guarded else error_type + original = error_class('caller error') + caught: Optional[BaseException] = None + returned: object = None + + def raise_caller_error() -> NoReturn: + try: + raise inner + except ValueError: + if chain == 'explicit': + raise original from cause + if chain == 'suppressed': + raise original from None + raise original # noqa: B904 — The implicit context is the behavior under test. + + try: + raise outer + except LookupError: + try: + returned = operation(raise_caller_error) + except BaseException as error: # noqa: BLE001 — Assert identity without formatting caller-owned exception chains. + caught = error + assert caught is original, returned + assert _field(original, 'args') == ('caller error',) + assert _field(original, '__context__') is inner + assert _field(inner, '__context__') is outer + assert _field(original, '__cause__') is (cause if chain == 'explicit' else None) + assert _field(original, '__suppress_context__') == (chain != 'implicit') + + +@pytest.mark.parametrize('window', ['warmup', 'measurement']) +@pytest.mark.parametrize('error_type', [ValueError, RuntimeError, KeyboardInterrupt]) +@pytest.mark.parametrize('chain', ['implicit', 'explicit', 'suppressed']) +@pytest.mark.parametrize('guarded', [False, True]) +def test_suite_second_conversion_keeps_caller_exception_graph( + monkeypatch: pytest.MonkeyPatch, window: str, error_type: Type[BaseException], chain: str, guarded: bool, +) -> None: + """An enclosing caller except must not replace the inner converter context after suite cleanup.""" + runner = BenchmarkRunner() + monkeypatch.setattr(api, 'BenchmarkRunner', lambda: runner) + + def operation(raise_error: Callable[[], NoReturn]) -> object: + class Seconds(float): + calls = 0 + + def __float__(self) -> float: + self.calls += 1 + if self.calls == 2: + raise_error() + return float.__float__(self) + + return run_all(warmup_seconds=Seconds(0) if window == 'warmup' else 0, + measurement_seconds=Seconds(.1) if window == 'measurement' else .1) + + _check_exception_graph(operation, error_type, chain, guarded) + assert_runner_reusable(runner) + + +@pytest.mark.parametrize('phase', ['validation', 'execution']) +@pytest.mark.parametrize('chain', ['implicit', 'explicit', 'suppressed']) +@pytest.mark.parametrize('guarded', [False, True]) +def test_name_matcher_keeps_caller_exception_graph(phase: str, chain: str, guarded: bool) -> None: + """Both name-comparison boundaries retain the user's graph before or after handle allocation.""" + runner = BenchmarkRunner() + + def operation(raise_error: Callable[[], NoReturn]) -> object: + class Name(str): + __slots__ = ('__dict__',) + __hash__ = str.__hash__ + calls = 0 + + def __eq__(self, other: object) -> bool: + self.calls += 1 + if self.calls == (1 if phase == 'validation' else 2): + raise_error() + return str.__eq__(self, other) + + return runner.run(Name('single_core'), warmup_seconds=0, measurement_seconds=.01) + + _check_exception_graph(operation, ValueError, chain, guarded) + assert_runner_reusable(runner) + + +@pytest.mark.parametrize('primitive', ['success', 'ratio', 'name', 'ge', 'eq']) +@pytest.mark.parametrize('chain', ['implicit', 'explicit', 'suppressed']) +@pytest.mark.parametrize('guarded', [False, True]) +def test_aggregate_callback_keeps_caller_exception_graph(primitive: str, chain: str, guarded: bool) -> None: + """Lazy aggregate readers rethrow each caller property/operator with its complete original graph.""" + def operation(raise_error: Callable[[], NoReturn]) -> object: + class Result(BenchmarkResult): + @property + def success(self) -> bool: + if primitive == 'success': + raise_error() + return super().success + + def __getattribute__(self, name: str) -> Any: + if primitive == 'ratio' and name == 'pi_equivalent': + raise_error() + return super().__getattribute__(name) + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + raise_error() + + class Ratio(float): + __hash__ = float.__hash__ + + def __ge__(self, other: float) -> bool: + if primitive == 'ge': + raise_error() + return float.__ge__(self, other) + + def __eq__(self, other: object) -> bool: + raise_error() + + name = Name('single_core') if primitive == 'name' else 'single_core' + ratio = Ratio(1) if primitive in ('ge', 'eq') else 1 + return SuiteResult.from_results((Result(name, status='completed', pi_equivalent=ratio), + BenchmarkResult('multi_core', status='completed', pi_equivalent=4))) + + _check_exception_graph(operation, ValueError, chain, guarded) + valid = (BenchmarkResult('single_core', status='completed', pi_equivalent=1), + BenchmarkResult('multi_core', status='completed', pi_equivalent=4)) + assert SuiteResult.from_results(valid).aggregated_scores.pi_equivalent == 2 diff --git a/tests/test_float_transport.py b/tests/test_float_transport.py new file mode 100644 index 0000000..de7ffdd --- /dev/null +++ b/tests/test_float_transport.py @@ -0,0 +1,141 @@ +"""Keep Python's existing numeric-conversion protocol across quiet FFI copies.""" + +import ctypes +import subprocess +import sys +from typing import Any, List, Type, cast + +import pytest + +from benchset import BENCHMARKS, BenchmarkResult, SuiteResult, _native + + +@pytest.mark.parametrize('position', [0, 1]) +@pytest.mark.parametrize('override', ['different', 'error']) +def test_float_subclass_payload_does_not_invoke_its_float_override(position: int, override: str) -> None: + """PyFloat_AsDouble reads float-subclass storage without calling __float__. + + Quiet copying must retain this protocol rather than normalize every value + with Python float(), which would change either the score or its exception. + Integer subclasses have a separately approved conversion-based contract. + """ + calls: List[str] = [] + + class Ratio(float): + __slots__ = () + + def __float__(self) -> float: + calls.append('float') + if override == 'error': + raise RuntimeError('must not be called') + return 4.0 + + ratios = [1.0, 1.0] + ratios[position] = Ratio(16.0) + results = tuple(BenchmarkResult(name, status='completed', pi_equivalent=value) + for name, value in zip(BENCHMARKS, ratios)) + assert SuiteResult.from_results(results).aggregated_scores.pi_equivalent == pytest.approx(4.0) + assert not calls + + +@pytest.mark.parametrize('position', [0, 1]) +@pytest.mark.parametrize('stage', [0, 1], ids=['validation', 'logarithm']) +@pytest.mark.parametrize('error_type', [TypeError, RuntimeError, KeyboardInterrupt]) +def test_numeric_conversion_errors_preserve_identity_and_context( + position: int, stage: int, error_type: Type[BaseException], +) -> None: + """A C-API numeric conversion error must survive the quiet Rust copy unchanged. + + The getter may fail at validation or at the fresh log read. Python's error + indicator must cross the primitive without executing any further user + operations, replacing the exception, or changing its explicit cause. + """ + original, cause = error_type('conversion failed'), LookupError('cause') + events: List[str] = [] + + class Ratio: + __hash__ = object.__hash__ + + def __float__(self) -> float: + events.append('float') + if events.count('float') - 1 == stage: + raise original from cause + return 4.0 + + def __ge__(self, other: Any) -> bool: + assert type(other) is int + assert other == 0 + events.append('ge') + return True + + def __eq__(self, other: object) -> bool: + assert type(other) is int + assert other == 0 + events.append('eq') + return False + + ratios: List[Any] = [1.0, 1.0] + ratios[position] = Ratio() + results = tuple(BenchmarkResult(name, status='completed', pi_equivalent=value) + for name, value in zip(BENCHMARKS, ratios)) + with pytest.raises(error_type) as caught: + SuiteResult.from_results(results) + assert caught.value is original + assert original.__cause__ is cause + assert original.__context__ is None + assert original.__suppress_context__ + assert events == (['float'] if stage == 0 else ['float', 'ge', 'eq', 'float']) + + +@pytest.mark.parametrize('position', [0, 1]) +def test_non_numeric_strings_are_not_accepted_by_the_copy_primitive(position: int) -> None: + """Numeric-looking strings keep PyFloat_AsDouble's TypeError, not float(str).""" + value = '4' + with pytest.raises(TypeError) as expected: + ctypes.c_double(cast(Any, value)) + ratios: List[Any] = [1.0, 1.0] + ratios[position] = value + results = tuple(BenchmarkResult(name, status='completed', pi_equivalent=ratio) + for name, ratio in zip(BENCHMARKS, ratios)) + with pytest.raises(TypeError) as actual: + SuiteResult.from_results(results) + assert str(actual.value) == str(expected.value) + + +def test_rejected_copy_cannot_publish_an_uninitialized_number(monkeypatch: pytest.MonkeyPatch) -> None: + """A failed native copy is an error, never a fabricated zero coefficient.""" + monkeypatch.setattr(_native, '_DoubleCopy', lambda _address: lambda *_arguments: 0) + results = tuple(BenchmarkResult(name, status='completed', pi_equivalent=4.0) for name in BENCHMARKS) + with pytest.raises(RuntimeError, match='native numeric conversion is unavailable'): + SuiteResult.from_results(results) + + +def test_numeric_reader_keeps_python_thread_state_in_a_subprocess() -> None: + """The C-API reader must retain the GIL/thread state during user conversion. + + A regular ctypes call detaches that state and can crash while the C API + invokes __float__. Isolate this regression so a bad bridge cannot terminate + the test runner; the child verifies both conversions and the final score. + """ + code = '''import ctypes +import sys +from pathlib import Path +sys.path.insert(0, str(Path.cwd())) +from benchset import BENCHMARKS, BenchmarkResult, SuiteResult +check = ctypes.pythonapi.PyGILState_Check +check.restype = ctypes.c_int +check.argtypes = [] +calls = [] +class Ratio: + def __float__(self): + assert check() == 1 + calls.append('float') + return 4.0 + def __ge__(self, other): return True + def __eq__(self, other): return False +results = tuple(BenchmarkResult(name, status='completed', pi_equivalent=Ratio()) for name in BENCHMARKS) +assert SuiteResult.from_results(results).aggregated_scores.pi_equivalent == 4.0 +assert calls == ['float'] * 4 +''' + completed = subprocess.run([sys.executable, '-c', code], capture_output=True, text=True, check=False, timeout=30) + assert completed.returncode == 0, completed.stdout + completed.stderr diff --git a/tests/test_installer_copy.py b/tests/test_installer_copy.py new file mode 100644 index 0000000..6db6279 --- /dev/null +++ b/tests/test_installer_copy.py @@ -0,0 +1,379 @@ +"""Filter the publication lock without changing registered installer copying.""" + +from __future__ import annotations + +import os +import stat +from distutils.errors import DistutilsFileError # type: ignore[import-not-found] +from pathlib import Path +from typing import Any, Dict, Tuple + +import pytest +from filelock import FileLock + +from tests import test_setup + +installer_setup = test_setup.setup_module + + +def _source(directory: Path) -> Tuple[Path, Dict[str, bytes]]: + """Create complete opaque build outputs; no compiler or native load is needed.""" + contents = { + 'benchset/__init__.py': b'ANSWER = 42\n', + 'benchset/_metadata.json': b'{"generation": "complete"}\n', + 'benchset/_native/libbenchset.so': b'complete native image', + 'benchset/data/weights/table.bin': b'calibration payload', + } + for name, data in contents.items(): + path = directory / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return directory, contents + + +def _command(setup: Dict[str, Any], source: Path, destination: object) -> Any: + command = setup['distribution'].get_command_obj('install_lib') + command.build_dir, command.install_dir = str(source), destination + return command + + +def _link(path: Path, target: Path, *, directory: bool) -> None: + """Skip only platforms that cannot create the required symbolic link.""" + try: + path.symlink_to(target, target_is_directory=directory) + except NotImplementedError: + pytest.skip('Symbolic links are unavailable') + except OSError as error: + if os.name == 'nt' and getattr(error, 'winerror', None) == 1314: + pytest.skip('Creating Windows symbolic links requires privileges') + raise + + +@pytest.mark.parametrize('linked', ['package', 'native', 'nested']) +@pytest.mark.parametrize('spelling', ['plain', 'dot']) +def test_registered_installer_follows_directory_links_and_reports_every_file( + installer_setup: Dict[str, Any], tmp_path: Path, linked: str, spelling: str, +) -> None: + """Directory links remain usable build outputs, including native products. + + The normal setuptools copier followed directory links. Activating its + archive copier merely to exclude a lock silently omits these directories, + yielding missing native code or even an entirely absent Python package. + Exercise the registered command, both deep and top-level links, and raw + destination spelling rather than depending on a particular copy backend. + """ + source, contents = _source(tmp_path / 'build') + relative = {'package': 'benchset', 'native': 'benchset/_native', + 'nested': 'benchset/data/weights'}[linked] + original = source / relative + actual = tmp_path / 'actual-products' + original.rename(actual) + _link(original, actual, directory=True) + output = tmp_path / 'installed' + raw = str(output) if spelling == 'plain' else str(tmp_path) + '/./installed' + command = _command(installer_setup, source, raw) + returned = command.install() + assert {Path(os.path.relpath(path, raw)).as_posix() for path in returned} == set(contents) + for name, data in contents.items(): + installed = output / name + assert installed.is_file() + assert not installed.is_symlink() + assert installed.read_bytes() == data + + +@pytest.mark.parametrize('force', [False, True]) +@pytest.mark.parametrize('destination_age', ['older', 'equal', 'newer']) +def test_registered_installer_retains_force_and_timestamp_update_semantics( + installer_setup: Dict[str, Any], tmp_path: Path, force: bool, destination_age: str, +) -> None: + """Without force, equal or newer installed data must not be overwritten. + + An older destination is updated and force overrides all timestamp cases. + The returned output list still includes a file whose copy was skipped, + preserving setuptools' reporting contract for incremental installations. + """ + source, _contents = _source(tmp_path / 'build') + original = source / 'benchset/__init__.py' + output = tmp_path / 'installed' + destination = output / 'benchset/__init__.py' + destination.parent.mkdir(parents=True) + destination.write_bytes(b'newer local data') + timestamp = 1_600_000_000 + offset = {'older': -10, 'equal': 0, 'newer': 10}[destination_age] + os.utime(original, (timestamp, timestamp)) + os.utime(destination, (timestamp + offset, timestamp + offset)) + command = _command(installer_setup, source, str(output)) + command.force = force + returned = command.install() + copied = force or destination_age == 'older' + assert destination.read_bytes() == (original.read_bytes() if copied else b'newer local data') + assert destination.stat().st_mtime == (timestamp if copied else timestamp + offset) + assert str(destination) in returned + + +@pytest.mark.parametrize('existing', [False, True]) +@pytest.mark.parametrize('force', [False, True]) +def test_registered_installer_dry_run_reports_outputs_without_writing( + installer_setup: Dict[str, Any], tmp_path: Path, existing: bool, force: bool, +) -> None: + """A dry run predicts files while preserving existing data and directories. + + Supported older setuptools honored this flag before lock filtering selected + an archive copier that writes unconditionally. Preserve the intended dry-run + contract on newer setuptools too, including force=True and a fresh target. + """ + source, contents = _source(tmp_path / 'build') + output = tmp_path / 'installed' + preserved = output / 'keep.txt' + if existing: + output.mkdir() + preserved.write_bytes(b'caller data') + before = sorted(str(path.relative_to(output)) for path in output.rglob('*')) if existing else [] + command = _command(installer_setup, source, str(output)) + command.dry_run, command.force = True, force + returned = command.install() + assert {Path(os.path.relpath(path, output)).as_posix() for path in returned} == set(contents) + assert output.exists() is existing + assert sorted(str(path.relative_to(output)) for path in output.rglob('*')) == before + if existing: + assert preserved.read_bytes() == b'caller data' + + +def test_registered_installer_skips_nfs_tombstones_at_every_depth( + installer_setup: Dict[str, Any], tmp_path: Path, +) -> None: + """Transient NFS files and directories must not become installed resources. + + The original copier skipped .nfs* entries while copying normal hidden data. + Keep that behavior when walking linked native-product directories as well. + """ + source, contents = _source(tmp_path / 'build') + for name in ('.nfs-root', 'benchset/_native/.nfs-library', 'benchset/.nfs-dir/stale'): + path = source / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b'open deleted file') + normal = source / 'benchset/.keep' + normal.write_bytes(b'ordinary hidden resource') + contents['benchset/.keep'] = normal.read_bytes() + output = tmp_path / 'installed' + returned = _command(installer_setup, source, str(output)).install() + assert {Path(os.path.relpath(path, output)).as_posix() for path in returned} == set(contents) + assert {path.relative_to(output).as_posix() for path in output.rglob('*') if path.is_file()} == set(contents) + + +@pytest.mark.skipif(os.name != 'posix', reason='POSIX file and directory permission semantics') +@pytest.mark.parametrize('existing', [False, True]) +def test_registered_installer_preserves_file_metadata_and_existing_directory_modes( + installer_setup: Dict[str, Any], tmp_path: Path, existing: bool, +) -> None: + """File modes/mtimes survive copying without inheriting source directory modes. + + Reusing command copy helpers preserves executable and read-only files, keeps + an existing installation directory's permissions, and honors the creation + umask for new directories. This rejects an unfiltered copytree replacement. + """ + source, contents = _source(tmp_path / 'build') + (source / 'benchset').chmod(0o770) + modes = {'benchset/__init__.py': 0o640, 'benchset/_native/libbenchset.so': 0o444} + timestamp = 1_600_000_000 + for name, mode in modes.items(): + path = source / name + path.chmod(mode) + os.utime(path, (timestamp, timestamp)) + output = tmp_path / 'installed' + package = output / 'benchset' + if existing: + package.mkdir(parents=True) + package.chmod(0o705) + mask = os.umask(0o022) + try: + _command(installer_setup, source, str(output)).install() + finally: + os.umask(mask) + assert stat.S_IMODE(package.stat().st_mode) == (0o705 if existing else 0o755) + for name, mode in modes.items(): + copied = output / name + assert copied.read_bytes() == contents[name] + assert stat.S_IMODE(copied.stat().st_mode) == mode + assert copied.stat().st_mtime == timestamp + + +def test_registered_installer_keeps_empty_directories_and_dereferences_file_links( + installer_setup: Dict[str, Any], tmp_path: Path, +) -> None: + """Tree copying retains empty directories and copies file-link contents. + + A filter must not drop directories with no regular files or turn a source + file symlink into an installed symlink when preserve_symlinks is false. + """ + source, _contents = _source(tmp_path / 'build') + (source / 'benchset/empty').mkdir() + link = source / 'benchset/linked.bin' + _link(link, source / 'benchset/data/weights/table.bin', directory=False) + output = tmp_path / 'installed' + _command(installer_setup, source, str(output)).install() + assert (output / 'benchset/empty').is_dir() + copied = output / 'benchset/linked.bin' + assert copied.read_bytes() == b'calibration payload' + assert not copied.is_symlink() + + +@pytest.mark.parametrize('kind', ['missing', 'file']) +def test_registered_tree_copy_rejects_a_non_directory_input( + installer_setup: Dict[str, Any], tmp_path: Path, kind: str, +) -> None: + """Invalid copy roots raise the normal distutils error rather than succeeding. + + os.walk alone yields no files for these inputs. Keep the original explicit + directory validation and leave the proposed installation target untouched. + """ + source = tmp_path / 'source' + if kind == 'file': + source.write_bytes(b'not a directory') + output = tmp_path / 'installed' + command = _command(installer_setup, source, str(output)) + with pytest.raises(DistutilsFileError, match='not a directory'): + command.copy_tree(str(source), str(output)) + assert not output.exists() + + +@pytest.mark.parametrize('kind', ['missing', 'file']) +def test_registered_tree_dry_run_tolerates_an_unavailable_input( + installer_setup: Dict[str, Any], tmp_path: Path, kind: str, +) -> None: + """A direct dry-run copy treats missing or non-directory input as empty. + + The original copier could predict no outputs before a build had produced + its directory. Real copying must still reject the same invalid roots. + """ + source = tmp_path / 'source' + if kind == 'file': + source.write_bytes(b'not a directory') + output = tmp_path / 'installed' + command = _command(installer_setup, source, str(output)) + command.dry_run = True + assert command.copy_tree(str(source), str(output)) == [] + assert not output.exists() + + +@pytest.mark.skipif(os.name != 'posix' or getattr(os, 'geteuid', lambda: 0)() == 0, + reason='POSIX DAC permissions must be enforced for the test user') +def test_registered_installer_reports_an_unreadable_subdirectory( + installer_setup: Dict[str, Any], tmp_path: Path, +) -> None: + """Unreadable input must fail visibly instead of installing a partial tree. + + Archive os.walk silently discarded scan errors. A native or package-data + directory that cannot be enumerated must retain the usual distutils error. + """ + source, _contents = _source(tmp_path / 'build') + protected = source / 'benchset/private' + protected.mkdir() + (protected / 'data.bin').write_bytes(b'unreadable data') + protected.chmod(0) + output = tmp_path / 'installed' + try: + with pytest.raises(DistutilsFileError, match='error listing files'): + _command(installer_setup, source, str(output)).install() + finally: + protected.chmod(0o700) + + +@pytest.mark.skipif(os.name != 'posix' or getattr(os, 'geteuid', lambda: 0)() == 0, + reason='POSIX DAC permissions must be enforced for the test user') +@pytest.mark.parametrize('method', ['copy_tree', 'install']) +@pytest.mark.parametrize('blocked', ['root', 'nested']) +def test_registered_installer_dry_run_skips_unreadable_directories( + installer_setup: Dict[str, Any], tmp_path: Path, method: str, blocked: str, +) -> None: + """Dry runs ignore scan errors while retaining every readable planned file. + + An unreadable root predicts no files; an unreadable child must not discard + readable siblings. Exercise direct copying and the registered installation + path without weakening the real-copy error contract or creating outputs. + """ + source, contents = _source(tmp_path / 'build') + protected = source if blocked == 'root' else source / 'benchset/private' + protected.mkdir(exist_ok=True) + (protected / 'hidden.bin').write_bytes(b'inaccessible data') + protected.chmod(0) + output = tmp_path / 'installed' + command = _command(installer_setup, source, str(output)) + command.dry_run = True + try: + returned = command.copy_tree(str(source), str(output)) if method == 'copy_tree' else command.install() + finally: + protected.chmod(0o700) + assert {os.path.relpath(path, output) for path in returned} == (set() if blocked == 'root' else set(contents)) + assert not output.exists() + + +@pytest.mark.parametrize('spelling', ['plain', 'dot']) +def test_registered_installer_excludes_publication_locks_without_touching_live_inodes( + installer_setup: Dict[str, Any], tmp_path: Path, monkeypatch: pytest.MonkeyPatch, spelling: str, +) -> None: + """Filtering skips only the publication lock before copying or removing it. + + Hold both source and existing destination locks. Neither live inode nor its + timestamp may be changed by installation; ordinary other-package lock data + must still be copied. A post-copy unlink would break the destination owner. + Also inspect paths during each file copy, so temporarily renaming a source + lock cannot hide behind otherwise identical before/after inode snapshots. + """ + source, contents = _source(tmp_path / 'build') + other = source / 'other/.benchset.publish.lock' + other.parent.mkdir() + other.write_bytes(b'other package data') + contents['other/.benchset.publish.lock'] = other.read_bytes() + output = tmp_path / 'installed' + destination = output / 'benchset/.benchset.publish.lock' + destination.parent.mkdir(parents=True) + original = source / 'benchset/.benchset.publish.lock' + raw = str(output) if spelling == 'plain' else str(tmp_path) + '/./installed' + with FileLock(str(original)), FileLock(str(destination)): + os.utime(original, (1_600_000_000, 1_600_000_000)) + os.utime(destination, (1_600_000_010, 1_600_000_010)) + + def states() -> list[tuple[int, int]]: + return [(path.stat().st_ino, path.stat().st_mtime_ns) for path in (original, destination)] + + before = states() + command = _command(installer_setup, source, raw) + command.force = True + copy = command.copy_file + + def observe(*arguments: Any, **options: Any) -> Any: + assert states() == before + return copy(*arguments, **options) + + monkeypatch.setattr(command, 'copy_file', observe) + returned = command.install() + assert states() == before + assert {Path(os.path.relpath(path, raw)).as_posix() for path in returned} == set(contents) + for name, data in contents.items(): + assert (output / name).read_bytes() == data + + +def test_registered_installer_retains_inherited_namespace_exclusions( + installer_setup: Dict[str, Any], tmp_path: Path, +) -> None: + """The lock filter keeps setuptools' existing namespace-package exclusions. + + Replacing the copying backend must not reinstall namespace __init__ files + while excluding the new lock. Other namespace resources remain available. + """ + source, _contents = _source(tmp_path / 'build') + namespace = source / 'legacy_namespace' + namespace.mkdir() + (namespace / '__init__.py').write_bytes(b'namespace marker') + (namespace / 'data.bin').write_bytes(b'namespace resource') + distribution = installer_setup['distribution'] + distribution.namespace_packages = ['legacy_namespace'] + install = distribution.get_command_obj('install') + install.single_version_externally_managed = True + install.record = str(tmp_path / 'installed-files.txt') + output = tmp_path / 'installed' + returned = _command(installer_setup, source, str(output)).install() + assert not (output / 'legacy_namespace/__init__.py').exists() + assert str(output / 'legacy_namespace/__init__.py') not in returned + assert (output / 'legacy_namespace/data.bin').read_bytes() == b'namespace resource' diff --git a/tests/test_interpreter_boundary.py b/tests/test_interpreter_boundary.py new file mode 100644 index 0000000..3d17f73 --- /dev/null +++ b/tests/test_interpreter_boundary.py @@ -0,0 +1,241 @@ +"""Preserve public APIs on ctypes implementations without CPython's C API.""" + +import ctypes +import os +import shutil +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Any, List + +import pytest + +from benchset import _native +from tests import test_quiet_validation + +quiet_environment = test_quiet_validation.quiet_environment + +_PUBLIC = '''import ctypes, _ctypes, math, sys +from pathlib import Path +root = Path(sys.argv[1]) +sys.path.insert(0, str(root)) +mock = sys.argv[2] == 'mock' +if mock: + del ctypes.pythonapi +import benchset +from benchset import _native +loads = [] +if mock: + path = root / 'benchset/_native' / ('benchset.dll' if sys.platform == 'win32' else 'libbenchset.dylib' if sys.platform == 'darwin' else 'libbenchset.so') + library = ctypes.CDLL(str(path)) + def factory(name, **kwargs): + assert str(path) == name and not kwargs + loads.append(name) + return library + ctypes.CDLL = ctypes.cdll._dlltype = factory + def forbidden(*args): raise AssertionError('manual dlopen cannot own this handle') + ctypes._dlopen = forbidden +first, second = _native.Native(), _native.Native() +assert first.lib.benchset_abi_version is not second.lib.benchset_abi_version +ratios = tuple(benchset.BenchmarkResult(name, status='completed', pi_equivalent=ratio) for name, ratio in zip(benchset.BENCHMARKS, (1., 4.))) +assert benchset.SuiteResult.from_results(ratios).aggregated_scores.pi_equivalent == 2. +unknown = benchset.run('unknown', warmup_seconds=0, measurement_seconds=.1) +assert (unknown.status, unknown.reason) == ('failed', 'unknown benchmark: unknown') +single = benchset.run('single_core', warmup_seconds=0, measurement_seconds=.1) +assert single.success and single.pi_equivalent is not None, single +suite = benchset.run_all(warmup_seconds=0, measurement_seconds=.1) +assert [item.status for item in suite.results] == ['completed', 'completed'], suite +assert suite.aggregated_scores.pi_equivalent is not None +runner = benchset.BenchmarkRunner() +assert runner.run('single_core', warmup_seconds=0, measurement_seconds=.1, on_progress=lambda p: runner.cancel()).status == 'cancelled' +assert runner.run('single_core', warmup_seconds=0, measurement_seconds=.1).success +for parent in (float, int): + calls = [] + class Ratio(parent): + def __float__(self): calls.append('float'); return 4. + values = (benchset.BenchmarkResult('single_core', status='completed', pi_equivalent=Ratio(16)), benchset.BenchmarkResult('multi_core', status='completed', pi_equivalent=1.)) + assert benchset.SuiteResult.from_results(values).aggregated_scores.pi_equivalent == 2. + assert calls == ['float', 'float'] +for stage in (0, 1): + calls = [] + original, cause = RuntimeError('conversion failed'), LookupError('cause') + class Ratio: + def __float__(self): + calls.append('float') + if calls.count('float') - 1 == stage: raise original from cause + return 4. + def __ge__(self, other): return True + def __eq__(self, other): return False + values = (benchset.BenchmarkResult('single_core', status='completed', pi_equivalent=Ratio()), benchset.BenchmarkResult('multi_core', status='completed', pi_equivalent=1.)) + try: benchset.SuiteResult.from_results(values) + except RuntimeError as caught: + assert caught is original and caught.__cause__ is cause and caught.__context__ is None + else: raise AssertionError('conversion error was lost') + assert calls == ['float'] * (stage + 1) +values = (benchset.BenchmarkResult('single_core', status='completed', pi_equivalent='4'), benchset.BenchmarkResult('multi_core', status='completed', pi_equivalent=1.)) +try: math.isfinite('4') +except TypeError as expected: message = str(expected) +try: benchset.SuiteResult.from_results(values) +except TypeError as error: assert str(error) == message +else: raise AssertionError('numeric strings were accepted') +if mock: assert len(loads) == 1 +print('interpreter API compatible') +''' + + +def _pypy() -> str: + executable = os.environ.get('BENCHSET_TEST_PYPY') or shutil.which('pypy3') + if executable is None: + pytest.skip('Set BENCHSET_TEST_PYPY or provide pypy3 for interpreter compatibility checks') + return str(Path(executable).resolve()) + + +@pytest.mark.parametrize('implementation', ['mock', 'pypy']) +def test_public_api_without_cpython_api(implementation: str) -> None: + """Missing pythonapi must not break imports, runs, suites or host conversion. + + Real PyPy characterizes the supported alternative interpreter. A CPython + capability mock additionally exercises these branches during ordinary + coverage runs, including the standard loader for opaque handles. The child + checks subclass conversion, fresh log reads, original errors and strings. + """ + root = Path(__file__).resolve().parents[1] + executable = _pypy() if implementation == 'pypy' else sys.executable + environment = dict(os.environ, LC_ALL='C', LANG='C', PYTHONPATH=str(root)) + completed = subprocess.run([executable, '-c', _PUBLIC, str(root), implementation], env=environment, + capture_output=True, text=True, check=False, timeout=60) + assert completed.returncode == 0, completed.stdout + completed.stderr + assert 'interpreter API compatible' in completed.stdout + + +@pytest.mark.parametrize('kind', ['float', 'int', 'object']) +@pytest.mark.parametrize('write_output', [False, True]) +def test_missing_c_api_numeric_copy_preserves_conversion_and_destination( + monkeypatch: pytest.MonkeyPatch, kind: str, write_output: bool, +) -> None: + """The fallback converts once and writes either its holder or a borrowed field. + + PyPy math converts numeric subclasses through __float__; its ctypes scalar + constructor alone ignores that override. Normalize such subclasses before + copying while preserving the ordinary protocol for other numeric objects. + Canaries around the output also guard the borrowed field's exact width. + """ + calls: List[object] = [] + + class Number: + def __float__(self) -> float: + calls.append(self) + return 4.0 + + parent = {'float': float, 'int': int, 'object': object}[kind] + value_type = type('Ratio', (Number, parent), {}) + value = value_type() if kind == 'object' else value_type(16) + native = _native.Native() + scalar_type = ctypes.c_double + output = (scalar_type * 3)(11.0, -123.0, 13.0) + + def scalar(value: Any = 0.0) -> Any: + # Model PyPy's scalar constructor, which reads numeric subclass storage. + if issubclass(type(value), float): + value = float.__float__(value) + elif issubclass(type(value), int): + value = int.__float__(value) + return scalar_type(value) + + monkeypatch.setattr(_native, '_AsDouble', None) + monkeypatch.setattr(ctypes, 'c_double', scalar) + number = native.double(value, ctypes.byref(output, ctypes.sizeof(scalar_type)) if write_output else None) + assert (output[1] if write_output else number.value) == 4.0 + assert output[0] == 11.0 + assert output[2] == 13.0 + if not write_output: + assert output[1] == -123.0 + assert calls == [value] + + +def test_alternative_ctypes_function_owner_keeps_instances_independent(monkeypatch: pytest.MonkeyPatch) -> None: + """A function exposing its owner as dll must still receive a separate signature. + + PyPy uses dll rather than CPython's ownership-dictionary entry. Exercise + that shape with a real address-bound ctypes function, so ordinary CPython + runs cover the independent-binding branch without a fake function object. + """ + original = _native.Native().lib.benchset_abi_version + library: Any = SimpleNamespace(benchset_abi_version=original) + monkeypatch.setattr(original, 'dll', library, raising=False) + first = _native._Functions(library).benchset_abi_version + second = _native._Functions(library).benchset_abi_version + assert first is not original + assert first is not second + first.argtypes, first.restype = [], ctypes.c_uint32 + second.argtypes, second.restype = [ctypes.c_int], ctypes.c_uint64 + assert first() == _native.ABI_VERSION + assert first.argtypes == [] + assert first.restype is ctypes.c_uint32 + + +def test_alternative_interpreter_quiet_rejection_preserves_flags_and_payloads(quiet_environment: Any) -> None: + """Real PyPy rejects NaN/inf quietly without losing payloads or existing flags. + + PyPy's interpreter can already set FE_INEXACT during ctypes dispatch, so + compare against the immediately preceding state rather than assume zero. + A seeded divide-by-zero flag must survive aggregate and both run modes. + """ + executable = _pypy() + root = Path(__file__).resolve().parents[1] + code = '''import ctypes, struct, sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +from benchset import BENCHMARKS, BenchmarkResult, SuiteResult, run, run_all +from benchset._native import Native +native = Native() +SuiteResult.from_results(tuple(BenchmarkResult(name, status='completed', pi_equivalent=1.) for name in BENCHMARKS)) +probe = ctypes.CDLL(sys.argv[2]) +probe.begin.argtypes, probe.begin.restype = [ctypes.c_int], ctypes.c_int +for name in ('flags', 'end'): + getattr(probe, name).argtypes = [] + getattr(probe, name).restype = ctypes.c_int +for bits in (0x7ff0000000000001, 0xfff0000000000001, 0x7ff8000000000001, 0xfff8000000000001, 0x7ff0000000000000, 0xfff0000000000000): + value = struct.unpack('=d', struct.pack('=Q', bits))[0] + for seed in (0, 1): + assert probe.begin(seed) == 0 + before = probe.flags() + try: + wire = native.double(value) + after = probe.flags() + finally: assert probe.end() == 0 + assert struct.unpack('=Q', struct.pack('=d', wire.value))[0] == bits + assert after == before, (hex(bits), seed, 'raw copy', before, after) + for position in (0, 1): + ratios = [1., 1.]; ratios[position] = value + results = tuple(BenchmarkResult(name, status='completed', pi_equivalent=ratio) for name, ratio in zip(BENCHMARKS, ratios)) + assert probe.begin(seed) == 0 + before = probe.flags() + try: + result = SuiteResult.from_results(results) + after = probe.flags() + finally: assert probe.end() == 0 + assert result.aggregated_scores.pi_equivalent is None + assert after == before, (hex(bits), seed, position, before, after) + for window in ('warmup_seconds', 'measurement_seconds'): + options = dict(warmup_seconds=0., measurement_seconds=1.) + options[window] = value + for suite in (False, True): + assert probe.begin(seed) == 0 + before = probe.flags() + try: + result = run_all(**options).results[0] if suite else run('single_core', **options) + after = probe.flags() + finally: assert probe.end() == 0 + assert result.status == 'failed' + assert result.reason == window + (' must be finite and nonnegative' if window == 'warmup_seconds' else ' must be finite and positive') + assert struct.unpack('=Q', struct.pack('=d', getattr(result, window)))[0] == bits + assert after == before, (hex(bits), seed, window, suite, before, after) +print('quiet interpreter validation compatible') +''' + environment = dict(os.environ, LC_ALL='C', LANG='C', PYTHONPATH=str(root)) + completed = subprocess.run([executable, '-c', code, str(root), quiet_environment._name], env=environment, + capture_output=True, text=True, check=False, timeout=60) + assert completed.returncode == 0, completed.stdout + completed.stderr + assert 'quiet interpreter validation compatible' in completed.stdout diff --git a/tests/test_loader_failures.py b/tests/test_loader_failures.py new file mode 100644 index 0000000..e0bfc79 --- /dev/null +++ b/tests/test_loader_failures.py @@ -0,0 +1,391 @@ +"""Loader failure and real-symbol ownership must not create new lifetime regressions.""" + +import _ctypes +import ctypes +import gc +import os +import sys +import weakref +from _ctypes import CFuncPtr +from pathlib import Path +from types import SimpleNamespace +from typing import Any, List, Tuple, Type, cast + +import pytest + +from benchset import _native, run +from benchset._native import ABI_VERSION, Native +from tests.test_resources import ( + CallbackProbe, + without_cyclic_gc, # noqa: F401 — Shared pytest fixture. +) + + +def _clear_standard_loader() -> None: + loader = getattr(_native, '_load_library', None) + if loader is not None: + loader.cache_clear() + + +def _function_type_count() -> int: + return sum(isinstance(value, type) and issubclass(value, CFuncPtr) and value.__name__ == '_FuncPtr' + for value in gc.get_objects()) + + +def _library_path() -> str: + filename = 'benchset.dll' if os.name == 'nt' else ( + 'libbenchset.dylib' if sys.platform == 'darwin' else 'libbenchset.so' + ) + return str(Path(_native.__file__).parent / '_native' / filename) + + +@pytest.mark.usefixtures('without_cyclic_gc') +def test_failed_native_loads_do_not_accumulate_function_types(monkeypatch: pytest.MonkeyPatch) -> None: + """Repeated invalid API calls must not retain one dynamic ctypes class per missing Rust image. + + Keep the standard CDLL factory and fail its low-level open. HEAD rejects + the argument without loading; the new Rust validation path still has to + return failed results without growing temporary constructor types. + """ + _clear_standard_loader() + + def missing(_path: object, _mode: int) -> int: + raise OSError('missing Rust image') + + monkeypatch.setattr(ctypes, '_dlopen', missing, raising=False) + gc.collect() + baseline = _function_type_count() + for _batch in range(3): + for _attempt in range(20): + result = run('unknown', warmup_seconds=0, measurement_seconds=.1) + assert result.status == 'failed' + assert _function_type_count() - baseline <= 1, 'failed loading retains a cyclic function class per call' + + +def test_standard_loader_retries_failures_and_recovers(monkeypatch: pytest.MonkeyPatch) -> None: + """Failures are retried as fresh attempts so a subsequently available Rust image can load.""" + _clear_standard_loader() + original_open = cast(Any, getattr(ctypes, '_dlopen', getattr(_ctypes, 'LoadLibrary', None))) + attempts = 0 + + def open_after_two_failures(path: Any, mode: int) -> int: + nonlocal attempts + attempts += 1 + if attempts <= 2: + raise OSError('transient load failure ' + str(attempts)) + return cast(int, original_open(path, mode)) + + monkeypatch.setattr(ctypes, '_dlopen', open_after_two_failures, raising=False) + for attempt in (1, 2): + with pytest.raises(OSError, match='transient load failure ' + str(attempt)): + Native() + assert Native().lib.benchset_abi_version() == ABI_VERSION + assert attempts == 3 + + +def test_standard_loader_preflights_and_adopts_the_open_handle(monkeypatch: pytest.MonkeyPatch) -> None: + """A failed open must precede wrapper allocation, with the successful handle adopted once. + + Check the host's original ctypes search mode: POSIX DEFAULT_MODE or the + secure Windows absolute-path defaults. This is a new private bridge check; + HEAD had no cached loader helper. + """ + loader = getattr(_native, '_load_library', None) + if loader is None: + pytest.skip('HEAD has no standard loader helper') + loader.cache_clear() + original_open = cast(Any, getattr(ctypes, '_dlopen', getattr(_ctypes, 'LoadLibrary', None))) + original_initialize = ctypes.CDLL.__init__ + calls: List[Tuple[str, object, object]] = [] + + def observe_open(path: Any, mode: int) -> int: + handle = cast(int, original_open(path, mode)) + calls.append(('open', path, mode)) + return handle + + def initialize(library: ctypes.CDLL, name: Any, *args: Any, **kwargs: Any) -> None: + calls.append(('wrapper', name, kwargs.get('handle'))) + original_initialize(library, name, *args, **kwargs) + + monkeypatch.setattr(ctypes, '_dlopen', observe_open, raising=False) + monkeypatch.setattr(ctypes.CDLL, '__init__', initialize) + path = _library_path() + library = loader(path) + expected_mode = 0x1100 if os.name == 'nt' else ctypes.DEFAULT_MODE + assert calls == [('open', path, expected_mode), ('wrapper', path, library._handle)] + + +def test_cold_native_opens_the_real_image_once_and_reuses_it(monkeypatch: pytest.MonkeyPatch) -> None: + """A successful cold binding owns one open, including with newer ctypes. + + Python 3.14's POSIX CDLL hook can reopen a path despite receiving a handle. + Observe actual image opens, then check a warm binding still shares the + image while configuring its own function objects. No fake handle is called. + """ + _clear_standard_loader() + original_open = cast(Any, getattr(ctypes, '_dlopen', getattr(_ctypes, 'LoadLibrary', None))) + calls: List[Tuple[str, int]] = [] + + def observe_open(path: str, mode: int) -> int: + calls.append((path, mode)) + return cast(int, original_open(path, mode)) + + monkeypatch.setattr(ctypes, '_dlopen', observe_open, raising=False) + first, second = Native(), Native() + assert first.lib.benchset_abi_version() == second.lib.benchset_abi_version() == ABI_VERSION + assert first.lib._library is second.lib._library + assert first.lib.benchset_abi_version is not second.lib.benchset_abi_version + expected_mode = 0x1100 if os.name == 'nt' else ctypes.DEFAULT_MODE + assert calls == [(_library_path(), expected_mode)], 'a supplied native handle must not reopen its path' + + +@pytest.mark.parametrize('platform', ['darwin', 'linux', 'win32']) +def test_modern_cdll_hook_adopts_the_preopened_handle_without_reopening( + monkeypatch: pytest.MonkeyPatch, platform: str, +) -> None: + """Exercise modern ctypes handle adoption on every supported Python version. + + A simulated canonical constructor dispatches through the modern hook. Its + base hook deliberately ignores the supplied handle, as Python 3.14 POSIX + does. The bridge must adopt the first open, preserve its normalized name, + and keep platform search flags without relying on Python version checks. + Fake handles stay inside this constructor; no native symbol is invoked. + """ + canonical = cast(Any, ctypes.cdll)._dlltype + calls: List[Tuple[Any, ...]] = [] + + def open_image(path: str, mode: int) -> int: + handle = 701 + len(calls) + calls.append(('open', path, mode, handle)) + return handle + + def reopen(library: Any, name: str, mode: int, handle: int, winmode: object) -> int: + library._name = name + calls.append(('reopen', name, handle, winmode)) + return open_image(name, mode) + + def initialize(library: Any, name: str, *_args: Any, **kwargs: Any) -> None: + handle = kwargs.get('handle') + calls.append(('wrapper', name, handle)) + library._handle = library._load_library(name, ctypes.DEFAULT_MODE, handle, None) + + monkeypatch.setattr(_native, 'sys', SimpleNamespace(platform=platform)) + monkeypatch.setattr(ctypes, '_dlopen', open_image, raising=False) + monkeypatch.setattr(canonical, '_load_library', reopen, raising=False) + monkeypatch.setattr(canonical, '__init__', initialize) + requested = 'subdir/../adoption-probe.dll' + expected_path = os.path.abspath(requested) if platform == 'win32' else requested # noqa: PTH100 — Match CDLL. + expected_mode = 0x1100 if platform == 'win32' else ctypes.DEFAULT_MODE + library = _native._load_library.__wrapped__(requested) + assert calls == [('open', expected_path, expected_mode, 701), ('wrapper', expected_path, 701)] + assert library._handle == 701 + assert library._name == expected_path + + +def test_address_bound_real_symbol_keeps_library_owner_alive() -> None: + """A retained real symbol must retain its library's resources after the function table is gone.""" + functions = getattr(_native, '_Functions', None) + if functions is None: + pytest.skip('HEAD has no address-bound function table') + + class Library(ctypes.CDLL): + payload: CallbackProbe + + library = Library(_library_path()) + library.payload = CallbackProbe(None) + observed = weakref.ref(library) + payload = weakref.ref(library.payload) + table = functions(library) + function = table.benchset_abi_version + del table, library + gc.collect() + assert observed() is not None, 'the copied symbol lost its real library owner' + assert payload() is not None + # Only call after proving that its owner is still live. + function.argtypes = [] + function.restype = ctypes.c_uint32 + assert function() == ABI_VERSION + del function + gc.collect() + assert observed() is None + assert payload() is None + + +@pytest.mark.parametrize('platform', ['darwin', 'linux', 'win32']) +@pytest.mark.parametrize('alias_present', [True, False]) +def test_standard_loader_keeps_platform_modes_and_normalizes_windows_path( + monkeypatch: pytest.MonkeyPatch, platform: str, alias_present: bool, +) -> None: + """Exercise platform loading rules on every host without loading a foreign binary. + + Spies provide a handle and its wrapper; simulated Windows must normalize + the relative path exactly as CDLL does and keep its secure search flags. + Both POSIX variants keep the path and DEFAULT_MODE unchanged. Removing + ctypes' legacy alias reproduces modern Windows: the raw platform opener + must still work without changing ownership, paths or search flags. + """ + loader = getattr(_native, '_load_library', None) + if loader is None: + pytest.skip('HEAD has no standard loader helper') + calls: List[Tuple[str, str, object]] = [] + handle = 12345 + result = object() + + def open_image(path: str, mode: int) -> int: + calls.append(('open', path, mode)) + return handle + + def construct(path: str, *, handle: int) -> object: + calls.append(('wrapper', path, handle)) + return result + + monkeypatch.setattr(_native, 'sys', SimpleNamespace(platform=platform)) + if alias_present: + monkeypatch.setattr(ctypes, '_dlopen', open_image, raising=False) + else: + monkeypatch.delattr(ctypes, '_dlopen', raising=False) + monkeypatch.setattr(_ctypes, 'LoadLibrary' if platform == 'win32' else 'dlopen', + open_image, raising=False) + monkeypatch.setattr(ctypes.cdll, '_dlltype', construct) + requested = 'subdir/../benchset.dll' + expected_path = os.path.abspath(requested) if platform == 'win32' else requested # noqa: PTH100 — Match CDLL without resolving symlinks. + expected_mode = 0x1100 if platform == 'win32' else ctypes.DEFAULT_MODE + assert loader.__wrapped__(requested) is result + assert calls == [('open', expected_path, expected_mode), ('wrapper', expected_path, handle)] + + +@pytest.mark.parametrize('platform', ['darwin', 'linux', 'win32']) +@pytest.mark.parametrize('error_type', [MemoryError, KeyboardInterrupt, SystemExit]) +def test_failed_wrapper_adoption_closes_only_its_open_handle_and_retries( # noqa: PLR0915 — Observe open/adopt/rollback/recovery in one lifetime. + monkeypatch: pytest.MonkeyPatch, platform: str, error_type: Type[BaseException], +) -> None: + """A successful raw open must be rolled back if ctypes cannot allocate its wrapper. + + HEAD constructs its dynamic function class before opening the image, so it + cannot reach this new private-helper boundary. Simulated platforms use + fake handles and intercepted OS closers; no foreign or fake handle reaches + a real close function. Two failed adoptions must each close exactly their + own handle, preserve the original error and remain uncached. Recovery then + adopts and caches its handle without closing the successful image. + """ + loader = getattr(_native, '_load_library', None) + if loader is None: + pytest.skip('HEAD has no pre-opened wrapper adoption helper') + loader.cache_clear() + calls: List[Tuple[Any, ...]] = [] + context, cause = LookupError('existing context'), ValueError('existing cause') + errors = [error_type('first wrapper allocation failed'), error_type('second wrapper allocation failed')] + for error in errors: + error.__context__, error.__cause__ = context, cause + result = object() + opened: List[int] = [] + + def open_image(path: str, mode: int) -> int: + handle = 101 + len(opened) + opened.append(handle) + calls.append(('open', path, mode, handle)) + return handle + + def construct(path: str, *, handle: int) -> object: + calls.append(('wrapper', path, handle)) + if len(opened) <= len(errors): + raise errors[len(opened) - 1] + return result + + def close_posix(handle: int) -> None: + calls.append(('dlclose', handle)) + + def close_windows(handle: int) -> None: + calls.append(('FreeLibrary', handle)) + + monkeypatch.setattr(_native, 'sys', SimpleNamespace(platform=platform)) + monkeypatch.setattr(ctypes, '_dlopen', open_image, raising=False) + monkeypatch.setattr(ctypes.cdll, '_dlltype', construct) + monkeypatch.setattr(_ctypes, 'dlclose', close_posix, raising=False) + monkeypatch.setattr(_ctypes, 'FreeLibrary', close_windows, raising=False) + requested = 'subdir/../adoption-probe.dll' + expected_path = os.path.abspath(requested) if platform == 'win32' else requested # noqa: PTH100 + expected_mode = 0x1100 if platform == 'win32' else ctypes.DEFAULT_MODE + closer = 'FreeLibrary' if platform == 'win32' else 'dlclose' + expected: List[Tuple[Any, ...]] = [] + try: + for index, original in enumerate(errors): + with pytest.raises(error_type) as caught: + loader(requested) + assert caught.value is original, 'rollback must preserve the constructor exception' + assert caught.value.__context__ is context + assert caught.value.__cause__ is cause + assert caught.value.__suppress_context__ is True + handle = 101 + index + expected.extend([('open', expected_path, expected_mode, handle), + ('wrapper', expected_path, handle), (closer, handle)]) + assert calls == expected, 'each failed adoption must close its raw image exactly once' + assert loader.cache_info().currsize == 0, 'a failed adoption must not enter the loader cache' + assert loader(requested) is result + expected.extend([('open', expected_path, expected_mode, 103), ('wrapper', expected_path, 103)]) + assert calls == expected, 'successful adoption must retain its opened image' + assert loader(requested) is result + assert calls == expected, 'a recovered standard image must be cached' + assert loader.cache_info().currsize == 1 + finally: + loader.cache_clear() + + +@pytest.mark.parametrize('platform', ['darwin', 'linux', 'win32']) +def test_failed_raw_open_never_constructs_or_closes_a_wrapper( + monkeypatch: pytest.MonkeyPatch, platform: str, +) -> None: + """An unsuccessful open owns no handle to close and must remain retryable. + + This complements failed-adoption coverage: rollback must begin only after + _dlopen returned a real handle, while a recovered successful image remains + open. Both close entry points are intercepted on every host. + """ + loader = getattr(_native, '_load_library', None) + if loader is None: + pytest.skip('HEAD has no pre-opened wrapper adoption helper') + loader.cache_clear() + calls: List[Tuple[Any, ...]] = [] + original = OSError('raw open failed before owning a handle') + attempts = 0 + result = object() + + def open_image(path: str, mode: int) -> int: + nonlocal attempts + attempts += 1 + calls.append(('open', path, mode)) + if attempts == 1: + raise original + return 701 + + def construct(path: str, *, handle: int) -> object: + calls.append(('wrapper', path, handle)) + return result + + def close_posix(handle: int) -> None: + calls.append(('dlclose', handle)) + + def close_windows(handle: int) -> None: + calls.append(('FreeLibrary', handle)) + + monkeypatch.setattr(_native, 'sys', SimpleNamespace(platform=platform)) + monkeypatch.setattr(ctypes, '_dlopen', open_image, raising=False) + monkeypatch.setattr(ctypes.cdll, '_dlltype', construct) + monkeypatch.setattr(_ctypes, 'dlclose', close_posix, raising=False) + monkeypatch.setattr(_ctypes, 'FreeLibrary', close_windows, raising=False) + requested = 'subdir/../adoption-probe.dll' + expected_path = os.path.abspath(requested) if platform == 'win32' else requested # noqa: PTH100 + expected_mode = 0x1100 if platform == 'win32' else ctypes.DEFAULT_MODE + try: + with pytest.raises(OSError, match='raw open failed before owning a handle') as caught: + loader(requested) + assert caught.value is original + assert calls == [('open', expected_path, expected_mode)], 'a failed open must never be closed' + assert loader.cache_info().currsize == 0 + assert loader(requested) is result + assert calls == [('open', expected_path, expected_mode), ('open', expected_path, expected_mode), + ('wrapper', expected_path, 701)] + assert loader(requested) is result + assert len(calls) == 3 + finally: + loader.cache_clear() diff --git a/tests/test_loader_imports.py b/tests/test_loader_imports.py new file mode 100644 index 0000000..6f0709d --- /dev/null +++ b/tests/test_loader_imports.py @@ -0,0 +1,123 @@ +"""Import guards must not become permanent owners of native-loader state.""" + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +_BOOTSTRAP = ''' +import ctypes +import gc +import pathlib +import sys +import weakref +assert 'benchset._native' not in sys.modules +standard_loader = ctypes.CDLL +calls = [] +class Payload: + def __init__(self): + self.data = bytearray(256 * 1024) +class Loader: + def __init__(self, payload): + self.payload = payload + def __call__(self, *args): + calls.append(args) + raise AssertionError('import must not load Rust') +def guarded_function(payload): + def load(*args): + calls.append(args) + assert payload is not None + raise AssertionError('import must not load Rust') + return load +payload = Payload() +loader = guarded_function(payload) if sys.argv[1] == 'function' else Loader(payload) +payload_reference = weakref.ref(payload) +loader_reference = weakref.ref(loader) +ctypes.CDLL = loader +import benchset +assert pathlib.Path(benchset.__file__).resolve().parent == pathlib.Path.cwd() / 'benchset' +ctypes.CDLL = standard_loader +assert calls == [] +del loader, payload +''' + + +def _run(program: str, *arguments: str) -> None: + environment = dict(os.environ) + # This scenario requires an untouched import state, including under coverage. + for key in ('COVERAGE_PROCESS_START', 'COVERAGE_FILE'): + environment.pop(key, None) + result = subprocess.run([sys.executable, '-c', _BOOTSTRAP + program, *arguments], cwd=ROOT, + env=environment, capture_output=True, text=True, check=False) + assert result.returncode == 0, result.stdout + result.stderr + assert not result.stderr + + +@pytest.mark.parametrize('factory_kind', ['callable', 'function']) +def test_native_free_import_releases_a_temporary_loader(factory_kind: str) -> None: + """A pre-import guard and its data are released after restoration, even after an explicit collection. + + Fresh subprocesses prevent prior imports from hiding factory capture. Cover + both a callable's copied attributes and a function's captured payload. + """ + _run(''' +gc.collect() +assert loader_reference() is None, 'native import retained the temporary factory' +assert payload_reference() is None, 'native import retained the factory payload' +''', factory_kind) + + +@pytest.mark.parametrize('operation', ['aggregate', 'cli', 'invalid_run']) +def test_restored_standard_loader_keeps_native_memory_bounded(operation: str) -> None: + """After a guarded import, normal operations must still reuse the bounded standard library. + + Observe weak function-type references and actual DLL objects in three + GC-disabled batches. This isolates loader ownership from argparse cycles + and allocator caching while permitting one process-lived native image. + """ + _run(''' +import contextlib +import io +from benchset import BenchmarkResult, SuiteResult, _native, api, cli +base_native = _native.Native +function_types = [] +class ObservedNative(base_native): + def __init__(self): + super().__init__() + function_types.append(weakref.ref(type(self.lib.benchset_create))) +for module in (_native, api, cli): + module.Native = ObservedNative +values = (BenchmarkResult('single_core', status='completed', pi_equivalent=1), + BenchmarkResult('multi_core', status='completed', pi_equivalent=4)) +def invoke(): + if sys.argv[2] == 'aggregate': + assert SuiteResult.from_results(values).aggregated_scores.pi_equivalent == 2 + elif sys.argv[2] == 'cli': + with contextlib.redirect_stderr(io.StringIO()): + try: + cli.settings(['--warmup=-1', '--duration=.1']) + except SystemExit as error: + assert error.code == 2 + else: + raise AssertionError('invalid duration must be rejected') + else: + result = api.run('unknown', warmup_seconds=0, measurement_seconds=.1) + assert result.status == 'failed' + assert result.reason == 'unknown benchmark: unknown' +ObservedNative() +invoke() +gc.collect() +function_types.clear() +baseline = sum(isinstance(value, standard_loader) for value in gc.get_objects()) +gc.disable() +for batch in range(3): + for attempt in range(20): + invoke() + surviving_types = {reference() for reference in function_types if reference() is not None} + assert len(surviving_types) <= 1, 'restored loader creates a cyclic function type per call' + live = sum(isinstance(value, standard_loader) for value in gc.get_objects()) + assert live <= baseline + 1, 'restored loader creates a DLL wrapper per call' +''', 'callable', operation) diff --git a/tests/test_metadata.py b/tests/test_metadata.py new file mode 100644 index 0000000..c0eb0bc --- /dev/null +++ b/tests/test_metadata.py @@ -0,0 +1,537 @@ +"""Native-free public metadata and Rust-generated package snapshots.""" + +import errno +import json +import math +import os +import shutil +import signal +import subprocess +import sys +import zipfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import List, Optional, Type + +import pytest + +from scripts.cargo import build, publish + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize('argument', [None, '--help', '--version']) +def test_zip_import_keeps_metadata_and_cli_inspection_native_free(tmp_path: Path, argument: Optional[str]) -> None: + """Imports and metadata inspection work from ZIP packages without loading Rust. + + Archive Python files and the generated data, omitting native binaries. + A fresh interpreter must use that ZIP and preserve public DTO serialization, + defaults and CLI help/version rather than treating __file__ as a disk path. + """ + archive = tmp_path / 'benchset-package.zip' + with zipfile.ZipFile(archive, 'w') as package: + for source in (ROOT / 'benchset').glob('*.py'): + package.write(source, 'benchset/' + source.name) + package.write(ROOT / 'benchset/_metadata.json', 'benchset/_metadata.json') + program = ''' +import ctypes +import json +import sys +from dataclasses import asdict +def forbidden(*args, **kwargs): + raise AssertionError('ZIP metadata must not load the native library') +ctypes.CDLL = forbidden +sys.path.insert(0, sys.argv[1]) +import benchset +assert benchset.__file__.startswith(sys.argv[1]) +assert benchset.BENCHMARKS == ('single_core', 'multi_core') +assert type(benchset.DEFAULT_WARMUP_SECONDS) is float +assert type(benchset.DEFAULT_MEASUREMENT_SECONDS) is float +result = benchset.BenchmarkResult('single_core') +assert result.reference_machine is benchset.PI_REFERENCE_MACHINE +assert result.reference_run is benchset.PI_REFERENCE_RUN +assert json.loads(json.dumps(asdict(result)))['reference_machine']['short_name'] == 'pi' +if len(sys.argv) > 2: + from benchset.cli import main + assert main([sys.argv[2]]) == 0 +''' + command = [sys.executable, '-c', program, str(archive)] + if argument is not None: + command.append(argument) + environment = dict(os.environ) + for key in ('COVERAGE_PROCESS_START', 'COVERAGE_FILE'): + environment.pop(key, None) + result = subprocess.run(command, cwd=tmp_path, env=environment, capture_output=True, text=True, check=False) + assert result.returncode == 0, result.stdout + result.stderr + assert not result.stderr + if argument == '--help': + assert 'usage: benchset' in result.stdout + elif argument == '--version': + assert result.stdout.strip() + else: + assert not result.stdout + + +@pytest.mark.skipif(os.name == 'nt', reason='Windows filenames cannot contain line breaks') +@pytest.mark.skipif(shutil.which('cargo') is None, reason='Cargo checks metadata path transport') +@pytest.mark.parametrize('separator', ['\n', '\r', '\r\n']) +def test_cargo_metadata_preserves_target_path_bytes(tmp_path: Path, separator: str) -> None: + """Cargo's line protocol must preserve control characters in output paths. + + Real Cargo output catches truncation before Python receives the JSON event; + a mocked event would conceal corruption inside the build-script protocol. + """ + cargo = shutil.which('cargo') + assert cargo is not None + target = tmp_path / ('target' + separator + 'π') + target.mkdir() + environment = dict(os.environ, CARGO_TARGET_DIR=str(target)) + metadata = json.loads(build(ROOT, [cargo, 'check', '--locked'], environment)) + assert metadata == json.loads((ROOT / 'benchset/_metadata.json').read_text(encoding='utf-8')) + + +@pytest.mark.skipif(os.name == 'nt', reason='POSIX output path byte semantics') +@pytest.mark.skipif(shutil.which('rustc') is None, reason='Rust compiler checks the real metadata exporter') +@pytest.mark.parametrize('separator', ['\n', '\r', '\r\n', '\udcff']) +def test_metadata_exporter_preserves_output_path_bytes(tmp_path: Path, separator: str) -> None: + """The Rust exporter transports exact OS bytes without Cargo-line truncation. + + Compile at a UTF-8 path before providing OUT_DIR at runtime: rustc parses + its command-line arguments as UTF-8 and panics on non-UTF8 target paths. + The exporter itself must preserve those bytes wherever the filesystem + supports them. Real Cargo integration above separately checks its events. + """ + compiler = shutil.which('rustc') + assert compiler is not None + executable = tmp_path / 'exporter' + subprocess.run([compiler, '--edition=2021', str(ROOT / 'native/build.rs'), '-o', str(executable)], check=True) + destination = tmp_path / ('output' + separator + 'π') + try: + destination.mkdir() + except OSError as error: + if separator == '\udcff' and error.errno == errno.EILSEQ: + pytest.skip('This filesystem rejects non-UTF8 filenames') + raise + result = subprocess.run([str(executable)], env=dict(os.environ, OUT_DIR=str(destination)), + capture_output=True, text=True, check=True) + prefix = 'cargo:rustc-env=BENCHSET_METADATA_PATH_HEX=' + paths = [os.fsdecode(bytes.fromhex(line[len(prefix):])) for line in result.stdout.splitlines() if line.startswith(prefix)] + assert paths == [str(destination / 'benchset-metadata.json')] + assert json.loads(Path(paths[0]).read_bytes()) == json.loads((ROOT / 'benchset/_metadata.json').read_bytes()) + + +@pytest.mark.parametrize('handler', ['thread', 'custom', 'ignored']) +@pytest.mark.parametrize('failure', [False, True]) +def test_publication_preserves_thread_and_signal_handler_support( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, handler: str, failure: bool, +) -> None: + """Publication works off the main thread and preserves caller SIGINT policy on every exit.""" + package = tmp_path / 'package' + compiled = tmp_path / 'libbenchset.so' + compiled.write_bytes(b'new library') + replace = Path.replace + + def failed_replace(source: Path, destination: Path) -> Path: + assert signal.getsignal(signal.SIGINT) is expected + if failure and source.name == '_metadata.json': + raise OSError('metadata rename failed') + return replace(source, destination) + + monkeypatch.setattr(Path, 'replace', failed_replace) + expected = ((lambda _signum, _frame: None) if handler == 'custom' else + signal.SIG_IGN if handler == 'ignored' else signal.default_int_handler) + previous = signal.signal(signal.SIGINT, expected) + try: + with ThreadPoolExecutor(max_workers=1) as executor: + def install() -> Path: + if handler == 'thread': + return executor.submit(publish, package, compiled, b'new metadata').result(timeout=5) + return publish(package, compiled, b'new metadata') + + if failure: + with pytest.raises(OSError, match='metadata rename failed'): + install() + assert not (package / '_native' / compiled.name).exists() + else: + assert install().read_bytes() == b'new library' + assert (package / '_metadata.json').read_bytes() == b'new metadata' + assert signal.getsignal(signal.SIGINT) is expected + finally: + signal.signal(signal.SIGINT, previous) + + +@pytest.mark.parametrize('existing', [False, True]) +@pytest.mark.parametrize('error_type', [OSError, KeyboardInterrupt]) +def test_sigint_during_publication_rollback_keeps_complete_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, existing: bool, error_type: Type[BaseException], +) -> None: + """Finish rollback before reporting Ctrl+C, including a repeated interruption. + + Metadata replacement fails after the library has changed; SIGINT then arrives + just before restoring the previous library or removing a first installation. + The previous signal handler must also survive the failed publication. + """ + package = tmp_path / 'package' + installed = package / '_native/libbenchset.so' + installed.parent.mkdir(parents=True) + snapshot = package / '_metadata.json' + if existing: + installed.write_bytes(b'old library') + snapshot.write_bytes(b'old metadata') + compiled = tmp_path / installed.name + compiled.write_bytes(b'new library') + replace, unlink = Path.replace, Path.unlink + fired: List[bool] = [] + + def interrupt() -> None: + fired.append(True) + signal.raise_signal(signal.SIGINT) + + def interrupted_replace(source: Path, destination: Path) -> Path: + if source.name == snapshot.name: + raise error_type('metadata rename failed') + if source.name == 'previous-library': + interrupt() + return replace(source, destination) + + def interrupted_unlink(path: Path, **options: bool) -> None: + if path == installed: + interrupt() + unlink(path, **options) + + monkeypatch.setattr(Path, 'replace', interrupted_replace) + monkeypatch.setattr(Path, 'unlink', interrupted_unlink) + previous = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + with pytest.raises(KeyboardInterrupt): + publish(package, compiled, b'new metadata') + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + finally: + signal.signal(signal.SIGINT, previous) + assert fired == [True] + if existing: + assert installed.read_bytes() == b'old library' + assert snapshot.read_bytes() == b'old metadata' + assert list(installed.parent.iterdir()) == [installed] + else: + assert not snapshot.exists() + assert list(installed.parent.iterdir()) == [] + + +@pytest.mark.parametrize('existing', [False, True]) +@pytest.mark.parametrize('boundary', ['library', 'metadata']) +def test_publication_signal_after_rename_keeps_one_complete_generation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, existing: bool, boundary: str, +) -> None: + """SIGINT after either completed rename must not mix library and metadata generations. + + The first rename needs rollback; after the metadata rename both new files + are already committed. Inject a real signal after the filesystem mutation, + including first installations without a previous library to restore. + """ + package = tmp_path / 'package' + installed = package / '_native/libbenchset.so' + installed.parent.mkdir(parents=True) + snapshot = package / '_metadata.json' + if existing: + installed.write_bytes(b'old library') + snapshot.write_bytes(b'old metadata') + compiled = tmp_path / installed.name + compiled.write_bytes(b'new library') + replace = Path.replace + fired: List[bool] = [] + + def interrupted_replace(source: Path, destination: Path) -> Path: + result = replace(source, destination) + name = compiled.name if boundary == 'library' else snapshot.name + if source.name == name and not fired: + fired.append(True) + signal.raise_signal(signal.SIGINT) + return result + + monkeypatch.setattr(Path, 'replace', interrupted_replace) + previous = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + with pytest.raises(KeyboardInterrupt): + publish(package, compiled, b'new metadata') + finally: + signal.signal(signal.SIGINT, previous) + assert fired == [True] + if boundary == 'metadata' or existing: + generation = b'new' if boundary == 'metadata' else b'old' + assert installed.read_bytes() == generation + b' library' + assert snapshot.read_bytes() == generation + b' metadata' + assert list(installed.parent.iterdir()) == [installed] + else: + assert not snapshot.exists() + assert list(installed.parent.iterdir()) == [] + + +def test_python_metadata_and_signature_defaults_are_native_free() -> None: + """Keep ordered names, float defaults and calibration readable without Rust. + + A fresh interpreter forbids CDLL before importing benchset, so a cached + library cannot conceal eager loading while replacing handwritten constants. + Signature inspection also catches defaults becoming None or a sentinel. + """ + program = ''' +import ctypes +import inspect +import json +from dataclasses import asdict +def forbidden(*args, **kwargs): + raise AssertionError('metadata must not load the native library') +ctypes.CDLL = forbidden +import benchset +assert benchset.BENCHMARKS == ('single_core', 'multi_core') +assert isinstance(benchset.BENCHMARKS, tuple) +assert type(benchset.DEFAULT_WARMUP_SECONDS) is float +assert type(benchset.DEFAULT_MEASUREMENT_SECONDS) is float +for function in (benchset.run, benchset.run_all, benchset.BenchmarkRunner.run): + parameters = inspect.signature(function).parameters + assert parameters['warmup_seconds'].default == 120.0 + assert parameters['measurement_seconds'].default == 60.0 +result = benchset.BenchmarkResult('single_core') +assert result.warmup_seconds == 120.0 and result.measurement_seconds == 60.0 +assert result.reference_machine is benchset.PI_REFERENCE_MACHINE +assert result.reference_run is benchset.PI_REFERENCE_RUN +payload = json.loads(json.dumps(asdict(result))) +assert payload['reference_machine']['memory_bytes'] == 8589934592 +assert type(payload['reference_machine']['memory_bytes']) is int +assert payload['reference_run']['results'] == [ + {'benchmark': 'single_core', 'score': 47231.227029007096}, + {'benchmark': 'multi_core', 'score': 181953.17419139884}, +] +assert type(benchset.PI_REFERENCE_RUN.results) is tuple +print('native-free metadata') +''' + result = subprocess.run([sys.executable, '-c', program], capture_output=True, text=True, check=False) + assert result.returncode == 0, result.stderr + assert result.stdout == 'native-free metadata\n' + assert not result.stderr + + +@pytest.mark.parametrize('failure', [None, 'compile', 'missing', 'duplicate', 'read', 'copy', 'incoming-copy', 'write', 'library-replace', 'replace']) +def test_build_publishes_exact_reported_metadata_atomically( # noqa: PLR0915 — one publication contract across failure boundaries. + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str, +) -> None: + """Publish the current Cargo-reported snapshot without guessing an output path. + + A stale file in the conventional target directory and an unrelated build + event catch path guessing. Compilation/report/read/copy/write failures + retain both installed files, including failures at either replacement. + Staging both products before replacing either prevents interrupted copies + from mixing new metadata with an old library. Failures clean staging files. + """ + package = tmp_path / 'package' + package.mkdir() + installed = package / '_metadata.json' + installed.write_bytes(b'old complete snapshot') + library = package / '_native/libbenchset.so' + library.parent.mkdir() + library.write_bytes(b'old complete library') + stale = tmp_path / 'target/release/benchset-metadata.json' + stale.parent.mkdir(parents=True) + stale.write_bytes(b'stale metadata') + fresh = tmp_path / 'custom-output-π-данные/build-script-out/benchset-metadata.json' + fresh.parent.mkdir(parents=True) + fresh.write_bytes(b'fresh complete snapshot') + compiled = fresh.parent / library.name + compiled.write_bytes(b'fresh complete library') + records = [ + {'reason': 'compiler-artifact'}, + {'reason': 'build-script-executed', 'env': [['UNRELATED_PATH', str(stale)]]}, + {'reason': 'build-script-executed', 'env': [['BENCHSET_METADATA_PATH_HEX', os.fsencode(fresh).hex()]]}, + ] + environment = {'CARGO_TARGET_DIR': str(tmp_path / 'custom-output-π-данные')} + + def compile_(arguments: list, **options: object) -> subprocess.CompletedProcess: + assert arguments == ['selected-cargo', 'build', '--message-format=json-render-diagnostics'] + assert options == dict(cwd=tmp_path, env=environment, stdout=subprocess.PIPE, encoding='utf-8', check=True) + assert installed.read_bytes() == b'old complete snapshot' + if failure == 'compile': + raise subprocess.CalledProcessError(1, arguments) + if failure == 'missing': + records.pop() + if failure == 'duplicate': + records.append(records[-1]) + if failure == 'read': + fresh.unlink() + return subprocess.CompletedProcess(arguments, 0, stdout='\n'.join(json.dumps(record, ensure_ascii=False) for record in records)) + + monkeypatch.setattr(subprocess, 'run', compile_) + replace = Path.replace + + def replace_with_reader(source: Path, destination: Path) -> Path: + assert installed.read_bytes() == b'old complete snapshot' + expected = (b'fresh complete snapshot' if destination == installed else + b'old complete library' if source.name == 'previous-library' else b'fresh complete library') + assert source.read_bytes() == expected + if failure == 'replace' and destination == installed: + raise OSError('publication interrupted') + if failure == 'library-replace' and destination == library: + raise OSError('library publication interrupted') + return replace(source, destination) + + monkeypatch.setattr(Path, 'replace', replace_with_reader) + write = Path.write_bytes + + def write_with_reader(path: Path, data: bytes) -> int: + assert library.read_bytes() == b'old complete library' + assert installed.read_bytes() == b'old complete snapshot' + if failure == 'write': + raise OSError('snapshot write interrupted') + return write(path, data) + + monkeypatch.setattr(Path, 'write_bytes', write_with_reader) + if failure in ('copy', 'incoming-copy'): + copy = shutil.copy2 + + def interrupted_copy(source: Path, destination: Path) -> Path: + if failure == 'copy' or source == compiled: + raise OSError('library copy interrupted') + return copy(source, destination) + monkeypatch.setattr(shutil, 'copy2', interrupted_copy) + + def install() -> None: + metadata = build(tmp_path, ['selected-cargo', 'build'], environment) + publish(package, compiled, metadata) + + if failure: + with pytest.raises((OSError, RuntimeError, subprocess.CalledProcessError)): + install() + assert installed.read_bytes() == b'old complete snapshot' + assert library.read_bytes() == b'old complete library' + else: + install() + assert installed.read_bytes() == b'fresh complete snapshot' + assert library.read_bytes() == b'fresh complete library' + assert sorted(path for path in package.iterdir() if path.name != '.benchset.publish.lock') == sorted([installed, library.parent]) + assert list(library.parent.iterdir()) == [library] + + +@pytest.mark.parametrize('error_type', [OSError, KeyboardInterrupt]) +def test_first_install_metadata_failure_removes_unpublished_library( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, error_type: Type[BaseException], +) -> None: + """Leave no half-installed native library when first metadata publication fails. + + An existing installation restores its previous library in the failure + matrix above; a first installation has no backup and must remove the new + library instead. Both cases leave only complete generations after failure. + """ + compiled = tmp_path / 'libbenchset.so' + compiled.write_bytes(b'fresh library') + package = tmp_path / 'package' + replace = Path.replace + + def fail_metadata(source: Path, destination: Path) -> Path: + if destination.name == '_metadata.json': + raise error_type('metadata rename interrupted') + return replace(source, destination) + + monkeypatch.setattr(Path, 'replace', fail_metadata) + with pytest.raises(error_type, match='metadata rename interrupted'): + publish(package, compiled, b'fresh metadata') + assert not (package / '_metadata.json').exists() + assert list((package / '_native').iterdir()) == [] + + +@pytest.mark.skipif(shutil.which('cargo') is None, reason='Rust snapshot generation needs Cargo') +def test_rust_generated_snapshot_tracks_registry_defaults_and_calibration(tmp_path: Path) -> None: + """Keep the checked snapshot fresh and derive every field from Rust definitions. + + A small real Cargo project compiles the actual build script. Rebuilding it + after changing only Rust names/order/defaults/calibration proves rerun paths, + exact float transport and JSON escaping. Importing a fresh Python package + with that snapshot catches handwritten fallback copies in the wrappers. + The historical calibration JSON is + independent evidence and is deliberately absent from this runtime source. + """ + project = tmp_path / 'project' + native = project / 'native' + (native / 'src').mkdir(parents=True) + (project / 'Cargo.toml').write_text( + '[package]\nname="metadata-probe"\nversion="0.0.0"\nedition="2021"\n' + 'build="native/build.rs"\n[lib]\npath="lib.rs"\n', + ) + (project / 'lib.rs').write_text('') + for relative in ('build.rs', 'src/catalog.rs', 'src/reference.rs'): + shutil.copy2(ROOT / 'native' / relative, native / relative) + environment = dict(os.environ, CARGO_TARGET_DIR=str(tmp_path / 'isolated-target-π-данные')) + cargo = shutil.which('cargo') + metadata = build(project, [cargo, 'build', '--offline'], environment) + snapshot = json.loads(metadata) + assert snapshot == json.loads((ROOT / 'benchset/_metadata.json').read_text(encoding='utf-8')) + catalog = native / 'src/catalog.rs' + catalog.write_text(catalog.read_text().replace( + '[Benchmark::SingleCore, Benchmark::MultiCore]', '[Benchmark::MultiCore, Benchmark::SingleCore]', + ).replace('"single_core"', '"renamed"').replace('120.0', '1.2345678901234567').replace('60.0', '9.876543210987654')) + metadata = build(project, [cargo, 'build', '--offline'], environment) + changed = json.loads(metadata) + assert changed['benchmarks'] == [{'id': 2, 'name': 'multi_core'}, {'id': 1, 'name': 'renamed'}] + assert changed['default_warmup_seconds'] == 1.2345678901234567 + assert changed['default_measurement_seconds'] == 9.876543210987654 + assert changed['reference_machine'] == snapshot['reference_machine'] + assert changed['reference_run'] == snapshot['reference_run'] + reference = native / 'src/reference.rs' + reference.write_text(reference.read_text(encoding='utf-8').replace( + '"Raspberry Pi 5 Model B Rev 1.0"', '"quote\\\" slash\\\\ line\\n π\\u{0001}"', + ).replace('47231.227029007096', '0.12345678901234567').replace('1248.0', '0.9876543210987654') + .replace('8 * 1024 * 1024 * 1024', 'u64::MAX'), encoding='utf-8') + metadata = build(project, [cargo, 'build', '--offline'], environment) + changed = json.loads(metadata) + assert changed['reference_machine']['name'] == 'quote" slash\\ line\n π\x01' + assert changed['reference_run']['results'][0]['score'] == 0.12345678901234567 + assert changed['reference_run']['warmup_seconds'] == 0.9876543210987654 + assert changed['reference_machine']['memory_bytes'] == 2**64 - 1 + source = project / 'python-source' + shutil.copytree(ROOT / 'benchset', source / 'benchset', ignore=shutil.ignore_patterns('_native', '__pycache__')) + (source / 'benchset/_metadata.json').write_bytes(metadata) + program = ''' +import ctypes +import inspect +import json +def forbidden(*args, **kwargs): + raise AssertionError('metadata inspection must remain native-free') +ctypes.CDLL = forbidden +import benchset +from benchset.cli import Settings +assert benchset.BENCHMARKS == ('multi_core', 'renamed') +assert benchset.DEFAULT_WARMUP_SECONDS == 1.2345678901234567 +assert benchset.DEFAULT_MEASUREMENT_SECONDS == 9.876543210987654 +for function in (benchset.run, benchset.run_all, benchset.BenchmarkRunner.run): + parameters = inspect.signature(function).parameters + assert parameters['warmup_seconds'].default == benchset.DEFAULT_WARMUP_SECONDS + assert parameters['measurement_seconds'].default == benchset.DEFAULT_MEASUREMENT_SECONDS +configuration = Settings(_sources=[]) +assert configuration.warmup == benchset.DEFAULT_WARMUP_SECONDS +assert configuration.duration == benchset.DEFAULT_MEASUREMENT_SECONDS +result = benchset.BenchmarkResult('renamed') +assert result.warmup_seconds == benchset.DEFAULT_WARMUP_SECONDS +assert result.measurement_seconds == benchset.DEFAULT_MEASUREMENT_SECONDS +assert result.reference_machine is benchset.PI_REFERENCE_MACHINE +assert result.reference_run is benchset.PI_REFERENCE_RUN +assert benchset.PI_REFERENCE_RUN.results[0].score == 0.12345678901234567 +assert benchset.PI_REFERENCE_RUN.warmup_seconds == 0.9876543210987654 +assert 'π' in benchset.PI_REFERENCE_MACHINE.name +assert benchset.PI_REFERENCE_MACHINE.memory_bytes == 2**64 - 1 +print('Rust metadata reaches Python') +''' + consumer_environment = dict(os.environ, PYTHONPATH=str(source)) + # This deliberately changed independent package is not checkout runtime + # coverage; inheriting its startup hook would count it as a second library. + for variable in ('COVERAGE_PROCESS_START', 'COVERAGE_FILE'): + consumer_environment.pop(variable, None) + consumer = subprocess.run([sys.executable, '-c', program], cwd=source, + env=consumer_environment, capture_output=True, text=True, check=False) + assert consumer.returncode == 0, consumer.stderr + assert consumer.stdout == 'Rust metadata reaches Python\n' + assert not consumer.stderr + original = reference.read_text(encoding='utf-8') + for literal, expected in [('1.0', 1.0), ('-0.0', -0.0), ('5e-324', 5e-324), ('1.7976931348623157e308', sys.float_info.max)]: + reference.write_text(original.replace('0.12345678901234567', literal), encoding='utf-8') + score = json.loads(build(project, [cargo, 'build', '--offline'], environment))['reference_run']['results'][0]['score'] + assert type(score) is float + assert score == expected + assert math.copysign(1, score) == math.copysign(1, expected) diff --git a/tests/test_name_diagnostics.py b/tests/test_name_diagnostics.py new file mode 100644 index 0000000..ea2d9cc --- /dev/null +++ b/tests/test_name_diagnostics.py @@ -0,0 +1,106 @@ +"""Caller string operators in unknown-benchmark diagnostics retain Python semantics.""" + +from typing import List, Type + +import pytest + +from benchset import BenchmarkRunner +from benchset._native import Native + + +def test_unknown_name_concatenation_preserves_prefix_and_result() -> None: + """The caller receives the original prefix and its returned string is stored by identity.""" + calls: List[str] = [] + + class Diagnostic(str): + __slots__ = () + + diagnostic = Diagnostic('caller-defined diagnostic') + + class Name(str): + __slots__ = () + + def __radd__(self, prefix: str) -> str: + calls.append(prefix) + return diagnostic + + name = Name('unknown') + result = BenchmarkRunner().run(name, warmup_seconds=0, measurement_seconds=0.01) + assert result.status == 'failed' + assert result.benchmark is name + assert result.reason is diagnostic + assert calls == ['unknown benchmark: '] + + +@pytest.mark.parametrize('error_type', [RuntimeError, OSError, OverflowError, ValueError, + KeyboardInterrupt, SystemExit, GeneratorExit]) +def test_unknown_name_concatenation_preserves_exception_and_runner_reuse(error_type: Type[BaseException]) -> None: + """Diagnostic operators preserve exception identity/context and leave the runner reusable.""" + calls: List[str] = [] + original = error_type('caller concatenation failed') + context = LookupError('caller context') + + class Name(str): + __slots__ = () + + def __radd__(self, prefix: str) -> str: + calls.append(prefix) + try: + raise context + except LookupError: + raise original from context + + runner = BenchmarkRunner() + with pytest.raises(error_type, match='caller concatenation failed') as caught: + runner.run(Name('unknown'), warmup_seconds=0, measurement_seconds=0.01) + assert caught.value is original + assert original.__context__ is context + assert original.__cause__ is context + assert original.__suppress_context__ + assert calls == ['unknown benchmark: '] + result = runner.run('single_core', warmup_seconds=0, measurement_seconds=0.01, + on_progress=lambda _progress: runner.cancel()) + assert result.status == 'cancelled' + + +@pytest.mark.parametrize('benchmark', ['single_core', 'multi_core']) +def test_valid_names_do_not_evaluate_diagnostic_concatenation(benchmark: str) -> None: + """A valid name never invokes an unused diagnostic operator, including on cancellation.""" + class Name(str): + __slots__ = () + + def __radd__(self, prefix: str) -> str: + raise AssertionError('valid names must not create diagnostics') + + runner = BenchmarkRunner() + result = runner.run(Name(benchmark), warmup_seconds=0, measurement_seconds=0.01, + on_progress=lambda _progress: runner.cancel()) + assert result.status == 'cancelled' + + +@pytest.mark.parametrize('error_type', [RuntimeError, OSError, OverflowError, ValueError, + KeyboardInterrupt, SystemExit, GeneratorExit]) +def test_native_diagnostic_without_carrier_preserves_original_exception(error_type: Type[BaseException]) -> None: + """The new private validation boundary directly re-raises diagnostics when no carrier is requested.""" + original = error_type('direct diagnostic failed') + context = LookupError('direct caller context') + calls: List[str] = [] + + class Name(str): + __slots__ = () + + def __radd__(self, prefix: str) -> str: + calls.append(prefix) + try: + raise context + except LookupError: + raise original from context + + native = Native() + with pytest.raises(error_type, match='direct diagnostic failed') as caught: + native.validation_error(Name('unknown'), 0, 0.01) + assert caught.value is original + assert original.__context__ is context + assert original.__cause__ is context + assert original.__suppress_context__ + assert calls == ['unknown benchmark: '] diff --git a/tests/test_name_operands.py b/tests/test_name_operands.py new file mode 100644 index 0000000..74fde67 --- /dev/null +++ b/tests/test_name_operands.py @@ -0,0 +1,181 @@ +"""Preserve canonical public string operands when Rust invokes host comparisons.""" + +import ctypes +from types import SimpleNamespace +from typing import Any, List, Type, cast + +import pytest + +from benchset import BENCHMARKS, BenchmarkResult, BenchmarkRunner, SuiteResult, _native + + +@pytest.mark.parametrize('stage', ['validation', 'execution']) +def test_public_run_name_operators_receive_the_canonical_catalog_operand(stage: str) -> None: + """An overridden str equality may recognize the exported catalog by identity. + + Validation and execution each compared against the same canonical Python + string before migration. A newly decoded equal string must not turn a + valid name into an unknown name or select the multicore fallback. + """ + operands: List[object] = [] + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + operands.append(other) + current_stage = 'validation' if len(operands) == 1 else 'execution' + return bool(str.__eq__(self, other)) and ( + current_stage != stage or other is BENCHMARKS[0] + ) + + runner = BenchmarkRunner() + result = runner.run(Name('single_core'), warmup_seconds=0, measurement_seconds=.01, + on_progress=lambda _progress: runner.cancel()) + assert result.status == 'cancelled' + assert len(operands) == 2 + assert operands[0 if stage == 'validation' else 1] is BENCHMARKS[0] + + +@pytest.mark.parametrize('count', [1, 2, 3]) +def test_aggregate_name_operators_receive_each_canonical_prefix_operand(count: int) -> None: + """Canonical operands remain visible at both positions before length rejection. + + The host string need not equal the catalog bytes: its overridden equality + remains authoritative. The Rust calculator must still compare the common + prefix before rejecting an incomplete or overlong caller tuple. + """ + operands: List[object] = [] + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + operands.append(other) + return other is BENCHMARKS[len(operands) - 1] + + results = tuple(BenchmarkResult(Name('host name'), status='completed', pi_equivalent=value) + for value in (4.0, 9.0, 16.0)[:count]) + equivalent = SuiteResult.from_results(results).aggregated_scores.pi_equivalent + assert equivalent == (6.0 if count == 2 else None) + assert len(operands) == min(count, 2) + assert all(operand is BENCHMARKS[index] for index, operand in enumerate(operands)) + + +@pytest.mark.parametrize('position', [0, 1]) +@pytest.mark.parametrize('error_type', [ValueError, RuntimeError, KeyboardInterrupt, SystemExit]) +@pytest.mark.parametrize('factory', ['aggregate', 'validation']) +def test_canonical_name_operator_errors_keep_identity_and_stop_at_their_position( + position: int, error_type: Type[BaseException], factory: str, +) -> None: + """A canonical-operand-dependent exception must escape unchanged and lazily. + + Freshly decoded strings hide the caller's exception by producing an ordinary + mismatch. Check both aggregate positions and both validation candidates, + including backend-looking exceptions which validation used to propagate. + """ + original = error_type('canonical comparison failed') + operands: List[object] = [] + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + operands.append(other) + if other is BENCHMARKS[position]: + raise original + return factory == 'aggregate' and other is BENCHMARKS[0] + + def invoke() -> object: + if factory == 'aggregate': + return SuiteResult.from_results(tuple( + BenchmarkResult(Name('host name'), status='completed', pi_equivalent=1.0) + for _index in range(2) + )) + return BenchmarkRunner().run(Name('host name'), warmup_seconds=0, measurement_seconds=.01) + + with pytest.raises(error_type, match='canonical comparison failed') as caught: + invoke() + assert caught.value is original + assert len(operands) == position + 1 + assert all(operand is BENCHMARKS[index] for index, operand in enumerate(operands)) + + +@pytest.mark.parametrize('error_type', [ValueError, KeyboardInterrupt, SystemExit]) +def test_execution_canonical_name_operator_errors_keep_identity(error_type: Type[BaseException]) -> None: + """Execution preserves exceptions raised only for the canonical single-core operand. + + Let validation succeed independently, then expose the execution comparison + without letting a wrong multicore selection consume a long benchmark run. + """ + original = error_type('canonical execution failed') + operands: List[object] = [] + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + operands.append(other) + if len(operands) == 1: + return True + if other is BENCHMARKS[0]: + raise original + return False + + runner = BenchmarkRunner() + with pytest.raises(error_type, match='canonical execution failed') as caught: + runner.run(Name('single_core'), warmup_seconds=0, measurement_seconds=.01, + on_progress=lambda _progress: runner.cancel()) + assert caught.value is original + assert len(operands) == 2 + assert operands[1] is BENCHMARKS[0] + + +@pytest.mark.parametrize('factory', ['validation', 'execution', 'aggregate']) +def test_unknown_core_name_operands_are_forwarded_without_catalog_substitution( + monkeypatch: pytest.MonkeyPatch, factory: str, +) -> None: + """Canonical-object reuse must leave future or injected core operands intact. + + Drive the actual host callbacks with an unknown borrowed UTF-8 name. The + transport may reuse known metadata strings, but must not select a catalog + position itself or silently replace an unrecognized core operand. + """ + from benchset import _aggregate # noqa: I001, PLC0415 — The old package has no lazy aggregate module. + + raw = b'future_core_name' + buffer = ctypes.create_string_buffer(raw) + operands: List[object] = [] + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + operands.append(other) + return other == raw.decode('utf-8') + + def match(callback: Any, user: object, _execution: int) -> int: + return 47 if callback(ctypes.addressof(buffer), len(raw), user) else 0 + + def calculate(_names: object, _count: int, reader: Any, user: object, _output: object, _host: int) -> int: + request = _aggregate._Input() + request.name = ctypes.addressof(buffer) + request.name_size = len(raw) + assert reader(0, 5, ctypes.byref(request), user) == 1 + assert request.has_pi_equivalent == 1 + return 1 + + native = _native.Native.__new__(_native.Native) + cast(Any, native).lib = SimpleNamespace(benchset_benchmark_match=match, benchset_aggregate_read=calculate) + if factory == 'aggregate': + monkeypatch.setattr(_native, 'Native', lambda: native) + assert SuiteResult.from_results((BenchmarkResult(Name('host name')),)).aggregated_scores.pi_equivalent is None + else: + assert native.benchmark_id(Name('host name'), execution=factory == 'execution') == 47 + assert operands == ['future_core_name'] + assert all(operands[0] is not candidate for candidate in BENCHMARKS) diff --git a/tests/test_native_lifetimes.py b/tests/test_native_lifetimes.py new file mode 100644 index 0000000..bb9b7d5 --- /dev/null +++ b/tests/test_native_lifetimes.py @@ -0,0 +1,286 @@ +"""Bound native-loader memory while keeping each Python binding independently configured.""" + +import ctypes +import gc +import weakref +from typing import Any, Callable, List, Optional, Tuple + +import pytest + +from benchset import BenchmarkResult, SuiteResult, __version__, _native, api, cli +from benchset._native import ABI_VERSION, Native +from tests.test_resources import ( + without_cyclic_gc, # noqa: F401 — Reuse the GC-disabled fixture. +) + +_Reference = Callable[[], Optional[object]] + + +def _live_count(references: List[_Reference]) -> int: + return len({id(value) for reference in references if (value := reference()) is not None}) + + +@pytest.mark.usefixtures('without_cyclic_gc') +@pytest.mark.parametrize('operation', ['aggregate', 'cli', 'invalid_run']) +def test_new_native_consumers_keep_loader_objects_bounded_without_gc( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], operation: str, +) -> None: + """Repeated scoring and validation must not add a CDLL/function-type cycle per call. + + Observe actual function pointers and their dynamic ctypes classes while + leaving the standard CDLL factory intact. A single process-lived library + is permitted; per-operation growth is not. Weak references distinguish + retained objects from allocator high-water marks and tracing overhead. + """ + pointers: List[_Reference] = [] + pointer_types: List[_Reference] = [] + + class ObservedNative(Native): + def __init__(self) -> None: + super().__init__() + function = self.lib.benchset_create + pointers.append(weakref.ref(function)) + pointer_types.append(weakref.ref(type(function))) + + for module in (_native, api, cli): + monkeypatch.setattr(module, 'Native', ObservedNative, raising=False) + values = (BenchmarkResult('single_core', status='completed', pi_equivalent=1), + BenchmarkResult('multi_core', status='completed', pi_equivalent=4)) + + def invoke() -> None: + if operation == 'aggregate': + assert SuiteResult.from_results(values).aggregated_scores.pi_equivalent == 2 + elif operation == 'cli': + with pytest.raises(SystemExit) as caught: + cli.settings(['--warmup=-1', '--duration=.1']) + assert caught.value.code == 2 + capsys.readouterr() + else: + result = api.run('unknown', warmup_seconds=0, measurement_seconds=.1) + assert result.status == 'failed' + assert result.reason == 'unknown benchmark: unknown' + + ObservedNative() + invoke() + # Exclude import, FFI-type and parser warmup before starting the no-GC batches. + gc.collect() + pointers.clear() + pointer_types.clear() + for _batch in range(3): + for _attempt in range(20): + invoke() + assert _live_count(pointers) <= 1, 'native function-pointer ownership grows with each call' + assert _live_count(pointer_types) <= 1, 'CDLL construction leaves a new cyclic function type per call' + + +def test_native_function_signatures_are_independent_between_instances(monkeypatch: pytest.MonkeyPatch) -> None: + """A shared raw library must not share mutable argtypes/restype or function monkeypatches.""" + first, second = Native(), Native() + original = second.lib.benchset_abi_version + assert first.lib.benchset_abi_version is not original + expected_arguments, expected_result = list(original.argtypes), original.restype + first.lib.benchset_abi_version.argtypes = [ctypes.c_int] + first.lib.benchset_abi_version.restype = None + assert original.argtypes == expected_arguments + assert original.restype is expected_result + assert original() == ABI_VERSION + monkeypatch.setattr(first.lib, 'benchset_abi_version', lambda: 999) + assert second.lib.benchset_abi_version() == ABI_VERSION + assert Native().lib.benchset_abi_version() == ABI_VERSION + + +@pytest.mark.usefixtures('without_cyclic_gc') +@pytest.mark.parametrize('operation', ['aggregate', 'invalid_run']) +def test_quiet_numeric_copies_release_their_function_pointers_without_gc( + monkeypatch: pytest.MonkeyPatch, operation: str, +) -> None: + """Public calls must not leave a quiet-copy function cycle for each Native. + + Casting a ctypes function pointer to void* mutates its ownership dictionary + into a self-cycle. Weakly observe the actual per-call helper and its type, + with collection disabled, to distinguish this leak from a loaded library's + permitted process lifetime and to keep repeated scoring/validation bounded. + """ + pointers: List[_Reference] = [] + pointer_types: List[_Reference] = [] + + class ObservedNative(Native): + def __init__(self) -> None: + super().__init__() + function = self.lib.benchset_copy_double + pointers.append(weakref.ref(function)) + pointer_types.append(weakref.ref(type(function))) + + monkeypatch.setattr(_native, 'Native', ObservedNative) + monkeypatch.setattr(api, 'Native', ObservedNative) + values = (BenchmarkResult('single_core', status='completed', pi_equivalent=1), + BenchmarkResult('multi_core', status='completed', pi_equivalent=4)) + + def invoke() -> None: + if operation == 'aggregate': + assert SuiteResult.from_results(values).aggregated_scores.pi_equivalent == 2 + else: + result = api.run('unknown', warmup_seconds=0, measurement_seconds=.1) + assert result.status == 'failed' + assert result.reason == 'unknown benchmark: unknown' + + def counts() -> Tuple[int, int]: + objects = gc.get_objects() + return (sum(issubclass(type(value), _native.CFuncPtr) for value in objects), + sum(issubclass(type(value), type) and issubclass(value, _native.CFuncPtr) for value in objects)) + + invoke() + gc.collect() + pointers.clear() + pointer_types.clear() + baseline = counts() + for _batch in range(3): + for _attempt in range(16): + invoke() + assert _live_count(pointers) <= 1, 'quiet numeric copying retains one cyclic function pointer per call' + assert _live_count(pointer_types) <= 1, 'quiet numeric copying retains per-call ctypes types' + actual = counts() + assert actual[0] <= baseline[0] + 1, 'quiet numeric copying retains temporary ctypes callables' + assert actual[1] <= baseline[1] + 1, 'quiet numeric copying creates unbounded ctypes callable types' + + +@pytest.mark.parametrize('failure', ['abi', 'version', 'symbol']) +def test_native_revalidates_cached_library_and_preserves_error_priority( + monkeypatch: pytest.MonkeyPatch, failure: str, +) -> None: + """Every binding rechecks ABI, package version and symbols, even after a successful warm load. + + Patch the standard loader's symbol resolver rather than replacing its + factory: this also exercises a cached raw library. ABI/version rejection + must precede required-symbol resolution and failed validation must recover. + """ + state = {'abi': ABI_VERSION, 'version': __version__.encode('utf-8'), 'missing': False} + lookups: List[str] = [] + resolve = ctypes.CDLL.__getattribute__ + fallback = ctypes.CDLL.__getattr__ + + def lookup(library: ctypes.CDLL, name: str) -> Any: + if name.startswith('benchset_'): + lookups.append(name) + if name == 'benchset_abi_version': + return lambda: state['abi'] + if name == 'benchset_version': + return lambda: state['version'] + if name == 'benchset_sample' and state['missing']: + raise AttributeError(name) + return resolve(library, name) + + def missing(library: ctypes.CDLL, name: str) -> Any: + # AttributeError from __getattribute__ must not resolve the hidden + # symbol through CDLL's normal __getattr__ fallback. + if name == 'benchset_sample' and state['missing']: + raise AttributeError(name) + return fallback(library, name) + + monkeypatch.setattr(ctypes.CDLL, '__getattribute__', lookup) + monkeypatch.setattr(ctypes.CDLL, '__getattr__', missing) + Native() + lookups.clear() + state['missing'] = True + if failure == 'abi': + state['abi'] = ABI_VERSION + 1 + elif failure == 'version': + state['version'] = b'not-the-package-version' + message = 'missing required symbol: benchset_sample' if failure == 'symbol' else 'version/ABI' + with pytest.raises(RuntimeError, match=message): + Native() + if failure != 'symbol': + assert set(lookups) <= {'benchset_abi_version', 'benchset_version'} + state.update(abi=ABI_VERSION, version=__version__.encode('utf-8'), missing=False) + assert Native().lib.benchset_abi_version() == ABI_VERSION + + +def test_custom_loader_is_called_again_after_success_and_failure(monkeypatch: pytest.MonkeyPatch) -> None: + """Caching the standard library cannot cache an injected loader's objects or suppress its errors.""" + library = Native().lib + calls: List[str] = [] + original = OSError('fresh injected loader failure') + + def load(path: str) -> Any: + calls.append(path) + if len(calls) in (2, 3): + raise original + return library + + monkeypatch.setattr(ctypes, 'CDLL', load) + assert Native().lib.benchset_abi_version() == ABI_VERSION + for _attempt in range(2): + with pytest.raises(OSError, match='fresh injected loader failure') as caught: + Native() + assert caught.value is original + assert Native().lib.benchset_abi_version() == ABI_VERSION + assert len(calls) == 4 + assert len(set(calls)) == 1 + + +@pytest.mark.usefixtures('without_cyclic_gc') +def test_custom_loader_closure_is_not_retained_after_restoration(monkeypatch: pytest.MonkeyPatch) -> None: + """An injected factory and its captured data must not become process-lived cache keys.""" + library = Native().lib + + class Loader: + def __init__(self) -> None: + self.payload = bytearray(64 * 1024) + + def __call__(self, _path: str) -> Any: + return library + + loader = Loader() + observed = weakref.ref(loader) + with monkeypatch.context() as patch: + patch.setattr(ctypes, 'CDLL', loader) + assert Native().lib.benchset_abi_version() == ABI_VERSION + del loader + assert observed() is None + + +def test_duck_library_is_not_probed_before_fixed_abi_checks(monkeypatch: pytest.MonkeyPatch) -> None: + """Function-table construction must not invoke arbitrary duck-library lookups before rejecting its ABI.""" + lookups: List[str] = [] + + class Library: + def __getattr__(self, name: str) -> Any: + lookups.append(name) + if name == 'benchset_abi_version': + return lambda: ABI_VERSION + 1 + if name == 'benchset_version': + return lambda: __version__.encode('utf-8') + raise AssertionError('unexpected lookup before ABI rejection: ' + name) + + monkeypatch.setattr(ctypes, 'CDLL', lambda _path: Library()) + with pytest.raises(RuntimeError, match='version/ABI'): + Native() + assert set(lookups) == {'benchset_abi_version', 'benchset_version'} + + +@pytest.mark.usefixtures('without_cyclic_gc') +def test_address_bound_function_keeps_its_original_owner_alive() -> None: + """A copied C function must keep a temporary callback's executable owner alive. + + This checks the new function table directly: HEAD had no address cloning. + Assert the original still exists before calling its address, so a broken + owner chain produces an assertion rather than invoking freed executable memory. + Releasing the function must also release its owner without another GC pass. + """ + functions = getattr(_native, '_Functions', None) + if functions is None: + pytest.skip('HEAD has no address-bound function table') + originals: List[_Reference] = [] + + class Library: + def __getattr__(self, _name: str) -> Any: + callback = ctypes.CFUNCTYPE(ctypes.c_int)(lambda: 71) + originals.append(weakref.ref(callback)) + return callback + + function = functions(Library()).example + gc.collect() + assert originals[0]() is not None, 'the copied address outlived its executable callback owner' + assert function() == 71 + del function + assert originals[0]() is None, 'resolving a callback introduced a cyclic owner reference' diff --git a/tests/test_publication.py b/tests/test_publication.py new file mode 100644 index 0000000..fcc8bf7 --- /dev/null +++ b/tests/test_publication.py @@ -0,0 +1,283 @@ +"""A package's native library and Rust metadata must publish as one generation.""" + +from __future__ import annotations + +import os +import stat +import subprocess +import sys +from contextlib import contextmanager +from pathlib import Path +from queue import Empty, Queue +from threading import Thread +from typing import Iterator, List, Optional, Tuple + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + +# Each publisher is a separate interpreter, so an in-process lock cannot make +# these tests pass. Announcing lock acquisition is only a scheduling handshake: +# the assertions inspect the resulting files, never the lock's implementation. +# If no lock exists, B announces its completed publication before A is released, +# deterministically exposing the original B-library/A-metadata interleaving. +_PUBLISHER = r''' +import os +import sys +from pathlib import Path +from filelock import FileLock +from scripts import cargo +from scripts.cargo import publish + +package, compiled, generation, pause, fail = sys.argv[1:] +package, compiled = Path(package), Path(compiled) +replace = Path.replace + +report_lock = os.environ.get('BENCHSET_PUBLISH_REPORT_LOCK') == '1' +if pause == 'no' or report_lock: + if hasattr(cargo, '_package_lock'): + acquire = cargo._package_lock + def announce_acquire(path): + print('LOCK=' + str(path.resolve()) if report_lock else 'LOCK', flush=True) + return acquire(path) + cargo._package_lock = announce_acquire + else: + # Characterize the previous backend using the same process handshake. + acquire = FileLock.acquire + def announce_acquire(self, *args, **kwargs): + print('LOCK=' + str(Path(self.lock_file).resolve()) if report_lock else 'LOCK', flush=True) + return acquire(self, *args, **kwargs) + FileLock.acquire = announce_acquire + +def controlled_replace(source, destination): + if source.name == '_metadata.json' and fail == 'yes': + raise OSError('metadata rename failed for ' + generation) + result = replace(source, destination) + if source.name == compiled.name and pause == 'yes': + print('LIBRARY', flush=True) + assert sys.stdin.readline() == 'release\n' + return result + +Path.replace = controlled_replace +print('STARTED', flush=True) +try: + publish(package, compiled, (generation + ' metadata').encode('ascii')) +except OSError as error: + print('FAILED: ' + str(error), flush=True) +else: + print('DONE', flush=True) +''' + +_Child = Tuple['subprocess.Popen[str]', 'Queue[str]', List[str], Thread] + + +@contextmanager +def _publisher( # noqa: PLR0913 — Independent process/environment fault controls. + package: Path, compiled: Path, generation: str, *, pause: bool, fail: bool, + temporary: Optional[Path] = None, report_lock: bool = False, +) -> Iterator[_Child]: + environment = dict(os.environ) + if temporary is not None: + temporary.mkdir() + environment.update(dict.fromkeys(('TMPDIR', 'TEMP', 'TMP'), str(temporary))) + if report_lock: + environment['BENCHSET_PUBLISH_REPORT_LOCK'] = '1' + process = subprocess.Popen( + [sys.executable, '-u', '-c', _PUBLISHER, str(package), str(compiled), generation, + 'yes' if pause else 'no', 'yes' if fail else 'no'], + cwd=ROOT, env=environment, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, + ) + messages: Queue[str] = Queue() + lines: List[str] = [] + + def read() -> None: + assert process.stdout is not None + for line in process.stdout: + lines.append(line.rstrip('\n')) + messages.put(line.rstrip('\n')) + + reader = Thread(target=read, daemon=True) + reader.start() + try: + yield process, messages, lines, reader + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + reader.join(timeout=10) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None: + stream.close() + + +def _receive(child: _Child) -> str: + process, messages, lines, _reader = child + try: + return messages.get(timeout=10) + except Empty: + pytest.fail('publisher stalled: status=%r, messages=%r' % (process.poll(), lines)) + + +def _release(child: _Child) -> None: + assert child[0].stdin is not None + child[0].stdin.write('release\n') + child[0].stdin.flush() + + +def _finish(child: _Child, *, failed: bool) -> None: + process, _messages, lines, reader = child + assert process.wait(timeout=10) == 0 + reader.join(timeout=10) + assert process.stderr is not None + assert not process.stderr.read() + if failed: + assert lines[-1].startswith('FAILED: metadata rename failed for ') + else: + assert lines[-1] == 'DONE' + + +def _seed(directory: Path, generation: str) -> Path: + library = directory / '_native/libbenchset.so' + library.parent.mkdir(parents=True) + library.write_bytes((generation + ' library').encode('ascii')) + (directory / '_metadata.json').write_bytes((generation + ' metadata').encode('ascii')) + return library + + +def _compiled(directory: Path, generation: str) -> Path: + path = directory / generation / 'libbenchset.so' + path.parent.mkdir(parents=True) + path.write_bytes((generation + ' library').encode('ascii')) + return path + + +def _assert_generation(package: Path, generation: str) -> None: + assert (package / '_native/libbenchset.so').read_bytes() == (generation + ' library').encode('ascii') + assert (package / '_metadata.json').read_bytes() == (generation + ' metadata').encode('ascii') + + +@pytest.mark.parametrize(('fail_first', 'fail_second'), [(False, False), (True, False), (True, True)]) +def test_concurrent_process_publishers_keep_a_complete_generation( + tmp_path: Path, fail_first: bool, fail_second: bool, +) -> None: + """Serialize the previous snapshot, staging, commit and rollback per package. + + A pauses immediately after installing its library. Without serialization B + publishes fully; A then installs A metadata, or rolls back over B's library. + Both leave mismatched files. Two failing publishers also cover taking the + rollback snapshot before acquiring a lock: B must not snapshot A's library + while A still has the old metadata and may roll back to the old generation. + All inputs are immutable files; no Cargo/source-edit race is involved. + """ + package = tmp_path / 'package' + _seed(package, 'old') + first, second = _compiled(tmp_path, 'A'), _compiled(tmp_path, 'B') + with _publisher(package, first, 'A', pause=True, fail=fail_first) as a: + assert _receive(a) == 'STARTED' + assert _receive(a) == 'LIBRARY' + with _publisher(package, second, 'B', pause=False, fail=fail_second) as b: + assert _receive(b) == 'STARTED' + boundary = _receive(b) + assert boundary in {'LOCK', 'DONE'} or boundary.startswith('FAILED: ') + _release(a) + _finish(a, failed=fail_first) + _finish(b, failed=fail_second) + _assert_generation(package, 'old' if fail_second else 'B') + + +def test_publication_in_an_independent_package_does_not_wait_for_another_package(tmp_path: Path) -> None: + """A held package lock must leave publication of unrelated packages available.""" + package_a, package_b = tmp_path / 'package-a', tmp_path / 'package-b' + _seed(package_a, 'old') + _seed(package_b, 'old') + first, second = _compiled(tmp_path, 'A'), _compiled(tmp_path, 'B') + with _publisher(package_a, first, 'A', pause=True, fail=False) as a: + assert _receive(a) == 'STARTED' + assert _receive(a) == 'LIBRARY' + with _publisher(package_b, second, 'B', pause=False, fail=False) as b: + assert _receive(b) == 'STARTED' + boundary = _receive(b) + if boundary == 'LOCK': + boundary = _receive(b) + assert boundary == 'DONE' + _finish(b, failed=False) + _release(a) + _finish(a, failed=False) + _assert_generation(package_a, 'A') + _assert_generation(package_b, 'B') + + +def test_package_publication_uses_the_same_lock_with_different_temporary_directories(tmp_path: Path) -> None: + """Per-process TMPDIR/TEMP/TMP must not separate one package's lock identity. + + If B announces a different lock, let B finish before releasing A to expose + the exact mixed generation. Matching locks can immediately release A: its + OS lock still prevents B from publishing until the complete pair is ready. + """ + package = tmp_path / 'package' + _seed(package, 'old') + first, second = _compiled(tmp_path, 'A'), _compiled(tmp_path, 'B') + with _publisher(package, first, 'A', pause=True, fail=False, + temporary=tmp_path / 'temp-a', report_lock=True) as a: + assert _receive(a) == 'STARTED' + first_lock = _receive(a) + assert first_lock.startswith('LOCK=') + assert _receive(a) == 'LIBRARY' + with _publisher(package, second, 'B', pause=False, fail=False, + temporary=tmp_path / 'temp-b', report_lock=True) as b: + assert _receive(b) == 'STARTED' + second_lock = _receive(b) + assert second_lock.startswith('LOCK=') + if first_lock != second_lock: + assert _receive(b) == 'DONE' + _finish(b, failed=False) + _release(a) + _finish(a, failed=False) + _finish(b, failed=False) + _assert_generation(package, 'B') + assert first_lock == second_lock + + +@pytest.mark.skipif(os.name != 'posix' or os.geteuid() == 0, reason='Requires non-root POSIX DAC permissions') +@pytest.mark.parametrize('mask', [0o022, 0o077]) +def test_generated_publication_lock_remains_usable_by_an_authorized_other_writer(tmp_path: Path, mask: int) -> None: + """The first writer must not make later package-owner publication impossible. + + A first build under another UID can leave its persistent lock foreign-owned, + even though the package owner can replace every product through writable + directories. Simulate that UID's effective access without privileges: after + the real first publication, map its generated lock's OTHER permission bits + onto our OWNER bits. Do not unlink the lock or change either output's modes. + Both ordinary and restrictive umasks must permit another coherent publication + using the same lock inode; readonly native images remain replaceable. Keep + a readonly descriptor open so an unsafe unlink/recreate cannot reuse the + just-freed inode number and make the identity assertion pass accidentally. + """ + # Keep build tooling separate from the runtime-only imports above. + from scripts.cargo import publish # noqa: PLC0415 + + package = tmp_path / 'package' + installed = _seed(package, 'old') + installed.chmod(0o444) + first, second = _compiled(tmp_path, 'A'), _compiled(tmp_path, 'B') + first.chmod(0o444) + second.chmod(0o444) + previous_mask = os.umask(mask) + try: + assert publish(package, first, b'A metadata') == installed + finally: + os.umask(previous_mask) + _assert_generation(package, 'A') + assert not installed.stat().st_mode & stat.S_IWUSR + + lock = package / '.benchset.publish.lock' + generated = lock.stat() + mode = stat.S_IMODE(generated.st_mode) + lock.chmod((mode & 0o077) | ((mode & 0o007) << 6)) + + with lock.open('rb') as identity: + assert publish(package, second, b'B metadata') == installed + _assert_generation(package, 'B') + assert lock.stat().st_ino == os.fstat(identity.fileno()).st_ino == generated.st_ino + assert not installed.stat().st_mode & stat.S_IWUSR diff --git a/tests/test_publication_signals.py b/tests/test_publication_signals.py new file mode 100644 index 0000000..1c04a91 --- /dev/null +++ b/tests/test_publication_signals.py @@ -0,0 +1,576 @@ +"""Publication locks keep descriptors owned across ordinary and signal errors.""" + +from __future__ import annotations + +import errno +import importlib.util +import os +import signal +import subprocess +import sys +import weakref +from pathlib import Path +from threading import Thread +from types import SimpleNamespace +from typing import Any + +import pytest +from filelock import FileLock, Timeout + +from scripts import cargo + +_OWNERS: weakref.WeakValueDictionary[int, Any] = weakref.WeakValueDictionary() + + +@pytest.fixture(autouse=True) +def _observe_owned_lock_files(monkeypatch: pytest.MonkeyPatch) -> None: + """Failing mutations close file objects, avoiding later closes of reused fds.""" + original = Path.open + + def opened(path: Path, *args: Any, **kwargs: Any) -> Any: + file = original(path, *args, **kwargs) + if path.name == '.benchset.publish.lock': + _OWNERS[file.fileno()] = file + return file + + monkeypatch.setattr(Path, 'open', opened) + + +def _products(tmp_path: Path) -> tuple[Path, Path]: + package = tmp_path / 'package' + native = package / '_native' + native.mkdir(parents=True) + (native / 'libbenchset.so').write_bytes(b'old image') + (package / '_metadata.json').write_bytes(b'old metadata') + image = tmp_path / 'libbenchset.so' + image.write_bytes(b'new image') + return package, image + + +def _open(descriptor: int) -> bool: + try: + os.fstat(descriptor) + except OSError: + return False + return True + + +def _close_orphaned(descriptors: list[int], lock: Path) -> None: + """Clean failing reproductions without closing descriptors reused elsewhere.""" + for descriptor in descriptors: + try: + actual, expected = os.fstat(descriptor), lock.stat() + if (actual.st_dev, actual.st_ino) == (expected.st_dev, expected.st_ino): + owner = _OWNERS.get(descriptor) + if owner is None or owner.closed: + os.close(descriptor) + else: + owner.close() + except OSError: + pass + + +@pytest.mark.skipif(os.name != 'posix', reason='Exercise real POSIX flock handoffs') +@pytest.mark.parametrize('when', ['before', 'after']) +@pytest.mark.parametrize('policy', ['default', 'custom-error', 'custom-return']) +@pytest.mark.parametrize('which', ['SIGINT', 'SIGUSR1']) +@pytest.mark.parametrize('stage', ['acquire', 'release']) +def test_signal_at_lock_handoff_keeps_descriptor_owned( # noqa: PLR0913 — Independent lock handoff, timing, and caller-signal policies. + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, when: str, policy: str, which: str, stage: str, +) -> None: + """Signals before/after either real lock primitive cannot orphan its descriptor. + + Include SIGUSR1: ownership must not rely on recognizing and deferring only + Ctrl+C. Custom exceptions keep identity/context; nonraising handlers keep + running and the same persistent inode remains available to another owner. + """ + import fcntl # noqa: PLC0415 — POSIX-only primitive, after the skip. + + package, image = _products(tmp_path) + original = fcntl.flock + touched: list[int] = [] + events: list[str] = [] + failure = RuntimeError('caller signal failed') + context = LookupError('caller context') + failure.__context__ = context + signum = getattr(signal, which) + + def handler(_signum: int, _frame: Any) -> None: + events.append('signal') + if policy == 'custom-error': + raise failure + + def interrupted(file: Any, flags: int) -> Any: + selected = (bool(flags & fcntl.LOCK_EX) if stage == 'acquire' else flags == fcntl.LOCK_UN) and not touched + if selected: + touched.append(file if isinstance(file, int) else file.fileno()) + if when == 'before': + signal.raise_signal(signum) + result = original(file, flags) + if selected and when == 'after': + signal.raise_signal(signum) + return result + + previous = signal.signal(signum, signal.default_int_handler if policy == 'default' else handler) + try: + with monkeypatch.context() as hooks: + hooks.setattr(fcntl, 'flock', interrupted) + if policy == 'custom-return': + cargo.publish(package, image, b'new metadata') + else: + with pytest.raises(KeyboardInterrupt if policy == 'default' else RuntimeError) as caught: + cargo.publish(package, image, b'new metadata') + if policy == 'custom-error': + assert caught.value is failure + assert failure.__context__ is context + assert len(touched) == 1 + assert not _open(touched[0]), 'the signal abandoned an open descriptor' + with FileLock(str(package / '.benchset.publish.lock'), timeout=0): + pass + expected = b'new' if stage == 'release' or policy == 'custom-return' else b'old' + assert (package / '_native/libbenchset.so').read_bytes() == expected + b' image' + assert (package / '_metadata.json').read_bytes() == expected + b' metadata' + assert events == ([] if policy == 'default' else ['signal']) + finally: + signal.signal(signum, previous) + _close_orphaned(touched, package / '.benchset.publish.lock') + + +@pytest.mark.skipif(os.name != 'posix', reason='Observe real POSIX descriptor ownership') +@pytest.mark.parametrize('when', ['before', 'after']) +@pytest.mark.parametrize('error_type', [KeyboardInterrupt, RuntimeError, OSError, ValueError]) +def test_publication_body_errors_close_the_lock_before_propagation( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, when: str, error_type: type[BaseException], +) -> None: + """Every body exception releases ownership, before or after a complete commit. + + Keep the exact exception and any completed pair. Closing the owned file + unlocks the kernel resource without a separately interruptible unlock/state + handoff; another FileLock client may acquire the persistent inode. + """ + import fcntl # noqa: PLC0415 + + package, image = _products(tmp_path) + flock, publish = fcntl.flock, cargo._publish + touched: list[int] = [] + failure = error_type('publication body failed') + + def acquired(file: Any, flags: int) -> Any: + if flags & fcntl.LOCK_EX: + touched.append(file if isinstance(file, int) else file.fileno()) + return flock(file, flags) + + def failed(*args: Any) -> Path: + if when == 'before': + raise failure + publish(*args) + raise failure + + try: + monkeypatch.setattr(fcntl, 'flock', acquired) + monkeypatch.setattr(cargo, '_publish', failed) + with pytest.raises(error_type) as caught: + cargo.publish(package, image, b'new metadata') + assert caught.value is failure + assert len(touched) == 1 + assert not _open(touched[0]) + with FileLock(str(package / '.benchset.publish.lock'), timeout=0): + pass + expected = b'old' if when == 'before' else b'new' + assert (package / '_native/libbenchset.so').read_bytes() == expected + b' image' + assert (package / '_metadata.json').read_bytes() == expected + b' metadata' + finally: + _close_orphaned(touched, package / '.benchset.publish.lock') + + +@pytest.mark.skipif(os.name != 'posix', reason='Exercise real POSIX flock handoffs') +@pytest.mark.parametrize('raises', [False, True]) +@pytest.mark.parametrize('which', ['SIGINT', 'SIGUSR1']) +@pytest.mark.parametrize('stage', ['acquire', 'release']) +def test_lock_handoff_preserves_handler_policy_changes_and_exact_errors( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, raises: bool, which: str, stage: str, +) -> None: + """Handlers may select SIG_IGN without later restoration overwriting them. + + Deliver twice to detect stale handler replay. A raising handler preserves + its own error and still leaves the descriptor closed and the lock reusable. + """ + import fcntl # noqa: PLC0415 + + package, image = _products(tmp_path) + original = fcntl.flock + touched: list[int] = [] + events: list[str] = [] + failure = RuntimeError('caller selected ignore') + signum = getattr(signal, which) + + def handler(_signum: int, _frame: Any) -> None: + events.append('signal') + signal.signal(signum, signal.SIG_IGN) + if raises: + raise failure + + def interrupted(file: Any, flags: int) -> Any: + result = original(file, flags) + selected = bool(flags & fcntl.LOCK_EX) if stage == 'acquire' else flags == fcntl.LOCK_UN + if selected and not touched: + touched.append(file if isinstance(file, int) else file.fileno()) + signal.raise_signal(signum) + signal.raise_signal(signum) + return result + + previous = signal.signal(signum, handler) + try: + with monkeypatch.context() as hooks: + hooks.setattr(fcntl, 'flock', interrupted) + if raises: + with pytest.raises(RuntimeError) as caught: + cargo.publish(package, image, b'new metadata') + assert caught.value is failure + else: + cargo.publish(package, image, b'new metadata') + assert signal.getsignal(signum) is signal.SIG_IGN + assert events == ['signal'] + assert not _open(touched[0]) + with FileLock(str(package / '.benchset.publish.lock'), timeout=0): + pass + finally: + signal.signal(signum, previous) + _close_orphaned(touched, package / '.benchset.publish.lock') + + +def test_waiting_for_another_publisher_remains_cancellable() -> None: + """Ctrl+C exits contention rather than waiting for the other owner's release. + + A FileLock owner releases after publication has returned, with a bounded + fallback to expose a swallowed interruption without hanging the test runner. + This also verifies interoperability with the previous locking backend. + """ + code = '''import os, signal, tempfile, _thread +from pathlib import Path +from threading import Event, Thread, Timer +from filelock import FileLock +from scripts.cargo import publish +with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + package = root / 'package' + package.mkdir() + image = root / 'libbenchset.so' + image.write_bytes(b'image') + owner = FileLock(str(package / '.benchset.publish.lock'), thread_local=False) + owner.acquire() + returned = Event() + fallback = [] + def release(): + fallback.append(not returned.wait(2)) + owner.release() + worker = Thread(target=release) + worker.start() + timer = Timer(.1, lambda: _thread.interrupt_main() if os.name == 'nt' else os.kill(os.getpid(), signal.SIGINT)) + timer.start() + caught = False + try: + try: + publish(package, image, b'metadata') + except KeyboardInterrupt: + caught = True + finally: + returned.set() + timer.cancel() + timer.join() + worker.join(5) + owner.release() + assert caught + assert fallback == [False], 'waiting swallowed Ctrl+C' + assert not worker.is_alive() +''' + result = subprocess.run([sys.executable, '-c', code], capture_output=True, text=True, check=False, timeout=10) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_background_publication_does_not_install_process_signal_handlers( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, +) -> None: + """A background builder needs no process signal hook and may publish normally.""" + package, image = _products(tmp_path) + errors: list[BaseException] = [] + + def forbidden(*_args: Any) -> None: + raise AssertionError('a background publisher installed a process signal handler') + + def publish() -> None: + try: + cargo.publish(package, image, b'new metadata') + except BaseException as error: # noqa: BLE001 — Return child-thread errors to the test. + errors.append(error) + + monkeypatch.setattr(signal, 'signal', forbidden) + worker = Thread(target=publish) + worker.start() + worker.join(5) + assert not worker.is_alive() + assert not errors + assert (package / '_metadata.json').read_bytes() == b'new metadata' + + +def test_kernel_lock_remains_exclusive_through_the_complete_publication( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, +) -> None: + """The body owns the actual kernel lock before and after committing both files. + + An independent FileLock descriptor must fail nonblockingly while the body + runs. This deterministically rejects a context that acquires and closes its + file before yielding, which a process scheduling handshake alone can miss. + """ + package, image = _products(tmp_path) + original = cargo._publish + checked: list[str] = [] + + def inspect(*args: Any) -> Path: + for phase in ('before', 'after'): + if phase == 'after': + result = original(*args) + with pytest.raises(Timeout), FileLock(str(package / '.benchset.publish.lock'), timeout=0): + pass + checked.append(phase) + return result + + monkeypatch.setattr(cargo, '_publish', inspect) + assert cargo.publish(package, image, b'new metadata') == package / '_native/libbenchset.so' + assert checked == ['before', 'after'] + + +@pytest.mark.skipif(os.name != 'posix' or not hasattr(os, 'fork'), reason='POSIX fork descriptor inheritance') +@pytest.mark.parametrize('stage', ['body-return', 'body-error', 'acquire-interrupt']) +def test_publication_unlocks_while_a_forked_child_keeps_its_inherited_descriptor(stage: str) -> None: + """Parent cleanup releases the shared lock even while a child retains its fd. + + Closing only the parent's file leaves flock held by the inherited open-file + description. Keep the child alive on a pipe and verify another FileLock can + acquire before releasing that pipe. Cover normal return, a body error and a + signal after successful acquisition but before the context reaches yield. + """ + code = f'''import fcntl, os, signal, tempfile +from pathlib import Path +from filelock import FileLock +from scripts import cargo +stage = {stage!r} +flock, publish = fcntl.flock, cargo._publish +read_fd, write_fd = os.pipe() +children = [] +failure = RuntimeError('body failed') +def fork_child(): + child = os.fork() + if child == 0: + os.close(write_fd) + os.read(read_fd, 1) + os._exit(0) + children.append(child) + os.close(read_fd) +def acquired(file, flags): + result = flock(file, flags) + if stage == 'acquire-interrupt' and flags & fcntl.LOCK_EX and not children: + fork_child() + signal.raise_signal(signal.SIGINT) + return result +def body(*args): + if stage != 'acquire-interrupt': + fork_child() + if stage == 'body-error': + raise failure + return publish(*args) +fcntl.flock, cargo._publish = acquired, body +try: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + image = root / 'libbenchset.so' + image.write_bytes(b'image') + package = root / 'package' + try: + cargo.publish(package, image, b'metadata') + except BaseException as error: + if stage == 'acquire-interrupt': + assert isinstance(error, KeyboardInterrupt) + elif stage == 'body-error': + assert error is failure + else: + raise + else: + assert stage == 'body-return' + assert len(children) == 1 + assert os.waitpid(children[0], os.WNOHANG) == (0, 0) + with FileLock(str(package / '.benchset.publish.lock'), timeout=0): + pass +finally: + if children: + os.write(write_fd, b'release') + for child in children: + os.waitpid(child, 0) + else: + os.close(read_fd) + os.close(write_fd) +print('released-with-live-child') +''' + result = subprocess.run([sys.executable, '-c', code], capture_output=True, text=True, check=False, timeout=10) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout == 'released-with-live-child\n' + + +@pytest.mark.skipif(importlib.util.find_spec('_xxsubinterpreters') is None, + reason='The experimental CPython subinterpreter harness is unavailable') +@pytest.mark.parametrize('custom', [False, True], ids=['foreign-default', 'foreign-custom']) +def test_subinterpreter_publication_does_not_attempt_unsupported_signal_setup(custom: bool) -> None: + """The lock preserves the existing subinterpreter publication boundary. + + Their current thread still looks like the threading module's main thread, + while signal.signal is forbidden. The unchanged unlocked publisher supplies + the interpreter's original success/error boundary: Python 3.8 can have a + locally identical default handler and reject signal setup, while newer + interpreters see a foreign handler and publish normally. Locking must not + add another error or retry forever in either case. + """ + root = str(Path(__file__).resolve().parents[1]) + inner = f'''import sys, tempfile +sys.path.insert(0, {root!r}) +from pathlib import Path +from scripts.cargo import _publish, publish +with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + image = root / 'libbenchset.so' + image.write_bytes(b'image') + def attempt(package, operation): + try: + operation(package, image, b'metadata') + except BaseException as error: + return type(error), str(error) + assert (package / '_native/libbenchset.so').read_bytes() == b'image' + assert (package / '_metadata.json').read_bytes() == b'metadata' + return None + expected = attempt(root / 'reference', _publish) + assert attempt(root / 'package', publish) == expected +print('equivalent', flush=True) +''' + code = f'''import _xxsubinterpreters as interpreters +import signal +previous = signal.getsignal(signal.SIGINT) +if {custom!r}: + signal.signal(signal.SIGINT, lambda _signum, _frame: None) +interpreter = interpreters.create() +try: + interpreters.run_string(interpreter, {inner!r}) +finally: + interpreters.destroy(interpreter) + signal.signal(signal.SIGINT, previous) +''' + # Coverage's startup hook deadlocks CPython 3.10 subinterpreter teardown. + # This helper tests build tooling, outside the measured benchset package. + environment = os.environ.copy() + environment.pop('COVERAGE_PROCESS_START', None) + result = subprocess.run([sys.executable, '-c', code], env=environment, + capture_output=True, text=True, check=False, timeout=10) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout == 'equivalent\n' + + +@pytest.mark.parametrize('failures', [0, 2]) +def test_windows_lock_uses_byte_zero_and_retries_only_contention( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, failures: int, +) -> None: + """Emulate the Windows CRT protocol, including nonempty files and EACCES retry.""" + package, image = _products(tmp_path) + lock = package / '.benchset.publish.lock' + lock.write_bytes(b'keep this inode and data') + identity = lock.stat().st_ino + calls: list[int] = [] + sleeps: list[float] = [] + + def locking(descriptor: int, mode: int, count: int) -> None: + assert mode == 77 + assert count == 1 + assert os.lseek(descriptor, 0, os.SEEK_CUR) == 0 + calls.append(descriptor) + assert len(calls) <= failures + 1, 'locking continued after success' + if len(calls) <= failures: + os.lseek(descriptor, 7, os.SEEK_SET) + raise OSError(errno.EACCES, 'another owner') + + monkeypatch.setattr(cargo, 'os', SimpleNamespace(name='nt')) + monkeypatch.setitem(sys.modules, 'msvcrt', SimpleNamespace(LK_NBLCK=77, locking=locking)) + monkeypatch.setattr(cargo, 'sleep', sleeps.append, raising=False) + try: + cargo.publish(package, image, b'new metadata') + assert len(calls) == failures + 1 + assert sleeps == [.05] * failures + assert not _open(calls[-1]) + assert lock.stat().st_ino == identity + assert lock.read_bytes() == b'keep this inode and data' + assert (package / '_metadata.json').read_bytes() == b'new metadata' + finally: + _close_orphaned(calls, lock) + + +@pytest.mark.parametrize('failure', [ + OSError(errno.EBADF, 'bad descriptor'), OSError(errno.EIO, 'I/O failure'), + OSError(errno.EAGAIN, 'unexpected retry code'), KeyboardInterrupt(), RuntimeError('caller failed'), +]) +def test_windows_lock_errors_propagate_once_and_close_the_descriptor( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, failure: BaseException, +) -> None: + """Non-contention errors and asynchronous exceptions cannot leak or retry. + + The fake grants the lock after repeated failures so an overbroad exception + handler fails assertions rather than hanging the test runner indefinitely. + """ + package, image = _products(tmp_path) + calls: list[int] = [] + + def locking(descriptor: int, _mode: int, _count: int) -> None: + calls.append(descriptor) + if len(calls) >= 4: + return + raise failure + + monkeypatch.setattr(cargo, 'os', SimpleNamespace(name='nt')) + monkeypatch.setitem(sys.modules, 'msvcrt', SimpleNamespace(LK_NBLCK=77, locking=locking)) + monkeypatch.setattr(cargo, 'sleep', lambda _seconds: None, raising=False) + try: + with pytest.raises(type(failure)) as caught: + cargo.publish(package, image, b'new metadata') + assert caught.value is failure + assert len(calls) == 1 + assert not _open(calls[0]) + assert (package / '_metadata.json').read_bytes() == b'old metadata' + finally: + _close_orphaned(calls, package / '.benchset.publish.lock') + + +@pytest.mark.skipif(os.name != 'posix', reason='POSIX fchmod protocol') +@pytest.mark.parametrize('error_type', [PermissionError, OSError]) +def test_posix_permission_handoff_ignores_only_foreign_ownership( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, error_type: type[OSError], +) -> None: + """A foreign-owned writable lock remains usable; other chmod errors close its fd.""" + package, image = _products(tmp_path) + descriptors: list[int] = [] + failure = error_type('permission change failed') + + def failed(descriptor: int, mode: int) -> None: + assert mode == 0o666 + descriptors.append(descriptor) + raise failure + + try: + with monkeypatch.context() as hooks: + hooks.setattr(os, 'fchmod', failed) + if error_type is PermissionError: + cargo.publish(package, image, b'new metadata') + else: + with pytest.raises(OSError, match='permission change failed') as caught: + cargo.publish(package, image, b'new metadata') + assert caught.value is failure + assert len(descriptors) == 1 + assert not _open(descriptors[0]) + with FileLock(str(package / '.benchset.publish.lock'), timeout=0): + pass + finally: + _close_orphaned(descriptors, package / '.benchset.publish.lock') diff --git a/tests/test_python_core.py b/tests/test_python_core.py new file mode 100644 index 0000000..2802967 --- /dev/null +++ b/tests/test_python_core.py @@ -0,0 +1,653 @@ +"""Cross-boundary behavior that must survive Rust-owned suite execution.""" + +import ctypes +import io +import math +import signal +import subprocess +import sys +from threading import Event, Thread, get_ident +from typing import Any, List, TextIO, Type + +import pytest + +from benchset import ( + AggregatedScores, + BenchmarkResult, + BenchmarkRunner, + Progress, + SuiteResult, + api, + cli, + run, + run_all, +) +from benchset._native import CSuiteResult, Native +from tests.helpers import assert_runner_reusable +from tests.test_semantics import ManagedNative, ResultLibrary + + +@pytest.mark.parametrize('failure', ['summary', 'free']) +def test_cli_output_abort_does_not_wait_for_a_second_backend_error(failure: str) -> None: + """After broken output, later backend failures must not park the worker again. + + A real suite delivers its first result before a summary or cleanup failure. + A child process bounds deadlocks and exposes both waiting thread stacks; + success requires exit 1, completed cleanup, and no traceback or leaked thread. + """ + program = r''' +import faulthandler +import signal +import sys +from benchset import api, cli +from benchset._native import Native + +native = Native() +if sys.argv[1] == 'summary': + native.lib.benchset_suite_result = lambda *_: 0 +else: + free = native.free + def broken_free(handle, suite=False): + free(handle, suite) + raise RuntimeError('backend cleanup failed') + native.free = broken_free +api.Native = lambda: native +seen = [] +def broken_output(result, _stream): + seen.append(result.benchmark) + raise BrokenPipeError('output closed') +cli.render = broken_output +previous = signal.getsignal(signal.SIGINT) +faulthandler.dump_traceback_later(3, exit=True) +assert cli.main(['--warmup=0', '--duration=.1']) == 1 +faulthandler.cancel_dump_traceback_later() +assert seen == ['single_core'] +assert signal.getsignal(signal.SIGINT) is previous +print('worker released') +''' + result = subprocess.run([sys.executable, '-c', program, failure], capture_output=True, + text=True, check=False, timeout=10) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout == 'worker released\n' + assert not result.stderr + + +@pytest.mark.parametrize('name', [None, 0, b'single_core', [], {}]) +def test_invalid_benchmark_types_cannot_select_a_suite(name: object) -> None: + """Untyped callers retain TypeError rather than accidentally starting a full suite.""" + with pytest.raises(TypeError): + run(name, warmup_seconds=0, measurement_seconds=0) # type: ignore[arg-type] + + +def test_numeric_conversion_keeps_priority_over_invalid_benchmark_type() -> None: + """Host float conversion still precedes string transport and native loading.""" + with pytest.raises(ValueError, match='invalid'): + run(None, warmup_seconds='invalid', measurement_seconds=0) # type: ignore[arg-type] + + +def test_busy_runner_uses_its_loaded_core_when_new_loading_fails(monkeypatch: pytest.MonkeyPatch) -> None: + """A valid losing call keeps the busy diagnostic and cannot disturb its owner.""" + runner = BenchmarkRunner() + entered, release = Event(), Event() + results = [] + + def progress(_value: Progress) -> None: + entered.set() + assert release.wait(5) + + worker = Thread(target=lambda: results.append(runner.run('single_core', warmup_seconds=0, + measurement_seconds=1, on_progress=progress))) + worker.start() + try: + assert entered.wait(5) + + def missing() -> Native: + raise OSError('loader now unavailable') + + monkeypatch.setattr(api, 'Native', missing) + contender = runner.run('multi_core', warmup_seconds=0, measurement_seconds=.01) + assert contender.reason == 'this runner is already running' + finally: + runner.cancel() + release.set() + worker.join(5) + assert not worker.is_alive() + assert results[0].status == 'cancelled' + + +@pytest.mark.parametrize('stage', ['loading', 'allocation']) +@pytest.mark.parametrize('error_type', [OSError, RuntimeError]) +def test_busy_initialization_load_failure_keeps_priority_without_releasing_owner( + monkeypatch: pytest.MonkeyPatch, stage: str, error_type: Type[Exception], +) -> None: + """Before a core is published, the agreed loader priority cannot clear the winning call.""" + backend = ManagedNative() + entered, release = Event(), Event() + calls = [] + if stage == 'allocation': + backend.delay_create = True + entered, release = backend.created, backend.release + + def factory() -> ManagedNative: + calls.append(1) + if len(calls) > 1: + raise error_type('contender loader failed') + if stage == 'loading': + entered.set() + assert release.wait(5) + return backend + + monkeypatch.setattr(api, 'Native', factory) + runner = BenchmarkRunner() + results = [] + worker = Thread(target=lambda: results.append(runner.run('single_core', warmup_seconds=0, measurement_seconds=.01))) + worker.start() + try: + assert entered.wait(5) + owner = runner._active + contender = runner.run('multi_core', warmup_seconds=0, measurement_seconds=.01) + assert contender.reason == 'contender loader failed' + assert runner._active is owner + assert backend.calls.count('free') == 0 + finally: + release.set() + worker.join(5) + assert not worker.is_alive() + assert results[0].success + assert backend.calls == ['create', 'execute', 'free'] + monkeypatch.setattr(api, 'Native', lambda: backend) + assert runner.run('single_core', warmup_seconds=0, measurement_seconds=.01).success + + +def test_cli_interrupt_during_first_result_render_stops_the_suite( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], +) -> None: + """An interrupt while printing the first completed snapshot suppresses later work and aggregate.""" + original = cli.render + names = [] + + def render(result: BenchmarkResult, stream: TextIO) -> None: + names.append(result.benchmark) + original(result, stream) + handler = signal.getsignal(signal.SIGINT) + assert callable(handler) + handler(signal.SIGINT, None) + + monkeypatch.setattr(cli, 'render', render) + assert cli.main(['--warmup=0', '--duration=.03']) == 130 + assert names == ['single_core'] + captured = capsys.readouterr() + assert 'single_core' in captured.out + # HEAD's restored default signal handler also bypassed the footer here. + assert 'Full Pi equivalent' not in captured.out + captured.err + + +@pytest.mark.parametrize('entrypoint', ['runner', 'run', 'run_all']) +@pytest.mark.parametrize(('warmup', 'measurement', 'error_type'), [ + ('invalid', 1, ValueError), (-1, 'invalid', ValueError), (0, None, TypeError), (0, 10**1000, OverflowError), + ('invalid', None, ValueError), +]) +def test_all_entrypoints_convert_both_numbers_before_validation_or_loading( + monkeypatch: pytest.MonkeyPatch, entrypoint: str, warmup: object, measurement: object, + error_type: Type[Exception], +) -> None: + """Numeric conversion errors keep precedence even with an unknown name or invalid warmup.""" + monkeypatch.setattr(api, 'Native', lambda: pytest.fail('conversion errors must precede native loading')) + def execute() -> None: + if entrypoint == 'run_all': + run_all(warmup_seconds=warmup, measurement_seconds=measurement) # type: ignore[arg-type] + else: + call = BenchmarkRunner().run if entrypoint == 'runner' else run + call('unknown', warmup_seconds=warmup, measurement_seconds=measurement) # type: ignore[arg-type] + + with pytest.raises(error_type): + execute() + + +@pytest.mark.parametrize(('warmup', 'measurement', 'reason'), [ + (math.nan, 0, 'warmup_seconds must be finite and nonnegative'), + (-1, math.nan, 'warmup_seconds must be finite and nonnegative'), + (-0.0, -0.0, 'measurement_seconds must be finite and positive'), + (0, math.inf, 'measurement_seconds must be finite and positive'), +]) +def test_suite_invalid_durations_keep_exact_diagnostics_without_allocation( + monkeypatch: pytest.MonkeyPatch, warmup: float, measurement: float, reason: str, +) -> None: + """Rust validation retains exact suite diagnostics, signed zeros and NaN without starting workers.""" + native = Native() + monkeypatch.setattr(api, 'Native', lambda: native) + monkeypatch.setattr(native, 'create', lambda *_args: pytest.fail('invalid durations must not allocate a handle')) + events: List[Progress] = [] + suite = run_all(warmup_seconds=warmup, measurement_seconds=measurement, on_progress=events.append) + assert len(suite.results) == 1 + result = suite.results[0] + assert result.reason == reason + assert result.status == 'failed' + assert not events + for actual, expected in [(result.warmup_seconds, warmup), (result.measurement_seconds, measurement)]: + if math.isnan(expected): + assert math.isnan(actual) + else: + assert actual == expected + assert math.copysign(1, actual) == math.copysign(1, expected) + + +def test_cli_invalid_warmup_precedes_invalid_measurement(capsys: pytest.CaptureFixture[str]) -> None: + """Moving validation into Rust must not swap the first actionable CLI diagnostic.""" + assert cli.main(['--warmup=-1', '--duration=0']) == 2 + output = capsys.readouterr() + assert not output.out + assert '--warmup must be finite and nonnegative' in output.err + + +def test_real_native_cli_flushes_first_result_before_second_progress(monkeypatch: pytest.MonkeyPatch) -> None: + """Actual sequencing waits for main-thread printing and flushing of each prior snapshot. + + Hold the first flush behind an event so prematurely acknowledging a result + cannot escape detection merely because printing happens to win a race. + """ + writers = [] + first_flush, allow_flush, second_progress = Event(), Event(), Event() + premature = [] + + class Stream(io.StringIO): + flushed = '' + + def write(self, text: str) -> int: + writers.append(get_ident()) + return super().write(text) + + def flush(self) -> None: + if not first_flush.is_set(): + first_flush.set() + assert allow_flush.wait(5) + self.flushed = self.getvalue() + + output = Stream() + checked = [] + + class ProgressProbe(cli._Progress): + def __call__(self, value: Progress) -> None: + if value.benchmark == 'multi_core': + second_progress.set() + assert 'single_core' in output.flushed + assert 'Samples:' in output.flushed + checked.append(1) + + def clear(self) -> None: + pass + + monkeypatch.setattr(cli.sys, 'stdout', output) + monkeypatch.setattr(cli, '_Progress', ProgressProbe) + + def observe() -> None: + assert first_flush.wait(5) + premature.append(second_progress.wait(.05)) + allow_flush.set() + + observer = Thread(target=observe, daemon=True) + observer.start() + try: + status = cli.main(['--warmup=0', '--duration=.05']) + finally: + first_flush.set() + allow_flush.set() + observer.join(5) + assert status == 0 + assert premature == [False] + assert not observer.is_alive() + assert checked + assert writers + assert set(writers) == {get_ident()} + + +def test_real_native_cli_render_failure_releases_blocked_observer( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], +) -> None: + """A failed intermediate write cancels, releases and joins the blocked native callback before returning.""" + names = [] + native = Native() + original_cancel = native.cancel + cancellations = [] + + def cancel(handle: int, suite: bool = False) -> None: + cancellations.append(suite) + original_cancel(handle, suite) + + class ProgressProbe: + def __call__(self, value: Progress) -> None: + names.append(value.benchmark) + + def clear(self) -> None: + pass + + def broken(_result: BenchmarkResult, _stream: TextIO) -> None: + raise OSError('intermediate render failed') + + monkeypatch.setattr(cli, '_Progress', ProgressProbe) + monkeypatch.setattr(cli, 'render', broken) + monkeypatch.setattr(api, 'Native', lambda: native) + monkeypatch.setattr(native, 'cancel', cancel) + assert cli.main(['--warmup=0', '--duration=.05']) == 1 + assert names + assert set(names) == {'single_core'} + assert cancellations + assert all(cancellations) + assert 'intermediate render failed' in capsys.readouterr().err + assert_runner_reusable(BenchmarkRunner()) + + +@pytest.mark.parametrize(('name', 'warmup', 'measurement', 'reason'), [ + ('unknown', -1, 0, 'unknown benchmark: unknown'), + ('single_core', -1, 0, 'warmup_seconds must be finite and nonnegative'), + ('single_core', 0, 0, 'measurement_seconds must be finite and positive'), +]) +def test_reentrant_invalid_request_keeps_validation_precedence( + name: str, warmup: float, measurement: float, reason: str, +) -> None: + """Invalid reentry reports the same option error without releasing the owner.""" + runner = BenchmarkRunner() + failures = [] + + def callback(_progress: Progress) -> None: + failures.append(runner.run(name, warmup_seconds=warmup, measurement_seconds=measurement)) + runner.cancel() + + result = runner.run('single_core', warmup_seconds=0, measurement_seconds=1, on_progress=callback) + assert result.status == 'cancelled' + assert failures + assert all(item.reason == reason for item in failures) + assert_runner_reusable(runner) + + +@pytest.mark.parametrize('benchmark', ['single_core', 'multi_core']) +@pytest.mark.parametrize('phase', ['measurement', 'completed']) +@pytest.mark.parametrize('error_type', [ValueError, KeyboardInterrupt, SystemExit, GeneratorExit]) +def test_suite_callback_failure_stops_notifications_and_keeps_completed_prefix( + benchmark: str, phase: str, error_type: Type[BaseException], +) -> None: + """A callback failure keeps prior snapshots, stops later work and permits reuse. + + Failing in a terminal callback catches implementations that accidentally + publish the native completed score after the host callback has failed. + """ + events: List[Progress] = [] + + def callback(value: Progress) -> None: + events.append(value) + if value.benchmark == benchmark and value.phase == phase: + raise error_type('suite callback failed') + + suite = run_all(warmup_seconds=0, measurement_seconds=.1, on_progress=callback) + assert suite.results[-1].benchmark == benchmark + assert suite.results[-1].status == 'failed' + assert suite.results[-1].reason == f'progress callback failed: {error_type.__name__}: suite callback failed' + assert suite.results[-1].score is suite.results[-1].pi_equivalent is None + assert suite.aggregated_scores.pi_equivalent is None + assert events[-1].benchmark == benchmark + assert events[-1].phase == phase + if benchmark == 'multi_core': + assert len(suite.results) == 2 + assert suite.results[0].success + else: + assert len(suite.results) == 1 + assert_runner_reusable(BenchmarkRunner()) + + +@pytest.mark.parametrize('trigger', ['error', 'cancel']) +def test_suite_cancellation_targets_its_owned_native_handle( + monkeypatch: pytest.MonkeyPatch, trigger: str, +) -> None: + """Both a callback error and explicit cancellation must actually cancel the owned suite token.""" + native = Native() + original = native.cancel + calls = [] + + def cancel(handle: int, suite: bool = False) -> None: + calls.append((handle, suite)) + original(handle, suite) + + monkeypatch.setattr(native, 'cancel', cancel) + monkeypatch.setattr(api, 'Native', lambda: native) + runner = BenchmarkRunner() + requested = False + + def progress(_value: Progress) -> None: + nonlocal requested + if not requested: + requested = True + if trigger == 'error': + raise ValueError('cancel native suite') + runner.cancel() + + result = runner._run_all(0, .03, progress) + assert calls + assert all(mode is True for _, mode in calls) + assert len({handle for handle, _ in calls}) == 1 + assert result.results[-1].status == ('failed' if trigger == 'error' else 'cancelled') + + +@pytest.mark.parametrize('entrypoint', ['runner', 'run', 'run_all']) +@pytest.mark.parametrize('error_type', [OSError, RuntimeError]) +def test_loading_failure_has_priority_over_invalid_arguments( + monkeypatch: pytest.MonkeyPatch, entrypoint: str, error_type: Type[Exception], +) -> None: + """The agreed loading-first contract retains ordinary failed result objects.""" + def missing() -> Native: + raise error_type('native unavailable') + + monkeypatch.setattr(api, 'Native', missing) + if entrypoint == 'run_all': + suite = run_all(warmup_seconds=-1, measurement_seconds=0) + assert len(suite.results) == 1 + assert suite.aggregated_scores.pi_equivalent is None + result = suite.results[0] + else: + execute = BenchmarkRunner().run if entrypoint == 'runner' else run + result = execute('unknown', warmup_seconds=-1, measurement_seconds=0) + assert result.status == 'failed' + assert result.reason == 'native unavailable' + assert result.score is result.pi_equivalent is None + + +@pytest.mark.parametrize('error_type', [OSError, RuntimeError]) +def test_cli_loader_failure_is_rendered_without_a_traceback( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], error_type: Type[Exception], +) -> None: + """The agreed CLI load-error priority overrides invalid durations without a traceback.""" + def missing() -> Native: + raise error_type('native library version/ABI or symbol unavailable') + + monkeypatch.setattr(cli, 'Native', missing) + monkeypatch.setattr(api, 'Native', missing) + assert cli.main(['--warmup=-1', '--duration=0']) == 1 + output = capsys.readouterr() + assert not output.out + assert 'native library version/ABI or symbol unavailable' in output.err + assert 'Full Pi equivalent : —' in output.err + assert 'Traceback' not in output.err + + +@pytest.mark.parametrize('operation', ['run', 'aggregate']) +def test_string_subclasses_do_not_control_native_name_encoding(operation: str) -> None: + """Keep accepting str subclasses without calling a newly introduced encoding hook. + + Name transport must read the string value as the original ID lookup did; + subclass encode methods must not replace validation errors or aggregation. + """ + class Name(str): + __slots__ = () + + def encode(self, encoding: str = 'utf-8', errors: str = 'strict') -> bytes: + raise AssertionError(f'the original API never called encode({encoding!r}, {errors!r})') + + name = Name('single_core') + if operation == 'run': + result = run(name, warmup_seconds=-1, measurement_seconds=0) + assert result.benchmark is name + assert result.reason == 'warmup_seconds must be finite and nonnegative' + else: + results = (BenchmarkResult(name, status='completed', pi_equivalent=.5), + BenchmarkResult(Name('multi_core'), status='completed', pi_equivalent=2)) + suite = SuiteResult.from_results(results) + assert suite.results is results + assert suite.aggregated_scores.pi_equivalent == 1 + + +@pytest.mark.parametrize('name', ['много_ядер', 'single_core\x00suffix', '🦀', '\ud800']) +def test_python_transports_native_registry_ids_and_utf8_byte_lengths(name: str) -> None: + """A binding forwards native IDs and complete encoded names without its own registry.""" + native = Native() + library = ResultLibrary(status=0, reason=None, has_variation=0) + calls = [] + + def lookup(name: bytes, size: int) -> int: + calls.append((name, size)) + return 73 + + library.benchset_benchmark_id = lookup # type: ignore[method-assign] + native.lib = library # type: ignore[assignment] + result = native.execute(123, name, 0, .01, None) + encoded = name.encode('utf-8', errors='surrogatepass') + assert calls == [(encoded, len(encoded))] + assert library.config is not None + assert library.config.benchmark == 73 + assert result.benchmark == name + + +@pytest.mark.parametrize('fail_at', [1, 2]) +@pytest.mark.parametrize('getter', ['benchset_result_v2', 'benchset_sample']) +def test_suite_child_decoder_failure_keeps_prefix_and_frees_once( + monkeypatch: pytest.MonkeyPatch, fail_at: int, getter: str, +) -> None: + """A missing child result stops the suite without losing earlier snapshots or leaking its handle.""" + native = Native() + original_result, original_free = getattr(native.lib, getter), native.lib.benchset_suite_free + calls = [] + frees = [] + + def result(*args: object) -> int: + calls.append(1) + return 0 if len(calls) == fail_at else int(original_result(*args)) + + def free(handle: int) -> None: + frees.append(handle) + original_free(handle) + + monkeypatch.setattr(native.lib, getter, result) + monkeypatch.setattr(native.lib, 'benchset_suite_free', free) + monkeypatch.setattr(api, 'Native', lambda: native) + suite = run_all(warmup_seconds=0, measurement_seconds=.03) + assert len(suite.results) == fail_at + assert all(item.success for item in suite.results[:-1]) + assert suite.results[-1].status == 'failed' + assert suite.results[-1].reason == ('native sample is unavailable' if getter == 'benchset_sample' + else 'native library did not return a result') + assert suite.aggregated_scores.pi_equivalent is None + assert len(frees) == 1 + + +def test_cli_suite_getter_failure_after_completed_children_is_not_success( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], +) -> None: + """A missing final suite snapshot cannot be hidden by previously streamed completed children.""" + native = Native() + original_free = native.lib.benchset_suite_free + frees = [] + + def free(handle: int) -> None: + frees.append(handle) + original_free(handle) + + monkeypatch.setattr(native.lib, 'benchset_suite_result', lambda *_args: 0) + monkeypatch.setattr(native.lib, 'benchset_suite_free', free) + monkeypatch.setattr(api, 'Native', lambda: native) + assert cli.main(['--warmup=0', '--duration=.03']) == 1 + captured = capsys.readouterr() + assert 'single_core' in captured.out + assert 'native library did not return a suite result' in captured.err + assert 'Full Pi equivalent : —' in captured.err + assert len(frees) == 1 + + +@pytest.mark.parametrize('presence', [0, 7, 2**32 - 1]) +@pytest.mark.parametrize('equivalent', [0.0, 1e-300, 1.2345678901234567]) +def test_suite_forwards_native_aggregate_without_recalculation( + monkeypatch: pytest.MonkeyPatch, presence: int, equivalent: float, +) -> None: + """Native aggregate values, including zero and noncanonical presence flags, cross the wrapper losslessly.""" + native = Native() + original = native.lib.benchset_suite_result + + def getter(handle: int, output: Any, size: int) -> int: + status = int(original(handle, output, size)) + value = ctypes.cast(output, ctypes.POINTER(CSuiteResult)).contents + value.aggregated_scores.has_pi_equivalent = presence + value.aggregated_scores.pi_equivalent = equivalent + return status + + monkeypatch.setattr(native.lib, 'benchset_suite_result', getter) + monkeypatch.setattr(api, 'Native', lambda: native) + suite = run_all(warmup_seconds=0, measurement_seconds=.03) + assert len(suite.results) == 2 + actual = suite.aggregated_scores.pi_equivalent + if presence: + assert actual is not None + assert actual.hex() == equivalent.hex() + else: + assert actual is None + + +def test_suite_rejects_mismatched_native_snapshot_count(monkeypatch: pytest.MonkeyPatch) -> None: + """A summary whose count disagrees with delivered child snapshots cannot become partial success.""" + native = Native() + original = native.lib.benchset_suite_result + + def getter(handle: int, output: Any, size: int) -> int: + status = int(original(handle, output, size)) + ctypes.cast(output, ctypes.POINTER(CSuiteResult)).contents.result_count += 1 + return status + + monkeypatch.setattr(native.lib, 'benchset_suite_result', getter) + monkeypatch.setattr(api, 'Native', lambda: native) + suite = run_all(warmup_seconds=0, measurement_seconds=.03) + assert suite.results[-1].status == 'failed' + assert suite.results[-1].reason == 'native suite results are unavailable' + assert suite.aggregated_scores.pi_equivalent is None + + +@pytest.mark.parametrize('interrupted', [False, True]) +def test_cli_rejects_a_prefix_only_backend_and_restores_worker_ownership( + monkeypatch: pytest.MonkeyPatch, interrupted: bool, +) -> None: + """A prefix-only backend cannot report success or leak a worker; late Ctrl+C retains precedence.""" + workers = [] + previous = signal.getsignal(signal.SIGINT) + + class Worker(Thread): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + workers.append(self) + + def join(self, *args: Any, **kwargs: Any) -> None: + if interrupted: + signal.raise_signal(signal.SIGINT) + super().join(*args, **kwargs) + + def malformed( + _runner: BenchmarkRunner, _warmup: float, _measurement: float, _progress: object, on_result: Any, + ) -> SuiteResult: + result = BenchmarkResult('single_core', status='completed', score=1, pi_equivalent=1) + assert not on_result(result) + return SuiteResult((result,), AggregatedScores()) + + monkeypatch.setattr(BenchmarkRunner, '_run_all', malformed) + monkeypatch.setattr(cli, 'Thread', Worker) + if interrupted: + assert cli.main(['--warmup=0', '--duration=.02']) == 130 + else: + with pytest.raises(RuntimeError, match='native execution did not return a result'): + cli.main(['--warmup=0', '--duration=.02']) + assert workers + assert all(not worker.is_alive() for worker in workers) + assert signal.getsignal(signal.SIGINT) is previous diff --git a/tests/test_quiet_validation.py b/tests/test_quiet_validation.py new file mode 100644 index 0000000..6ce4399 --- /dev/null +++ b/tests/test_quiet_validation.py @@ -0,0 +1,457 @@ +"""Reject invalid numeric snapshots without disturbing the caller's FPU state.""" + +import ctypes +import gc +import math +import os +import shutil +import struct +import subprocess +import sys +from contextlib import contextmanager +from typing import Any, Iterator, List, Tuple + +import pytest + +from benchset import ( + BenchmarkResult, + BenchmarkRunner, + SuiteResult, + _native, + api, + run_all, +) + +_BITS = ( + 0x7ff0000000000001, 0xfff0000000000001, + 0x7ff8000000000001, 0xfff8000000000001, + 0x7ff0000000000000, 0xfff0000000000000, +) + +_ENVIRONMENT = '''#include +static fenv_t saved; +int begin(int seed) { + if (fegetenv(&saved) || feclearexcept(FE_ALL_EXCEPT)) return -1; + return seed ? feraiseexcept(FE_DIVBYZERO) : 0; +} +int flags(void) { return fetestexcept(FE_ALL_EXCEPT); } +int rounding(void) { return fegetround(); } +int end(void) { return fesetenv(&saved); } +''' + + +@pytest.fixture(scope='module') +def quiet_environment(tmp_path_factory: pytest.TempPathFactory) -> Any: + """Read real platform exception flags and restore the complete FPU environment.""" + if os.name == 'nt': + pytest.skip('POSIX fenv probe; MSVC floating environment differs') + compiler = shutil.which('cc') or shutil.which('clang') or shutil.which('gcc') + if compiler is None: + pytest.skip('No C compiler for floating-environment validation') + directory = tmp_path_factory.mktemp('quiet-validation') + source = directory / 'environment.c' + source.write_text(_ENVIRONMENT) + library = directory / 'environment.so' + built = subprocess.run([compiler, '-shared', '-fPIC', str(source), '-lm', '-o', str(library)], + capture_output=True, text=True, check=False) + assert built.returncode == 0, built.stderr + probe = ctypes.CDLL(str(library)) + probe.begin.argtypes = [ctypes.c_int] + probe.begin.restype = ctypes.c_int + for name in ('flags', 'rounding', 'end'): + getattr(probe, name).argtypes = [] + getattr(probe, name).restype = ctypes.c_int + # Import/ctypes initialization is outside every measured validation window. + _native.Native() + SuiteResult.from_results(tuple(BenchmarkResult(name, status='completed', pi_equivalent=1.0) + for name in ('single_core', 'multi_core'))) + return probe + + +@contextmanager +def quiet_flags(probe: Any, seed: bool) -> Iterator[int]: + # CPython 3.15 GC statistics set FE_INEXACT independently of validation. + collecting = gc.isenabled() + gc.disable() + before = probe.rounding(), probe.flags() + try: + assert probe.begin(seed) == 0 + yield probe.flags() + finally: + try: + assert probe.end() == 0 + assert (probe.rounding(), probe.flags()) == before + finally: + if collecting: + gc.enable() + + +def number(bits: int) -> float: + return struct.unpack('=d', struct.pack('=Q', bits))[0] + + +@pytest.mark.parametrize('bits', _BITS, ids=['snan', 'negative-snan', 'qnan', 'negative-qnan', 'inf', 'negative-inf']) +@pytest.mark.parametrize('position', [0, 1]) +@pytest.mark.parametrize('seed', [False, True]) +def test_aggregate_rejects_nonfinite_values_without_new_fpu_flags( + quiet_environment: Any, bits: int, position: int, seed: bool, +) -> None: + """Both coefficient positions quietly reject every NaN/infinity encoding. + + Signaling NaNs must not reach speculative floating comparisons. Retaining + a preexisting divide-by-zero flag also rejects implementations that clear + the caller's flags instead of avoiding the invalid operation. + """ + values = [1.0, 1.0] + values[position] = number(bits) + results = tuple(BenchmarkResult(name, status='completed', pi_equivalent=value) + for name, value in zip(('single_core', 'multi_core'), values)) + with quiet_flags(quiet_environment, seed) as initial: + result = SuiteResult.from_results(results) + flags = quiet_environment.flags() + assert result.aggregated_scores.pi_equivalent is None + assert flags == initial + + +@pytest.mark.parametrize('bits', [*_BITS, 0xbff0000000000000]) +@pytest.mark.parametrize('window', ['warmup', 'measurement']) +@pytest.mark.parametrize('seed', [False, True]) +def test_duration_validation_rejects_invalid_values_without_new_fpu_flags( + quiet_environment: Any, bits: int, window: str, seed: bool, +) -> None: + """Rust validation classifies invalid durations before any floating comparison. + + Warmup keeps priority over an independently invalid measurement; rejected + names likewise precede duration inspection. No new flag may replace an + existing caller flag, including when the input is a signaling NaN. + """ + native = _native.Native() + warmup, measurement = (number(bits), 0.0) if window == 'warmup' else (0.0, number(bits)) + expected = 2 if window == 'warmup' else 3 + with quiet_flags(quiet_environment, seed) as initial: + code = native.lib.benchset_duration_error(native.double(warmup), native.double(measurement)) + flags = quiet_environment.flags() + unknown = native.lib.benchset_validation_error(0, native.double(warmup), native.double(measurement)) + unknown_flags = quiet_environment.flags() + assert code == expected + assert unknown == 1 + assert flags == unknown_flags == initial + + +@pytest.mark.parametrize('bits', _BITS) +@pytest.mark.parametrize('window', ['warmup', 'measurement']) +@pytest.mark.parametrize('suite', [False, True]) +def test_public_run_rejects_nonfinite_windows_quietly_and_preserves_requested_bits( + quiet_environment: Any, monkeypatch: pytest.MonkeyPatch, bits: int, window: str, suite: bool, +) -> None: + """Public Python entry points return the old failure and original duration bits. + + No worker/progress may start for a rejected window. The adapter must not + quiet or replace the caller's NaN just to avoid a hardware exception. + """ + progress: List[Any] = [] + options = {'warmup_seconds': number(bits) if window == 'warmup' else 0.0, + 'measurement_seconds': number(bits) if window == 'measurement' else 1.0} + runner = BenchmarkRunner() + monkeypatch.setattr(api, 'BenchmarkRunner', lambda: runner) + with quiet_flags(quiet_environment, True) as initial: + outcome = (run_all(on_progress=progress.append, **options) if suite else + runner.run('single_core', on_progress=progress.append, **options)) + flags = quiet_environment.flags() + result = outcome.results[0] if isinstance(outcome, SuiteResult) else outcome + assert result.status == 'failed' + expected = 'warmup_seconds must be finite and nonnegative' if window == 'warmup' else ( + 'measurement_seconds must be finite and positive' + ) + assert result.reason == expected + actual = result.warmup_seconds if window == 'warmup' else result.measurement_seconds + assert struct.unpack('=Q', struct.pack('=d', actual))[0] == bits + assert flags == initial + assert not progress + assert runner._active is None + assert runner._native is None + assert runner._handle is None + + +@pytest.mark.parametrize('value', [-1.0, -.25, -sys.float_info.max]) +@pytest.mark.parametrize('position', [0, 1]) +@pytest.mark.parametrize('seed', [False, True]) +def test_negative_host_log_domain_preserves_python_flags_and_zero_comparison_operand( + quiet_environment: Any, value: float, position: int, seed: bool, +) -> None: + """A custom accepted negative ratio raises Python's domain error without a cast. + + The log primitive's floating operand must never be converted to an unsigned + status field. Only ge/eq need that field, and their operand is integer zero. + """ + calls: List[Tuple[str, object]] = [] + + class Ratio(float): + __slots__ = () + __hash__ = float.__hash__ + + def __ge__(self, other: object) -> bool: + calls.append(('ge', other)) + return True + + def __eq__(self, other: object) -> bool: + calls.append(('eq', other)) + return False + + values = [1.0, 1.0] + values[position] = Ratio(value) + results = tuple(BenchmarkResult(name, status='completed', pi_equivalent=ratio) + for name, ratio in zip(('single_core', 'multi_core'), values)) + with quiet_flags(quiet_environment, seed): + with pytest.raises(ValueError, match=r'.') as expected: + math.log(value) + expected_flags = quiet_environment.flags() + with quiet_flags(quiet_environment, seed): + with pytest.raises(ValueError, match=r'.') as actual: + SuiteResult.from_results(results) + actual_flags = quiet_environment.flags() + assert str(actual.value) == str(expected.value) + assert actual_flags == expected_flags + assert calls == [('ge', 0), ('eq', 0)] + assert all(type(operand) is int for _name, operand in calls) + + +@pytest.mark.parametrize('bits', [0, 0x8000000000000000, 0x7fefffffffffffff, 1]) +def test_finite_controls_and_negative_zero_remain_accepted_by_duration_validation( + quiet_environment: Any, bits: int, +) -> None: + """Bit classification must keep finite endpoints and warmup negative zero valid. + + These values need not be representable as Rust Duration: the validation API + deliberately checks the public numeric range independently of conversion. + """ + native = _native.Native() + warmup, measurement = native.double(number(bits)), native.double(1.0) + with quiet_flags(quiet_environment, True) as initial: + code = native.lib.benchset_duration_error(warmup, measurement) + flags = quiet_environment.flags() + assert code == 0 + assert flags == initial + + +@pytest.mark.parametrize('bits', [0, 0x8000000000000000]) +@pytest.mark.parametrize('position', [0, 1]) +def test_aggregate_zero_remains_present_positive_zero_without_new_flags( + quiet_environment: Any, bits: int, position: int, +) -> None: + """Quiet sign checks must accept either zero sign and suppress all logarithms.""" + values = [1.0, 1.0] + values[position] = number(bits) + results = tuple(BenchmarkResult(name, status='completed', pi_equivalent=value) + for name, value in zip(('single_core', 'multi_core'), values)) + with quiet_flags(quiet_environment, True) as initial: + result = SuiteResult.from_results(results).aggregated_scores.pi_equivalent + flags = quiet_environment.flags() + assert result is not None + assert struct.unpack('=Q', struct.pack('=d', result))[0] == 0 + assert flags == initial + + +_C_CONSUMER = r''' +#include "benchset.h" +#include +#include +#include +#include +#include +#include +static uint64_t bits(double value) { uint64_t result; memcpy(&result, &value, 8); return result; } +static double value(uint64_t raw) { double result; memcpy(&result, &raw, 8); return result; } +static void single(BenchsetConfig *config, int seed, const char *reason) { + BenchsetHandle *handle = benchset_create(); assert(handle); + assert(begin(seed) == 0); int previous = flags(); + assert(benchset_run(handle, config, NULL, NULL) == BENCHSET_FAILED); + BenchsetResult result; assert(benchset_result(handle, &result) == 1); + assert(strcmp(benchset_reason(handle), reason) == 0); + assert(bits(result.warmup_seconds) == bits(config->warmup_seconds)); + assert(bits(result.measurement_seconds) == bits(config->measurement_seconds)); + int raised = flags(); assert(end() == 0); benchset_free(handle); + assert(raised == previous); +} +static void suite(BenchsetConfig *config, int seed) { + BenchsetSuiteHandle *handle = benchset_suite_create(); assert(handle); + assert(begin(seed) == 0); int previous = flags(); + assert(benchset_run_all(handle, config, NULL, NULL) == BENCHSET_FAILED); + BenchsetSuiteResult result; assert(benchset_suite_result(handle, &result, sizeof result) == 1); + assert(result.result_count == 1 && result.results[0].base.base.status == BENCHSET_FAILED); + assert(bits(result.results[0].base.base.warmup_seconds) == bits(config->warmup_seconds)); + assert(bits(result.results[0].base.base.measurement_seconds) == bits(config->measurement_seconds)); + assert(strcmp(benchset_suite_reason(handle, 0), "durations must be finite: warmup >= 0, measurement > 0") == 0); + int raised = flags(); assert(end() == 0); benchset_suite_free(handle); + assert(raised == previous); +} +typedef double (*NumberReader)(void *); +typedef int32_t (*CopyNumber)(NumberReader, void *, double *); +static unsigned reads; +static double read_number(void *user) { + ++reads; + uint64_t raw = UINT64_C(0x8000000000000000); + if (user) memcpy(&raw, user, sizeof raw); + return value(raw); +} +static void copy_number(const char *input, int seed) { + CopyNumber copy = (CopyNumber)dlsym(RTLD_DEFAULT, "benchset_copy_double"); + assert(copy != NULL); + uint64_t raw = strtoull(input, NULL, 16); + struct { uint64_t before; double out; uint64_t after; } guarded = { + UINT64_C(0x1122334455667788), 17, UINT64_C(0x8877665544332211)}; + reads = 0; + assert(begin(seed) == 0); int previous = flags(); + assert(copy(NULL, &raw, &guarded.out) == 0 && reads == 0 && guarded.out == 17); + assert(copy(read_number, &raw, NULL) == 0 && reads == 0); + assert(copy(read_number, &raw, &guarded.out) == 1 && reads == 1); + assert(bits(guarded.out) == raw); + assert(copy(read_number, NULL, &guarded.out) == 1 && reads == 2); + assert(bits(guarded.out) == UINT64_C(0x8000000000000000)); + double aliased = value(raw); + assert(copy(read_number, &aliased, &aliased) == 1 && reads == 3 && bits(aliased) == raw); + int raised = flags(); assert(end() == 0); assert(raised == previous); + assert(guarded.before == UINT64_C(0x1122334455667788)); + assert(guarded.after == UINT64_C(0x8877665544332211)); +} +int main(int argc, char **argv) { + unsigned cases = 0; + for (int arg = 1; arg < argc; ++arg) { + uint64_t raw = strtoull(argv[arg], NULL, 16); + copy_number(argv[arg], 0); copy_number(argv[arg], 1); + for (int seed = 0; seed != 2; ++seed) for (int window = 0; window != 2; ++window) { + BenchsetConfig config; assert(benchset_default_config(&config) == 1); + config.warmup_seconds = window ? 0 : value(raw); + config.measurement_seconds = window ? value(raw) : 1; + assert(begin(seed) == 0); int previous = flags(); + assert(benchset_duration_error(config.warmup_seconds, config.measurement_seconds) == (window ? 3 : 2)); + assert(benchset_validation_error(0, config.warmup_seconds, config.measurement_seconds) == 1); + int raised = flags(); assert(end() == 0); assert(raised == previous); + single(&config, seed, "durations must be finite: warmup >= 0, measurement > 0"); + suite(&config, seed); + config.benchmark = UINT32_MAX; config.reserved = 7; + single(&config, seed, "unknown benchmark"); + config.benchmark = 1; + single(&config, seed, "reserved config field must be zero"); + BenchsetAggregateInput inputs[] = {{1,0,1,1},{2,0,1,1}}; + for (unsigned position = 0; position != 2; ++position) { + inputs[position].pi_equivalent = value(raw); + assert(begin(seed) == 0); previous = flags(); + BenchsetAggregatedScores out = {7,7}; + assert(benchset_aggregate(inputs, 2, &out) == 1 && !out.has_pi_equivalent); + raised = flags(); assert(end() == 0); assert(raised == previous); + inputs[position].pi_equivalent = 1; + } + ++cases; + } + } + printf("%u C validation/aggregate matrices passed\n", cases); +} +''' + +_RUST_CONSUMER = r''' +use benchset::{Benchmark, BenchmarkResult, Status, SuiteResult, PI_REFERENCE_MACHINE, PI_REFERENCE_RUN}; +extern "C" { fn begin(seed: i32) -> i32; fn flags() -> i32; fn end() -> i32; } +fn result(benchmark: Benchmark, value: f64) -> BenchmarkResult { + BenchmarkResult { benchmark, version:"test", status:Status::Completed, reason:String::new(), + score:None,workers:1,warmup_seconds:0.0,measurement_seconds:1.0,elapsed_seconds:1.0, + iterations:1,checksum:0,samples:vec![],variation_percent:None,unstable:false, + pi_equivalent:Some(value),reference_machine:&PI_REFERENCE_MACHINE,reference_run:&PI_REFERENCE_RUN } +} +fn main() { + for raw in std::env::args().skip(1) { + let bits=u64::from_str_radix(&raw,16).unwrap(); + for seed in [0,1] { for position in 0..2 { + let mut values=vec![result(Benchmark::SingleCore,1.0),result(Benchmark::MultiCore,1.0)]; + values[position].pi_equivalent=Some(f64::from_bits(bits)); + assert_eq!(unsafe {begin(seed)},0); let previous=unsafe {flags()}; + let output=SuiteResult::new(values).aggregated_scores.pi_equivalent; + let raised=unsafe {flags()}; assert_eq!(unsafe {end()},0); + assert!(output.is_none()); assert_eq!(raised,previous,"input {bits:x}"); + }} + } +} +''' + + +@pytest.fixture(scope='module') +def quiet_consumers(tmp_path_factory: pytest.TempPathFactory) -> Any: + """Compile C/Rust consumers without executing the potentially broken adapter.""" + from pathlib import Path # noqa: PLC0415 — Consumer-only filesystem context. + + if os.name == 'nt': + pytest.skip('POSIX fenv probe; MSVC floating environment differs') + root = Path(__file__).resolve().parents[1] + directory = tmp_path_factory.mktemp('quiet-consumers') + compiler = shutil.which('cc') or shutil.which('clang') or shutil.which('gcc') + if compiler is None: + pytest.skip('No C compiler for floating-environment validation') + helper = directory / 'environment.c' + helper.write_text(_ENVIRONMENT) + source = directory / 'consumer.c' + source.write_text(_ENVIRONMENT + _C_CONSUMER) + executable = directory / 'consumer' + library = root / 'benchset/_native' + built = subprocess.run([compiler, '-std=c11', '-fno-builtin', str(source), '-I', str(root / 'include'), + '-L', str(library), '-lbenchset', '-lm', '-Wl,-rpath,' + str(library), + *(['-ldl'] if sys.platform != 'darwin' else []), '-o', str(executable)], capture_output=True, text=True, check=False) + assert built.returncode == 0, built.stdout + built.stderr + helper_object = directory / 'environment.o' + built = subprocess.run([compiler, '-c', str(helper), '-o', str(helper_object)], + capture_output=True, text=True, check=False) + assert built.returncode == 0, built.stdout + built.stderr + return root, directory, executable, helper_object + + +def test_public_c_run_validation_and_aggregate_reject_invalid_values_quietly(quiet_consumers: Any) -> None: + """Real C entry points reject runtime NaN/inf/negative payloads before conversion. + + The consumer checks generic one-read double copying (including NULLs and + aliasing), both run modes, both windows, requested-bit preservation, + unknown/reserved precedence, raw snapshot aggregation and preexisting flags. + Runtime argv payloads prevent compile-time constant folding from masking a + speculative comparison in an optimized Rust library. + """ + _root, _directory, executable, _helper = quiet_consumers + completed = subprocess.run([str(executable), *(format(bits, 'x') for bits in (*_BITS, 0xbfd0000000000000))], + capture_output=True, text=True, check=False, timeout=30) + assert completed.returncode == 0, completed.stdout + completed.stderr + assert '28 C validation/aggregate matrices passed' in completed.stdout + + +@pytest.mark.parametrize('optimized', [False, True], ids=['debug', 'optimized']) +def test_public_rust_runtime_aggregate_rejects_nonfinite_values_without_flags( + quiet_consumers: Any, tmp_path: Any, optimized: bool, +) -> None: + """Downstream Rust keeps the new quiet contract with debug and optimized rlibs. + + Values arrive through argv rather than constants, and a C helper observes + hardware flags around only aggregation. This catches LLVM speculation that + can differ between generic scalar and callback-backed implementations. + """ + from scripts.build_backend import rust_environment # noqa: PLC0415 + from scripts.version import read_version # noqa: PLC0415 + + root, _directory, _executable, helper = quiet_consumers + architecture = 'aarch64' if os.uname().machine in ('arm64', 'aarch64') else 'x86_64' + target = architecture + ('-apple-darwin' if sys.platform == 'darwin' else '-unknown-linux-gnu') + environment = rust_environment(root, target) + environment['CARGO_PKG_VERSION'] = read_version(root) + compiler = shutil.which('rustc', path=environment.get('PATH')) + assert compiler is not None + library = tmp_path / 'libbenchset.rlib' + profile = ['-C', 'opt-level=3' if optimized else 'opt-level=0'] + built = subprocess.run([compiler, '--edition=2021', '--crate-name=benchset', '--crate-type=rlib', + str(root / 'native/src/lib.rs'), *profile, '-o', str(library)], env=environment, + capture_output=True, text=True, check=False, timeout=120) + assert built.returncode == 0, built.stdout + built.stderr + source = tmp_path / 'consumer.rs' + source.write_text(_RUST_CONSUMER) + executable = tmp_path / 'consumer' + built = subprocess.run([compiler, '--edition=2021', str(source), '--extern', 'benchset=' + str(library), + '-C', 'link-arg=' + str(helper), *profile, '-o', str(executable)], env=environment, + capture_output=True, text=True, check=False, timeout=120) + assert built.returncode == 0, built.stdout + built.stderr + completed = subprocess.run([str(executable), *(format(bits, 'x') for bits in (*_BITS, 0xbfd0000000000000))], + capture_output=True, text=True, check=False, timeout=30) + assert completed.returncode == 0, completed.stdout + completed.stderr diff --git a/tests/test_reference.py b/tests/test_reference.py index 07dcf5d..baec9c6 100644 --- a/tests/test_reference.py +++ b/tests/test_reference.py @@ -40,7 +40,7 @@ run_all, ) from benchset._native import Native -from tests.helpers import assert_result_references +from tests.helpers import assert_result_references, mock_suite from tests.test_semantics import CallbackLibrary, ManagedNative, ResultLibrary REFERENCE_SCORES = { @@ -246,6 +246,7 @@ def test_calibration_imports_and_serialization_do_not_load_native_code() -> None import ctypes import json from dataclasses import asdict +load_native = ctypes.CDLL def forbidden(*args, **kwargs): raise AssertionError('calibration import must not load native code') ctypes.CDLL = forbidden @@ -258,6 +259,7 @@ def forbidden(*args, **kwargs): assert isinstance(PI_REFERENCE_RUN.results[0], ReferenceBenchmarkResult) assert PI_REFERENCE_RUN.started_at.utcoffset().total_seconds() == 0 assert json.loads(json.dumps(asdict(PI_REFERENCE_RUN)))['warmup_seconds'] == 1248 +ctypes.CDLL = load_native results = [BenchmarkResult('single_core'), run('unknown'), BenchmarkRunner().run('multi_core', warmup_seconds=-1)] results.extend(run_all(measurement_seconds=0).results) @@ -587,7 +589,8 @@ def execute(benchmark: str, *, warmup_seconds: float, measurement_seconds: float return BenchmarkResult(benchmark, status=current_status, score=123.5 if current_status == 'completed' else None, pi_equivalent=ratio if current_status == 'completed' else None) - monkeypatch.setattr(api, 'run', execute) + monkeypatch.setattr(BenchmarkRunner, 'run', lambda _self, name, **options: execute(name, **options)) + mock_suite(monkeypatch) results = run_all(warmup_seconds=0.25, measurement_seconds=0.75, on_progress=callback).results for result in results: assert_result_references(result) @@ -633,6 +636,7 @@ def execute(_self: BenchmarkRunner, name: str, **_options: Any) -> BenchmarkResu unstable=status == 'completed') monkeypatch.setattr(BenchmarkRunner, 'run', execute) + mock_suite(monkeypatch) assert cli.main(['--bench', benchmark, '--warmup', '0', '--duration', '0.01']) == code captured = capsys.readouterr() expected_calls = list(BENCHMARKS) if benchmark == 'all' else [benchmark] diff --git a/tests/test_reference_artifacts.py b/tests/test_reference_artifacts.py index 58075a6..c2b8932 100644 --- a/tests/test_reference_artifacts.py +++ b/tests/test_reference_artifacts.py @@ -136,6 +136,10 @@ def test_source_install_cli_capture_matches_its_echo_stream( # noqa: PLR0915 - monkeypatch.setenv('PATH', os.pathsep.join(map(str, tool_paths))) program = 'COMPLETED_COUNT = ' + str(completed_count) + ''' from benchset import BenchmarkResult, BenchmarkRunner, cli +import pytest +from tests.helpers import mock_suite +patch = pytest.MonkeyPatch() +mock_suite(patch) calls = 0 def completed(_self, name, *, warmup_seconds, measurement_seconds, **_options): global calls @@ -684,7 +688,7 @@ def test_installed_api_check_visits_both_workloads_and_accepts_only_exact_no_wor def execute(name: str, **_options: object) -> benchset.BenchmarkResult: if name == 'unknown': - return benchset.BenchmarkResult(name) + return benchset.BenchmarkResult(name, reason='unknown benchmark: unknown') calls.append(name) reason = ('unexpected native failure' if failure == 'generic failure' else 'measurement window was too short to complete any work') @@ -768,7 +772,13 @@ def test_native_build_entrypoints_synchronize_before_invoking_cargo( def test_sdist_builds_an_installable_wheel_with_api_and_both_cli_entrypoints( bumped_source_snapshot: Path, rust_tools: Tuple[Dict[str, str], str, str], tmp_path: Path, ) -> None: + """Build real distributions outside the checkout and replace stale source metadata. + + Poisoning the checked snapshot ensures the installed wheel consumes freshly + compiled Rust data rather than merely shipping a matching historical copy. + """ environment, _cargo, _target = rust_tools + (bumped_source_snapshot / 'benchset/_metadata.json').write_text('{"stale": true}\n', encoding='utf-8') sdists = tmp_path / 'sdists' execute([sys.executable, '-m', 'build', '--sdist', '--outdir', str(sdists)], bumped_source_snapshot, environment) @@ -779,6 +789,9 @@ def test_sdist_builds_an_installable_wheel_with_api_and_both_cli_entrypoints( with tarfile.open(archive) as package: package.extractall(unpacked) source = next(unpacked.iterdir()) + assert (source / 'native/build.rs').is_file() + assert (source / 'native/src/catalog.rs').is_file() + assert (source / 'benchset/_metadata.json').is_file() assert json.loads((source / 'native/reference-calibration.json').read_text(encoding='utf-8')) wheels = tmp_path / 'wheels' # Build from the unpacked sdist, whose manifest may differ from the checkout. @@ -787,7 +800,9 @@ def test_sdist_builds_an_installable_wheel_with_api_and_both_cli_entrypoints( wheel = next(wheels.glob('*.whl')) assert wheel.name.startswith('benchset-9.8.7-') with zipfile.ZipFile(wheel) as package: - assert {'benchset/py.typed', 'benchset/reference.py', 'benchset/models.py'} <= set(package.namelist()) + assert {'benchset/py.typed', 'benchset/reference.py', 'benchset/models.py', + 'benchset/_metadata.py', 'benchset/_metadata.json'} <= set(package.namelist()) + assert json.loads(package.read('benchset/_metadata.json'))['reference_machine']['cpu_cores'] == 4 virtualenv = tmp_path / 'venv' venv.EnvBuilder(with_pip=True, symlinks=os.name != 'nt').create(virtualenv) python = virtualenv / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python') diff --git a/tests/test_release.py b/tests/test_release.py index a882145..33a9e29 100644 --- a/tests/test_release.py +++ b/tests/test_release.py @@ -302,8 +302,9 @@ def test_release_snapshot_preserves_main_and_can_be_checked_out(source: Path) -> assert 'path: "Artifacts/' not in manifest swift = shutil.which('swift') if swift: + # Cold Swift module caches can take over a minute on shared runners. parsed = subprocess.run([swift, 'package', '--package-path', str(source), 'dump-package'], - capture_output=True, text=True, check=False, timeout=60) + capture_output=True, text=True, check=False, timeout=180) assert parsed.returncode == 0, parsed.stderr targets = {target['name']: target for target in json.loads(parsed.stdout)['targets']} assert targets['CBenchset']['url'] == ( diff --git a/tests/test_resources.py b/tests/test_resources.py index 89b6976..1f5b776 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -7,7 +7,14 @@ import pytest -from benchset import BenchmarkRunner, Progress, api +from benchset import ( + BenchmarkResult, + BenchmarkRunner, + Progress, + SuiteResult, + api, + run_all, +) from benchset._native import Native from tests.helpers import assert_runner_reusable from tests.test_semantics import CallbackLibrary, ManagedNative @@ -71,11 +78,14 @@ def broken_run(handle: int, config: Any, callback: Callable[..., None], user: ob @pytest.mark.usefixtures('without_cyclic_gc') @pytest.mark.parametrize('outcome', ['completed', 'cancelled', 'failed']) -def test_live_callback_capture_released_without_gc(outcome: str) -> None: +@pytest.mark.parametrize('suite', [False, True]) +def test_live_callback_capture_released_without_gc(outcome: str, suite: bool) -> None: + """Single and suite paths release callback payloads after success, cancel or failure without GC.""" runner = BenchmarkRunner() callback = CallbackProbe(ValueError if outcome == 'failed' else None, runner if outcome == 'cancelled' else None) observed = weakref.ref(callback) - result = runner.run('single_core', warmup_seconds=0, measurement_seconds=0.02, on_progress=callback) + result = (runner._run_all(0, .02, callback).results[-1] if suite else + runner.run('single_core', warmup_seconds=0, measurement_seconds=.02, on_progress=callback)) del callback assert result.status == outcome assert (result.pi_equivalent is not None) == (outcome == 'completed') @@ -83,20 +93,85 @@ def test_live_callback_capture_released_without_gc(outcome: str) -> None: assert_runner_reusable(runner) +@pytest.mark.usefixtures('without_cyclic_gc') +@pytest.mark.parametrize('getter', ['benchset_result_v2', 'benchset_sample', 'benchset_suite_result']) +@pytest.mark.parametrize('error_type', [None, ValueError, KeyboardInterrupt]) +def test_suite_getter_failures_release_callback_captures_without_gc( + monkeypatch: pytest.MonkeyPatch, getter: str, error_type: Optional[Type[BaseException]], +) -> None: + """Getter failures cannot retain a suite callback through captured native exception frames.""" + native = Native() + monkeypatch.setattr(api, 'Native', lambda: native) + monkeypatch.setattr(native.lib, getter, lambda *_args: 0) + callback = CallbackProbe(error_type) + observed = weakref.ref(callback) + suite = run_all(warmup_seconds=0, measurement_seconds=.02, on_progress=callback) + del callback + assert suite.results[-1].status == 'failed' + assert suite.aggregated_scores.pi_equivalent is None + assert observed() is None + + +@pytest.mark.usefixtures('without_cyclic_gc') +@pytest.mark.parametrize('error_type', [RuntimeError, KeyboardInterrupt, SystemExit, GeneratorExit]) +def test_result_observer_exceptions_wait_for_native_return_and_release_captures( + monkeypatch: pytest.MonkeyPatch, error_type: Type[BaseException], +) -> None: + """Result-observer errors cross C only after return, cancel the suite and release captures without GC.""" + native = Native() + original_run, original_cancel = native.lib.benchset_run_all_configured, native.cancel + returned, cancellations = [], [] + + def execute(*args: Any) -> int: + status = int(original_run(*args)) + returned.append(True) + return status + + def cancel(handle: int, suite: bool = False) -> None: + cancellations.append((handle, suite)) + original_cancel(handle, suite) + + class Observer: + def __call__(self, _result: BenchmarkResult) -> bool: + assert not returned, 'the observer runs before the native call returns' + raise error_type('result observer failed') + + callback = Observer() + observed = weakref.ref(callback) + monkeypatch.setattr(native.lib, 'benchset_run_all_configured', execute) + monkeypatch.setattr(native, 'cancel', cancel) + handle = native.create(True) + try: + with pytest.raises(error_type, match='result observer failed'): + native.execute_suite(handle, 0, .02, None, callback) + assert returned == [True] + assert cancellations == [(handle, True)] + finally: + native.free(handle, True) + del callback + assert observed() is None + assert_runner_reusable(BenchmarkRunner()) + + @pytest.mark.parametrize('error_type', [KeyboardInterrupt, RuntimeError]) def test_handle_freed_if_interrupted_before_publication( monkeypatch: pytest.MonkeyPatch, error_type: Type[BaseException], ) -> None: + """Interrupt the first lock entry after allocation, regardless of prior ownership checks. + + The guard fires once so cleanup can reacquire the lock and free the + unpublished handle before the same runner is used again. + """ backend = ManagedNative() monkeypatch.setattr(api, 'Native', lambda: backend) runner = BenchmarkRunner() class InterruptedLock: - entries = 0 + armed = True def __enter__(self) -> None: - self.entries += 1 - if self.entries == 2: + if self.armed and backend.calls == ['create']: + self.armed = False raise error_type('interrupted after allocation') def __exit__( @@ -116,3 +191,139 @@ def __exit__( runner.cancel() assert runner.run('single_core', warmup_seconds=0, measurement_seconds=0.01).success assert backend.calls == ['create', 'free', 'create', 'execute', 'free'] + + +def _fresh_error_payload( + operation: Callable[[CallbackProbe], None], error_type: Type[BaseException], +) -> Callable[[], Optional[CallbackProbe]]: + """Leave the complete exercising frame before checking a weak reference. + + Do not retain ExceptionInfo or the raised exception: either would itself + own its traceback and make intentional caller retention look like a leak. + """ + payload = CallbackProbe(None) + observed = weakref.ref(payload) + try: + operation(payload) + except error_type: + pass + else: + pytest.fail('the caller primitive must raise its fresh exception') + return observed + + +@pytest.mark.usefixtures('without_cyclic_gc') +@pytest.mark.parametrize('primitive', ['success', 'name', 'ratio', 'ge', 'eq']) +@pytest.mark.parametrize('error_type', [ValueError, KeyboardInterrupt, SystemExit, GeneratorExit]) +def test_aggregate_host_errors_release_caller_payloads_without_gc( + primitive: str, error_type: Type[BaseException], +) -> None: + """Fresh aggregate property/operator errors must not retain inputs through bridge traceback cycles.""" + def operation(payload: CallbackProbe) -> None: + class Result(BenchmarkResult): + @property + def success(self) -> bool: + if primitive == 'success': + raise error_type('fresh aggregate host error') + return super().success + + def __getattribute__(self, name: str) -> Any: + if primitive == 'ratio' and name == 'pi_equivalent': + raise error_type('fresh aggregate host error') + return super().__getattribute__(name) + + class Name(str): + __slots__ = ('__dict__',) + __hash__ = str.__hash__ + payload: CallbackProbe + + def __eq__(self, other: object) -> bool: + raise error_type('fresh aggregate host error') + + class Ratio(float): + __slots__ = ('__dict__',) + __hash__ = float.__hash__ + payload: CallbackProbe + + def __ge__(self, other: float) -> bool: + if primitive == 'ge': + raise error_type('fresh aggregate host error') + return float.__ge__(self, other) + + def __eq__(self, other: object) -> bool: + raise error_type('fresh aggregate host error') + + name = Name('single_core') if primitive == 'name' else 'single_core' + if isinstance(name, Name): + name.payload = payload + value = Ratio(1) if primitive in ('ge', 'eq') else 1 + if isinstance(value, Ratio): + value.payload = payload + first = Result(name, status='completed', pi_equivalent=value) + object.__setattr__(first, 'payload', payload) + SuiteResult.from_results((first, BenchmarkResult('multi_core', status='completed', pi_equivalent=4))) + + for _attempt in range(3): + observed = _fresh_error_payload(operation, error_type) + assert observed() is None, 'a discarded aggregate error must not retain its caller payload' + + +@pytest.mark.usefixtures('without_cyclic_gc') +@pytest.mark.parametrize('primitive', ['validation', 'execution', 'diagnostic']) +@pytest.mark.parametrize('error_type', [ValueError, KeyboardInterrupt, SystemExit, GeneratorExit]) +def test_name_host_errors_release_caller_payloads_without_gc( + primitive: str, error_type: Type[BaseException], +) -> None: + """Name comparisons and diagnostic concatenation release their payload after an unretained error.""" + def operation(payload: CallbackProbe) -> None: + class Name(str): + __slots__ = ('__dict__',) + __hash__ = str.__hash__ + payload: CallbackProbe + comparisons = 0 + + def __eq__(self, other: object) -> bool: + self.comparisons += 1 + if primitive == 'validation' or (primitive == 'execution' and self.comparisons == 2): + raise error_type('fresh name host error') + return str.__eq__(self, other) + + def __radd__(self, _prefix: str) -> str: + raise error_type('fresh name host error') + + name = Name('unknown' if primitive == 'diagnostic' else 'single_core') + name.payload = payload + BenchmarkRunner().run(name, warmup_seconds=0, measurement_seconds=.1) + + for _attempt in range(3): + observed = _fresh_error_payload(operation, error_type) + assert observed() is None, 'a discarded name error must not retain its caller payload' + + +@pytest.mark.usefixtures('without_cyclic_gc') +@pytest.mark.parametrize('window', ['warmup', 'measurement']) +@pytest.mark.parametrize('error_type', [ValueError, KeyboardInterrupt, SystemExit, GeneratorExit]) +def test_suite_configuration_errors_release_caller_payloads_without_gc( + window: str, error_type: Type[BaseException], +) -> None: + """Second-child conversion errors release option payloads after suite cleanup without cyclic collection.""" + def operation(payload: CallbackProbe) -> None: + class Seconds(float): + __slots__ = ('__dict__',) + payload: CallbackProbe + conversions = 0 + + def __float__(self) -> float: + self.conversions += 1 + if self.conversions == 2: + raise error_type('fresh configuration host error') + return float.__float__(self) + + seconds = Seconds(0 if window == 'warmup' else .1) + seconds.payload = payload + run_all(warmup_seconds=seconds if window == 'warmup' else 0, + measurement_seconds=seconds if window == 'measurement' else .1) + + for _attempt in range(3): + observed = _fresh_error_payload(operation, error_type) + assert observed() is None, 'a discarded configuration error must not retain its caller payload' diff --git a/tests/test_runner_thread_safety.py b/tests/test_runner_thread_safety.py new file mode 100644 index 0000000..ac9e041 --- /dev/null +++ b/tests/test_runner_thread_safety.py @@ -0,0 +1,201 @@ +"""Custom signal reentry must keep cancellation attached to its original call. + +The runner intentionally becomes idle during host validation. These regressions +exercise both sides of that handoff: a signal can cancel the still-active loader +owner, or complete another call while the outer call is idle. A reentrant RLock +alone cannot protect a shared cancellation flag from those same-thread changes. +""" + +import subprocess +import sys + +import pytest + +from benchset import BenchmarkRunner + +_PROGRAM = r''' +import inspect +import signal +import sys +from benchset import BenchmarkRunner, api +from benchset._native import Native + +stage, outer_mode, nested_mode = sys.argv[1:] +outer_suite, nested_suite = outer_mode == 'suite', nested_mode == 'suite' +runner = BenchmarkRunner() +live, allocations, cancellations, releases = {}, [], [], [] +nested, signals, progress_calls = [], [], [] +loaded = False +armed = True + +class AuditedNative(Native): + def create(self, suite=False): + handle = super().create(suite) if suite else super().create() + assert handle not in live, 'an allocation overlaps a still-live handle' + identity = (self, handle, suite) + live[handle] = identity + allocations.append(identity) + return handle + + def cancel(self, handle, suite=False): + identity = (self, handle, suite) + assert live.get(handle) == identity, 'cancel used a freed, foreign, or wrong-mode handle' + cancellations.append(identity) + return super().cancel(handle, suite) if suite else super().cancel(handle) + + def free(self, handle, suite=False): + identity = (self, handle, suite) + assert live.pop(handle) == identity, 'free used a foreign or wrong-mode handle' + releases.append(identity) + return super().free(handle, suite) if suite else super().free(handle) + + +def factory(): + global loaded + native = AuditedNative() + if not loaded: + loaded = True + if stage == 'completed': + # This cancellation belongs to the loader's provisional owner. + runner.cancel() + return native + +api.Native = factory + + +def execute(suite, callback=None): + return (runner._run_all(0, .1, callback) if suite else + runner.run('single_core', warmup_seconds=0, measurement_seconds=.1, on_progress=callback)) + + +def results(value, suite): + return value.results if suite else (value,) + + +def cancel_nested(value): + if not progress_calls: + progress_calls.append(value.phase) + runner.cancel() + + +def handler(_signum, _frame): + signals.append(runner._active is not None) + if stage in ('completed', 'cancelled'): + nested.append(execute(nested_suite, cancel_nested if stage == 'cancelled' else None)) + else: + runner.cancel() + + +method = getattr(BenchmarkRunner, '_run', BenchmarkRunner.run) +source, start = inspect.getsourcelines(method) +claims = [start + index for index, line in enumerate(source) if line.strip() == 'self._active = owner'] +handoff = len(claims) == 2 +if handoff: + if stage == 'release': + # Immediately before publication is cleared: the original owner is active. + boundary = next(start + index for index, line in enumerate(source) + if line.strip() == 'self._active = None') + elif stage == 'published': + # The next traced statement runs after the owner assignment completed. + boundary = claims[-1] + 1 + else: + boundary = claims[-1] +else: + # Actual HEAD has one uninterrupted single-run owner. Equivalent reentry + # before allocation must remain busy and must retain loader cancellation. + boundary = next(start + index for index, line in enumerate(source) + if line.strip() == ('native = Native()' if stage == 'published' else 'handle = native.create()')) + + +def trace(frame, event, _arg): + global armed + if armed and event == 'line' and frame.f_code is method.__code__ and frame.f_lineno == boundary: + armed = False + signal.raise_signal(signal.SIGINT) + return trace + + +previous_handler = signal.signal(signal.SIGINT, handler) +previous_trace = sys.gettrace() +sys.settrace(trace) +try: + outer = execute(outer_suite) +finally: + sys.settrace(previous_trace) + assert signal.getsignal(signal.SIGINT) is handler, 'the runner replaced the caller signal policy' + signal.signal(signal.SIGINT, previous_handler) + +assert not armed and len(signals) == 1, 'the selected signal boundary was not exercised' +assert signals == [stage in ('release', 'published') or not handoff] +assert not live and releases == allocations, 'every native handle must be released exactly once' +assert runner._active is None and runner._handle is None and runner._native is None +expected = 'completed' if stage == 'cancelled' else 'cancelled' +assert results(outer, outer_suite)[-1].status == expected, ( + stage, outer_suite, nested_suite, results(outer, outer_suite), +) +assert signal.getsignal(signal.SIGINT) is previous_handler + +if stage in ('completed', 'cancelled'): + assert len(nested) == 1 + child_results = results(nested[0], nested_suite) + if handoff: + expected_child = 'completed' if stage == 'completed' else 'cancelled' + assert child_results[-1].status == expected_child, child_results + assert len(allocations) == 2 + # The nested call allocates/frees before its suspended outer call. + child_identity, outer_identity = allocations + assert child_identity[2] == nested_suite and outer_identity[2] == outer_suite + assert cancellations == [outer_identity if stage == 'completed' else child_identity] + assert bool(progress_calls) == (stage == 'cancelled') + else: + assert child_results[0].reason == 'this runner is already running' + assert len(allocations) == 1 and not progress_calls + assert cancellations == (allocations if stage == 'completed' else []) +else: + assert len(allocations) == 1 + assert cancellations == allocations + +# Reuse under the same caller-owned policy checks complete owner and signal cleanup. +previous_handler = signal.signal(signal.SIGINT, handler) +try: + reused = execute(outer_suite) + assert all(result.success for result in results(reused, outer_suite)), reused + assert signal.getsignal(signal.SIGINT) is handler +finally: + signal.signal(signal.SIGINT, previous_handler) +assert not live and releases == allocations +assert runner._active is None and runner._handle is None and runner._native is None +''' + + +@pytest.mark.parametrize('outer_suite', [False, True], ids=['single', 'suite']) +@pytest.mark.parametrize(('stage', 'nested_suite'), [ + ('release', False), + ('published', False), + ('completed', False), + ('completed', True), + ('cancelled', False), + ('cancelled', True), +], ids=['release', 'published', 'completed-single', 'completed-suite', 'cancelled-single', 'cancelled-suite']) +def test_signal_reentry_preserves_each_calls_cancellation( + stage: str, outer_suite: bool, nested_suite: bool, +) -> None: + """Real custom signals cannot lose or import cancellation at ownership handoffs. + + Release: cancel after loading while the provisional owner is still active. + Completed: retain an outer loader cancellation across a completed nested call. + Cancelled: keep a nested cancellation from contaminating an uncancelled outer. + Published: cancel the newly published owner as a negative control for resets. + + Child processes isolate tracing, signal policy, and possible wrong-handle + faults. Actual native calls and handle audits verify cancellation and free, + rather than merely inspecting a final Python-side flag. + """ + if not hasattr(BenchmarkRunner, '_run_all') and (outer_suite or nested_suite): + pytest.skip('HEAD supports reusable runner control only for single runs') + result = subprocess.run( + [sys.executable, '-c', _PROGRAM, stage, + 'suite' if outer_suite else 'single', 'suite' if nested_suite else 'single'], + capture_output=True, text=True, check=False, timeout=15, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/test_rust_api.py b/tests/test_rust_api.py new file mode 100644 index 0000000..16b14cb --- /dev/null +++ b/tests/test_rust_api.py @@ -0,0 +1,102 @@ +"""Compile downstream Rust code against the current library's public API.""" + +from __future__ import annotations + +import os +import platform +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Dict, List, Tuple + +import pytest + +from scripts.build_backend import rust_environment +from scripts.version import read_version + +ROOT = Path(__file__).resolve().parents[1] + + +def _compile(arguments: List[str], environment: Dict[str, str], directory: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run(arguments, cwd=directory, env=environment, capture_output=True, + encoding='utf-8', check=False, timeout=120) + + +@pytest.fixture(scope='module') +def rust_library(tmp_path_factory: pytest.TempPathFactory) -> Tuple[List[str], Dict[str, str], Path]: + """Build a fresh rlib without sharing Cargo outputs with other tests.""" + architecture = 'aarch64' if platform.machine().lower() in ('aarch64', 'arm64') else 'x86_64' + system = {'darwin': 'apple-darwin', 'linux': 'unknown-linux-gnu', 'win32': 'pc-windows-msvc'}[sys.platform] + target = architecture + '-' + system + environment = rust_environment(ROOT, target) + environment['CARGO_PKG_VERSION'] = read_version(ROOT) + rustc = shutil.which('rustc', path=environment.get('PATH')) + assert rustc is not None + compiler = [rustc, '--edition=2021'] + # Direct rustc consumers need the same explicit MSVC linker as Cargo jobs. + linker = environment.get('CARGO_TARGET_' + target.upper().replace('-', '_') + '_LINKER') + if linker is not None: + compiler.extend(['-C', 'linker=' + linker]) + directory = tmp_path_factory.mktemp('rust-public-api') + library = directory / 'libbenchset.rlib' + result = _compile([*compiler, '--crate-name', 'benchset', '--crate-type=rlib', + str(ROOT / 'native/src/lib.rs'), '-o', str(library)], environment, directory) + assert result.returncode == 0, 'Fresh Rust library did not compile:\n' + result.stdout + result.stderr + return compiler, environment, library + + +@pytest.mark.parametrize('lookup', [ + pytest.param(('from_id', 'u32', '1', '2'), id='from_id'), + pytest.param(('from_name', "&'static str", '"single_core"', '"multi_core"'), id='from_name'), +]) +def test_benchmark_keeps_downstream_extension_trait_lookup( + rust_library: Tuple[List[str], Dict[str, str], Path], tmp_path: Path, + lookup: Tuple[str, str, str, str], +) -> None: + """Internal parsers must not shadow callers' existing Benchmark traits. + + This is a real external crate: both imported and fully qualified public + Benchmark paths must retain their identity and resolve the caller's trait. + Custom argument and Result types expose either newly public inherent helper + independently, even though the library's own Rust tests still compile. + """ + compiler, environment, library = rust_library + method, input_type, single, multi = lookup + source = tmp_path / 'consumer.rs' + source.write_text(f''' +use benchset::Benchmark; + +struct HostValue({input_type}); +#[derive(Debug, PartialEq)] +struct HostError; + +trait HostLookup: Sized {{ + fn {method}(input: HostValue) -> Result; +}} +impl HostLookup for Benchmark {{ + fn {method}(input: HostValue) -> Result {{ + match input.0 {{ + {single} => Ok(Self::SingleCore), + {multi} => Ok(Self::MultiCore), + _ => Err(HostError), + }} + }} +}} + +fn main() {{ + let imported: Result = Benchmark::{method}(HostValue({single})); + let qualified: Result = + benchset::Benchmark::{method}(HostValue({multi})); + assert_eq!(imported, Ok(Benchmark::SingleCore)); + assert_eq!(qualified, Ok(Benchmark::MultiCore)); + assert_eq!(std::any::type_name::(), "benchset::Benchmark"); + assert_eq!(std::any::type_name::(), "benchset::Benchmark"); +}} +''', encoding='utf-8') + executable = tmp_path / ('consumer.exe' if os.name == 'nt' else 'consumer') + result = _compile([*compiler, str(source), '--extern', 'benchset=' + str(library), + '-o', str(executable)], environment, tmp_path) + assert result.returncode == 0, 'Public Benchmark extension-trait lookup regressed:\n' + result.stdout + result.stderr + result = _compile([str(executable)], environment, tmp_path) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/test_semantics.py b/tests/test_semantics.py index f11737f..dc96321 100644 --- a/tests/test_semantics.py +++ b/tests/test_semantics.py @@ -18,6 +18,7 @@ BenchmarkRunner, Progress, Sample, + SuiteResult, api, cli, run, @@ -32,7 +33,7 @@ CSample, Native, ) -from tests.helpers import assert_result_references, assert_runner_reusable +from tests.helpers import assert_result_references, assert_runner_reusable, mock_suite def test_model_and_result_versions_come_from_package_metadata(monkeypatch: pytest.MonkeyPatch) -> None: @@ -59,17 +60,17 @@ def distribution_version(name: str) -> str: (math.nan, 1), (math.inf, 1), (-math.inf, 1), (-1, 1), (0, math.nan), (0, math.inf), (0, -math.inf), (0, -1), (0, -0.0), ]) -def test_python_validation_does_not_allocate_native( +def test_invalid_durations_do_not_allocate_a_native_handle( monkeypatch: pytest.MonkeyPatch, warmup: float, measurement: float, ) -> None: - """Reject bad durations before loading Rust and identify the offending option. + """Reject bad durations before allocating workers and identify the offending option. A generic failure reason hides whether the caller must fix warmup or measurement and whether zero is allowed for that particular duration. """ - def forbidden() -> Native: - pytest.fail('invalid options must not load native code') - monkeypatch.setattr(api, 'Native', forbidden) + native = Native() + monkeypatch.setattr(api, 'Native', lambda: native) + monkeypatch.setattr(native, 'create', lambda *_args: pytest.fail('invalid options must not allocate a handle')) result = run('single_core', warmup_seconds=warmup, measurement_seconds=measurement) assert_result_references(result) assert result.status == 'failed' @@ -92,6 +93,7 @@ def forbidden() -> Native: @pytest.mark.parametrize(('warmup', 'measurement'), [(sys.float_info.max, 1), (0, sys.float_info.max), (0, 1e-12)]) def test_native_duration_limits_preserve_requested_diagnostics(warmup: float, measurement: float) -> None: + """Unrepresentable and rounded-to-zero windows retain their exact native errors and requested values.""" result = run('single_core', warmup_seconds=warmup, measurement_seconds=measurement) assert_result_references(result) assert result.status == 'failed' @@ -99,7 +101,8 @@ def test_native_duration_limits_preserve_requested_diagnostics(warmup: float, me assert result.pi_equivalent is None assert result.warmup_seconds == warmup assert result.measurement_seconds == measurement - assert result.reason + assert result.reason == ('measurement duration must be positive' if measurement == 1e-12 + else 'durations must be finite: warmup >= 0, measurement > 0') @pytest.mark.parametrize('phase', ['warmup', 'measurement', 'completed']) @@ -193,12 +196,15 @@ def callback(progress: Progress) -> None: class ManagedNative(Native): def __init__(self) -> None: + self.lib = Native().lib self.calls: List[str] = [] + self.suites: List[bool] = [] self.error_at: Optional[str] = None self.created, self.release = Event(), Event() self.delay_create = False - def create(self) -> int: + def create(self, suite: bool = False) -> int: + self.suites.append(suite) self.calls.append('create') if self.error_at == 'create': raise RuntimeError('create failure') @@ -207,12 +213,14 @@ def create(self) -> int: assert self.release.wait(5) return 123 - def cancel(self, handle: int) -> None: + def cancel(self, handle: int, suite: bool = False) -> None: assert handle == 123 + assert suite == self.suites[-1] self.calls.append('cancel') - def free(self, handle: int) -> None: + def free(self, handle: int, suite: bool = False) -> None: assert handle == 123 + assert suite == self.suites[-1] self.calls.append('free') def execute( @@ -228,6 +236,19 @@ def execute( return BenchmarkResult(benchmark, status='completed', score=80, warmup_seconds=warmup_seconds, measurement_seconds=measurement_seconds) + def execute_suite(self, handle: int, warmup_seconds: float, measurement_seconds: float, + on_progress: Optional[Callable[[Progress], None]], + on_result: Optional[Callable[[BenchmarkResult], bool]] = None, + _convert_windows: Optional[Callable[[], Tuple[float, float]]] = None) -> SuiteResult: + results = [] + for name in ('single_core', 'multi_core'): + result = self.execute(handle, name, warmup_seconds, measurement_seconds, on_progress) + results.append(result) + stop = on_result(result) if on_result is not None else False + if not result.success or stop: + break + return SuiteResult.from_results(tuple(results)) + @pytest.mark.parametrize('entrypoint', ['runner', 'run', 'run_all']) @pytest.mark.parametrize('durations', [None, (1.2345678901234567, 9.876543210987654)]) @@ -266,7 +287,7 @@ def record(progress: Progress) -> None: assert [event.benchmark for event in events] == benchmarks assert callback_threads == [caller] * len(benchmarks) assert all(result.success for result in results) - assert backend.calls == ['create', 'execute', 'free'] * len(benchmarks) + assert backend.calls == ['create'] + ['execute'] * len(benchmarks) + ['free'] @pytest.mark.parametrize('stage', ['init', 'create', 'execute']) @@ -363,6 +384,9 @@ def blocked(*args: Any) -> BenchmarkResult: class ResultLibrary: def __init__(self, status: int, reason: Optional[bytes], has_variation: int) -> None: + self.core = Native().lib + # Keep numeric transport real while replacing benchmark result payloads. + self.benchset_copy_double = self.core.benchset_copy_double self.value = CResultV2(1, status, 2, 2**32 - 1, 123.5, 2.5, 3.5, 4.5, 0, 2**64 - 1, 2**64 - 2, 2, 7, has_variation, 0.375, 1) self.reason = reason @@ -370,6 +394,15 @@ def __init__(self, status: int, reason: Optional[bytes], has_variation: int) -> self.cancelled = 0 self.phases = [1, 2, 3, 4, 5] + def benchset_benchmark_id(self, name: bytes, size: int) -> int: + return int(self.core.benchset_benchmark_id(name, size)) + + def benchset_validation_error(self, benchmark: int, warmup: float, measurement: float) -> int: + return int(self.core.benchset_validation_error(benchmark, warmup, measurement)) + + def benchset_validation_message(self, error: int, cli: int) -> bytes: + return self.core.benchset_validation_message(error, cli) + def benchset_run(self, _handle: int, config: Any, callback: Callable[..., None], _user: object) -> int: raw = ctypes.cast(config, ctypes.POINTER(Config)).contents self.config = Config.from_buffer_copy(raw) @@ -510,7 +543,7 @@ def test_native_platform_library_selection(monkeypatch: pytest.MonkeyPatch, plat library = Native().lib paths: List[str] = [] - def load(path: str) -> ctypes.CDLL: + def load(path: str) -> Any: paths.append(path) return library monkeypatch.setattr(sys, 'platform', platform) @@ -567,7 +600,8 @@ def execute(name: str, *, warmup_seconds: float, measurement_seconds: float, assert (warmup_seconds, measurement_seconds, on_progress) == (0.75, 1.25, callback) names.append(name) return BenchmarkResult(name, status=status if len(names) - 1 == stop_index else 'completed') - monkeypatch.setattr(api, 'run', execute) + monkeypatch.setattr(BenchmarkRunner, 'run', lambda _self, name, **options: execute(name, **options)) + mock_suite(monkeypatch) results = run_all(warmup_seconds=0.75, measurement_seconds=1.25, on_progress=callback).results for result in results: assert_result_references(result) @@ -606,6 +640,7 @@ def execute(_self: BenchmarkRunner, name: str, **_kwargs: object) -> BenchmarkRe variation_percent=0 if status == 'completed' else None, samples=(Sample(1, 1, 80, 80),) if status == 'completed' else ()) monkeypatch.setattr(BenchmarkRunner, 'run', execute) + mock_suite(monkeypatch) assert cli.main(['--warmup', '0', '--duration', '0.1']) == code captured = capsys.readouterr() assert calls == (['single_core', 'multi_core'] if status == 'completed' else ['single_core']) @@ -676,6 +711,7 @@ def execute(_self: BenchmarkRunner, name: str, **_kwargs: object) -> BenchmarkRe names.append(name) return BenchmarkResult(name, status='completed', score=1) monkeypatch.setattr(BenchmarkRunner, 'run', execute) + mock_suite(monkeypatch) assert cli.main(['--bench', benchmark, '--warmup', '0', '--duration', '0.1']) == 0 assert names == [benchmark] @@ -703,6 +739,7 @@ def test_cli_repeated_interrupt_has_priority_over_completed_result( def execute(_self: BenchmarkRunner, name: str, **_kwargs: object) -> BenchmarkResult: return BenchmarkResult(name, status='completed', score=1) monkeypatch.setattr(BenchmarkRunner, 'run', execute) + mock_suite(monkeypatch) assert cli.main(['--bench', 'single_core']) == 130 captured = capsys.readouterr() assert not captured.out @@ -740,8 +777,8 @@ def test_cancel_racing_cleanup_never_reuses_freed_handle(monkeypatch: pytest.Mon freeing, release, cancelling = Event(), Event(), Event() class SlowFree(ManagedNative): - def free(self, handle: int) -> None: - super().free(handle) + def free(self, handle: int, suite: bool = False) -> None: + super().free(handle, suite) freeing.set() assert release.wait(5) backend = SlowFree() @@ -775,6 +812,7 @@ def cancel() -> None: def test_cli_information_does_not_initialize_native(monkeypatch: pytest.MonkeyPatch, argument: str) -> None: def forbidden() -> Native: pytest.fail('informational CLI commands must not load native code') + monkeypatch.setattr(cli, 'Native', lambda: pytest.fail('CLI information must not load native code')) monkeypatch.setattr(api, 'Native', forbidden) assert cli.main([argument]) == 0 diff --git a/tests/test_setup.py b/tests/test_setup.py index dae6a32..2aa5fb0 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -2,17 +2,21 @@ from __future__ import annotations +import json +import os import runpy import subprocess from pathlib import Path from types import SimpleNamespace from typing import Any +from zipfile import ZipFile import pytest import setuptools # type: ignore[import-untyped] from setuptools.command.build_py import build_py # type: ignore[import-untyped] from setuptools.command.develop import develop # type: ignore[import-untyped] from wheel.bdist_wheel import bdist_wheel # type: ignore[import-untyped] +from wheel.wheelfile import WheelFile # type: ignore[import-untyped] from scripts import version @@ -89,6 +93,7 @@ def test_native_build_installs_fresh_target_binary_in_wheel_or_editable_package( library, even if copying fails. Both editable installs and wheel staging use this path; interrupted staging must leave no extra package files. Cargo's child environment controls the output path, including relative paths. + The wheel/editable package must receive the matching Rust metadata snapshot. """ editable, deployment, cargo_output = mode system, machine, target, filename = configuration @@ -110,7 +115,8 @@ def test_native_build_installs_fresh_target_binary_in_wheel_or_editable_package( def compile_(arguments: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: calls.append(arguments) - assert arguments == ['/prepared/cargo', 'build', '--release', '--locked', '--target', target] + assert arguments == ['/prepared/cargo', 'build', '--release', '--locked', '--target', target, + '--message-format=json-render-diagnostics'] assert kwargs['cwd'] == tmp_path assert kwargs['check'] is True if system == 'darwin': @@ -118,9 +124,13 @@ def compile_(arguments: list[str], **kwargs: Any) -> subprocess.CompletedProcess assert kwargs['env']['MACOSX_DEPLOYMENT_TARGET'] == ('14.0' if deployment == '14.0' else minimum) compiled.parent.mkdir(parents=True, exist_ok=True) compiled.write_bytes(b'fresh compiler output for ' + target.encode()) - return subprocess.CompletedProcess(arguments, 0) + metadata = compiled.parent / 'reported-build-script-output/metadata.json' + metadata.parent.mkdir(parents=True, exist_ok=True) + metadata.write_text('{"fresh": true}\n') + message = {'reason': 'build-script-executed', 'env': [['BENCHSET_METADATA_PATH_HEX', os.fsencode(metadata).hex()]]} + return subprocess.CompletedProcess(arguments, 0, stdout=json.dumps(message) + '\n') - monkeypatch.setattr(setup_module['subprocess'], 'run', compile_) + monkeypatch.setattr(subprocess, 'run', compile_) command = setup_module['distribution'].get_command_obj('build_py') command.build_lib = str(tmp_path / 'wheel-staging') command.editable_mode = editable @@ -144,6 +154,7 @@ def copy_with_reader(source: Path, output: Path) -> Path: monkeypatch.setattr(setup_module['shutil'], 'copy2', copy_with_reader) command.run() assert (destination / filename).read_bytes() == compiled.read_bytes() + assert (destination.parent / '_metadata.json').read_text() == '{"fresh": true}\n' assert (destination / 'metadata.txt').read_text() == 'keep only metadata' previous = installed.read_bytes() with pytest.raises(OSError, match='copy interrupted'): @@ -206,3 +217,44 @@ def test_legacy_editable_install_builds_in_place_but_uninstall_does_not_compile( command.run() expected: list[tuple[str, Any]] = [('develop', None)] if uninstall else [('build_py', True), ('develop', None)] assert calls == expected + + +@pytest.mark.parametrize('relative_install_path', [False, True]) +def test_registered_native_installer_excludes_only_the_package_publication_lock_from_wheels( + setup_module: dict[str, Any], tmp_path: Path, monkeypatch: pytest.MonkeyPatch, relative_install_path: bool, +) -> None: + """Keep the persistent build lock out of actual wheel payloads without unlinking it. + + Package-data/manifest exclusions do not control install_lib.copy_tree: the + lock is created in build_lib after Python files were collected. Exercise + the registered install command and real WheelFile assembly, including the + relative '/./' install_dir spelling used by bdist_wheel. Preserve all other + files, including an unrelated package's equally named lock file. + """ + build, installed = tmp_path / 'build-lib', tmp_path / 'wheel-staging' + contents = { + 'benchset/__init__.py': b'', + 'benchset/_metadata.json': b'{}', + 'benchset/_native/libbenchset.so': b'native image', + 'benchset/.benchset.publish.lock': b'held lock inode', + 'benchset/keep.lock': b'ordinary package data', + 'other/.benchset.publish.lock': b'unrelated package data', + } + for name, content in contents.items(): + source = build / name + source.parent.mkdir(parents=True, exist_ok=True) + source.write_bytes(content) + monkeypatch.chdir(tmp_path) + command = setup_module['distribution'].get_command_obj('install_lib') + command.build_dir = str(build) + command.install_dir = 'wheel-staging/./' if relative_install_path else str(installed) + command.install() + wheel = tmp_path / 'benchset-1.2.3-py3-none-any.whl' + with WheelFile(str(wheel), 'w') as archive: + archive.write_files(str(installed)) + with ZipFile(wheel) as archive: + assert 'benchset/.benchset.publish.lock' not in archive.namelist() + for name, content in contents.items(): + if name != 'benchset/.benchset.publish.lock': + assert archive.read(name) == content + assert (build / 'benchset/.benchset.publish.lock').read_bytes() == b'held lock inode' diff --git a/tests/test_signals.py b/tests/test_signals.py index 2741cf7..bde1308 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -4,7 +4,7 @@ import signal import subprocess import sys -from typing import Iterator +from typing import Callable, Iterator import pytest @@ -88,8 +88,9 @@ def synchronized_execute(self, *args, **kwargs): @pytest.mark.usefixtures('default_sigint') @pytest.mark.parametrize('stage', ['loading', 'allocation', 'execute', 'free_before', 'free_after']) -def test_sigint_defers_through_handle_lifetime_and_restores_runner( - monkeypatch: pytest.MonkeyPatch, stage: str, +@pytest.mark.parametrize('suite', [False, True]) +def test_sigint_defers_through_handle_lifetime_and_restores_runner( # noqa: PLR0915 — Keep all lifetime fault injections in one matrix. + monkeypatch: pytest.MonkeyPatch, stage: str, suite: bool, ) -> None: """Deliver SIGINT to native cancellation while deferring the exception until cleanup. @@ -100,26 +101,29 @@ def test_sigint_defers_through_handle_lifetime_and_restores_runner( backend = ManagedNative() original_create, original_free = backend.create, backend.free armed = True + cancel_attempts = [] def interrupt() -> None: if armed: signal.raise_signal(signal.SIGINT) signal.raise_signal(signal.SIGINT) - def create() -> int: - handle = original_create() + def create(suite: bool = False) -> int: + handle = original_create(suite) if stage == 'allocation': interrupt() return handle - def free(handle: int) -> None: + def free(handle: int, suite: bool = False) -> None: if stage == 'free_before': interrupt() - original_free(handle) + original_free(handle, suite) if stage == 'free_after': interrupt() - def cancel(handle: int) -> None: + def cancel(handle: int, suite: bool = False) -> None: + cancel_attempts.append(suite) + assert suite == backend.suites[-1] assert 'free' not in backend.calls, 'a signal must never cancel a freed handle' assert not stage.startswith('free'), 'cancellation must be disabled before freeing starts' assert handle == 123 @@ -139,14 +143,18 @@ def progress(_value: Progress) -> None: monkeypatch.setattr(backend, 'cancel', cancel) monkeypatch.setattr(api, 'Native', factory) runner = BenchmarkRunner() + execute: Callable[[], object] = lambda: (runner._run_all(0, .01, progress) if suite else + runner.run('single_core', warmup_seconds=0, measurement_seconds=.01, on_progress=progress)) try: with pytest.raises(KeyboardInterrupt): - runner.run('single_core', warmup_seconds=0, measurement_seconds=.01, on_progress=progress) + execute() assert backend.calls.count('free') == backend.calls.count('create') assert runner._handle is None assert runner._native is None assert not runner._active assert ('cancel' in backend.calls) == (stage in ('loading', 'allocation', 'execute')) + assert bool(cancel_attempts) == (stage in ('loading', 'allocation', 'execute')) + assert all(mode == suite for mode in cancel_attempts) assert signal.getsignal(signal.SIGINT) is signal.default_int_handler finally: armed = False @@ -266,13 +274,14 @@ def handler(*_): def entry_line(method): source, start = inspect.getsourcelines(method) - return next(start + i for i, line in enumerate(source) if line.strip() == 'if self._active:') + marker = 'busy = self._active is not None' if method.__name__ == '_run' else 'if self._active:' + return next(start + i for i, line in enumerate(source) if line.strip() == marker) -run_entry, cancel_entry = entry_line(BenchmarkRunner.run), entry_line(BenchmarkRunner.cancel) +run_entry, cancel_entry = entry_line(BenchmarkRunner._run), entry_line(BenchmarkRunner.cancel) def trace(frame, event, _arg): if armed and event == 'line': - if frame.f_code is BenchmarkRunner.run.__code__: + if frame.f_code is BenchmarkRunner._run.__code__: if stage == 'entry' and frame.f_lineno == run_entry: deliver() elif stage == 'claimed' and runner._active and runner._native is None: @@ -315,3 +324,54 @@ def trace(frame, event, _arg): result = subprocess.run([sys.executable, '-c', code, stage], capture_output=True, text=True, check=False, timeout=10) assert result.returncode == 0, result.stdout + result.stderr + + +def test_reentrant_signal_before_suite_claim_preserves_native_handle_kind() -> None: + """A nested single run cannot leave the outer suite using single-handle cancel/free. + + A real signal at the ownership-publication line makes reentry deterministic. + The managed backend rejects incorrect handle kinds without invoking an + incompatible native destructor; the child isolates tracing and signal state. + """ + program = r''' +import inspect +import signal +import sys +from benchset import BenchmarkRunner, api +from tests.test_semantics import ManagedNative + +backend = ManagedNative() +api.Native = lambda: backend +runner = BenchmarkRunner() +nested = [] +armed = True +source, start = inspect.getsourcelines(BenchmarkRunner._run) +claim = next(start + index for index, line in enumerate(source) if line.strip() == 'self._active = owner') + +def handler(*_): + nested.append(runner.run('single_core', warmup_seconds=0, measurement_seconds=.01)) + +def trace(frame, event, _arg): + global armed + if armed and event == 'line' and frame.f_code is BenchmarkRunner._run.__code__ and frame.f_lineno == claim: + armed = False + signal.raise_signal(signal.SIGINT) + return trace + +previous = signal.signal(signal.SIGINT, handler) +sys.settrace(trace) +try: + suite = runner._run_all(0, .01, lambda _progress: runner.cancel()) +finally: + sys.settrace(None) + signal.signal(signal.SIGINT, previous) +assert not armed and len(nested) == 1 and nested[0].success +assert backend.suites == [False, True] +assert backend.calls.count('cancel') == 2 +assert backend.calls.count('free') == backend.calls.count('create') == 2 +assert len(suite.results) == 2 +assert runner._active is None and runner._handle is None and runner._native is None +assert runner.run('single_core', warmup_seconds=0, measurement_seconds=.01).success +''' + result = subprocess.run([sys.executable, '-c', program], capture_output=True, text=True, check=False, timeout=10) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/test_suite.py b/tests/test_suite.py index 480be59..fa62a64 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -4,8 +4,10 @@ import json import math import signal +import subprocess +import sys from dataclasses import FrozenInstanceError, asdict, replace -from typing import Callable, List, Optional +from typing import Callable, Dict, List, Optional, Tuple, Type import pytest @@ -17,16 +19,208 @@ Progress, Sample, SuiteResult, - api, cli, run_all, ) +from tests.helpers import assert_runner_reusable, mock_suite def completed(name: str, equivalent: float) -> BenchmarkResult: return BenchmarkResult(name, status='completed', score=999, pi_equivalent=equivalent) +def test_direct_suite_constructor_stores_supplied_fields_without_aggregation() -> None: + """Keep the documented storage constructor distinct from from_results. + + A supplied aggregate may intentionally disagree with incomplete results; + construction must preserve both objects instead of recalculating either. + """ + results = (completed('single_core', 0.5),) + scores = AggregatedScores(pi_equivalent=123.45678901234567) + suite = SuiteResult(results, scores) + assert suite.results is results + assert suite.aggregated_scores is scores + + +def test_direct_suite_constructor_remains_native_free_in_a_fresh_interpreter() -> None: + """The storage constructor must work without Rust, unlike the computing factory. + + A fresh process detects accidental eager imports or native recalculation + that an already imported/cached test process could conceal. + """ + program = """ +import ctypes +def forbidden(*args, **kwargs): + raise AssertionError('storage construction must not load Rust') +ctypes.CDLL = forbidden +from benchset import AggregatedScores, BenchmarkResult, SuiteResult +results = (BenchmarkResult('single_core'),) +scores = AggregatedScores(123.45678901234567) +suite = SuiteResult(results, scores) +assert suite.results is results and suite.aggregated_scores is scores +""" + result = subprocess.run([sys.executable, '-c', program], capture_output=True, text=True, check=False) + assert result.returncode == 0, result.stderr + assert not result.stdout + assert not result.stderr + + +@pytest.mark.parametrize('missing', [(0,), (1,), (0, 1)]) +def test_aggregate_requires_each_ratio_independently(missing: Tuple[int, ...]) -> None: + """Missing first, second or both coefficients cannot become present zero sentinels.""" + results = [completed('single_core', 0.0), completed('multi_core', 2.0)] + for position in missing: + results[position] = replace(results[position], pi_equivalent=None) + assert SuiteResult.from_results(tuple(results)).aggregated_scores.pi_equivalent is None + + +@pytest.mark.parametrize('position', [0, 1]) +def test_aggregate_negative_zero_is_present_canonical_positive_zero(position: int) -> None: + """A negative-zero input stays present but produces the documented positive-zero aggregate.""" + results = [completed('single_core', 0.5), completed('multi_core', 2.0)] + results[position] = replace(results[position], pi_equivalent=-0.0) + value = SuiteResult.from_results(tuple(results)).aggregated_scores.pi_equivalent + assert value is not None + assert value == 0.0 + assert math.copysign(1.0, value) == 1.0 + + +def test_aggregation_is_available_inside_active_benchmark_callbacks() -> None: + """Pure aggregation must remain usable while Rust's measurement runner is busy. + + Computing inside the first active progress callback catches an accidental + dependency on the single-run reservation; cancellation then keeps it fast + and a subsequent real run verifies that aggregation did not disturb ownership. + """ + runner = BenchmarkRunner() + values = (completed('single_core', 0.5), completed('multi_core', 2.0)) + aggregates: List[Optional[float]] = [] + + def progress(value: Progress) -> None: + if value.phase in ('warmup', 'measurement'): + aggregates.append(SuiteResult.from_results(values).aggregated_scores.pi_equivalent) + runner.cancel() + + result = runner.run('single_core', warmup_seconds=60, measurement_seconds=1, on_progress=progress) + assert result.status == 'cancelled' + assert aggregates == [1.0] + assert_runner_reusable(runner) + + +@pytest.mark.parametrize(('status', 'success', 'expected'), [ + ('completed', False, None), ('failed', True, 2.0), ('unknown', True, 2.0), +]) +def test_aggregate_preserves_overridden_success(status: str, success: bool, expected: Optional[float]) -> None: + """The public factory respects a typed result subclass's success property, as before.""" + class Result(BenchmarkResult): + @property + def success(self) -> bool: + return success + + results = (Result('single_core', status=status, pi_equivalent=1), + Result('multi_core', status=status, pi_equivalent=4)) + suite = SuiteResult.from_results(results) + assert suite.results is results + assert suite.aggregated_scores.pi_equivalent == expected + + +@pytest.mark.parametrize('error_type', [ValueError, RuntimeError, OverflowError, KeyboardInterrupt, SystemExit, GeneratorExit]) +def test_aggregate_preserves_success_property_exceptions(error_type: Type[BaseException]) -> None: + """A property error crosses back from Rust with its original Python type and message.""" + class Result(BenchmarkResult): + @property + def success(self) -> bool: + raise error_type('custom success failed') + + results = (Result('single_core', status='completed', pi_equivalent=1), completed('multi_core', 4)) + with pytest.raises(error_type, match='custom success failed'): + SuiteResult.from_results(results) + + +@pytest.mark.parametrize('first', [None, math.nan, -1.0]) +def test_aggregate_does_not_read_success_after_an_invalid_ratio(first: Optional[float]) -> None: + """An earlier unavailable coefficient hides later properties, preserving short-circuit order.""" + class Result(BenchmarkResult): + @property + def success(self) -> bool: + raise AssertionError('later success must not be evaluated') + + results = (replace(completed('single_core', 1), pi_equivalent=first), + Result('multi_core', status='completed', pi_equivalent=4)) + assert SuiteResult.from_results(results).aggregated_scores.pi_equivalent is None + + +@pytest.mark.parametrize(('case', 'expect_error'), [ + ('incomplete', False), ('extra', False), ('reordered', False), ('unknown', False), + ('failed-first', False), ('cancelled-second', False), ('missing-first', False), + ('nan-first', False), ('negative-first', False), ('huge-before-missing', True), + ('huge-before-failed', True), ('negative-huge-before-zero', True), +]) +def test_aggregate_preserves_short_circuit_before_numeric_conversion(case: str, expect_error: bool) -> None: + """Preserve which coefficients the public factory converts before rejecting a suite. + + Integers satisfy the float annotation, but an enormous integer cannot become + a C double. Structural rejection and earlier failed/missing/invalid values + must hide unused conversions, while an earlier enormous value must still + raise even if a later value would make the aggregate unavailable. + """ + huge = completed('single_core', 10**1000) + multi = completed('multi_core', 2) + cases: Dict[str, Tuple[BenchmarkResult, ...]] = { + 'incomplete': (huge,), + 'extra': (huge, multi, multi), + 'reordered': (replace(huge, benchmark='multi_core'), completed('single_core', 1)), + 'unknown': (huge, replace(multi, benchmark='unknown')), + 'failed-first': (replace(huge, status='failed'), multi), + 'cancelled-second': (completed('single_core', .5), replace(multi, status='cancelled', pi_equivalent=10**1000)), + 'missing-first': (replace(huge, pi_equivalent=None), replace(multi, pi_equivalent=10**1000)), + 'nan-first': (replace(huge, pi_equivalent=math.nan), replace(multi, pi_equivalent=10**1000)), + 'negative-first': (replace(huge, pi_equivalent=-1), replace(multi, pi_equivalent=10**1000)), + 'huge-before-missing': (huge, replace(multi, pi_equivalent=None)), + 'huge-before-failed': (huge, replace(multi, status='failed')), + 'negative-huge-before-zero': (replace(huge, pi_equivalent=-(10**1000)), replace(multi, pi_equivalent=0)), + } + results = cases[case] + if expect_error: + with pytest.raises(OverflowError, match='int too large to convert to float'): + SuiteResult.from_results(results) + else: + suite = SuiteResult.from_results(results) + assert suite.results is results + assert suite.aggregated_scores.pi_equivalent is None + + +@pytest.mark.parametrize(('single', 'multi', 'expected'), [ + (sys.float_info.max, sys.float_info.max, sys.float_info.max), + (float.fromhex('0x0.0000000000001p-1022'), float.fromhex('0x0.0000000000001p-1022'), + float.fromhex('0x0.0000000000001p-1022')), + (sys.float_info.max, float.fromhex('0x0.0000000000001p-1022'), + math.sqrt(sys.float_info.max) * math.sqrt(float.fromhex('0x0.0000000000001p-1022'))), +]) +def test_aggregate_preserves_actual_finite_float_endpoints(single: float, multi: float, expected: float) -> None: + """True maximum and minimum-subnormal doubles must survive finite log-space aggregation. + + Taking square roots separately supplies an independent oracle without the + overflow/underflow that a direct product would introduce at these endpoints. + """ + value = SuiteResult.from_results((completed('single_core', single), completed('multi_core', multi))) + assert value.aggregated_scores.pi_equivalent is not None + assert math.isclose(value.aggregated_scores.pi_equivalent, expected, rel_tol=1e-12, abs_tol=0) + + +@pytest.mark.parametrize('status', ['failed', 'cancelled', 'unknown']) +@pytest.mark.parametrize('position', [0, 1]) +def test_aggregate_rejects_unsuccessful_results_with_retained_ratios(status: str, position: int) -> None: + """Reject status independently of ratio presence, at either suite position. + + Manually assembled snapshots can retain valid coefficients after a status + change; those values must never turn a failed or cancelled suite into success. + """ + results = [completed('single_core', 0.5), completed('multi_core', 2.0)] + results[position] = replace(results[position], status=status) + assert SuiteResult.from_results(tuple(results)).aggregated_scores.pi_equivalent is None + + @pytest.mark.parametrize(('single', 'multi', 'expected', 'tolerance'), [ (0.5, 2.0, 1.0, 1e-15), (0.0, 0.0, 0.0, 1e-15), (1.0, 1.0, 1.0, 1e-15), (1.0, 4.0, 2.0, 1e-15), (4.0, 1.0, 2.0, 1e-15), @@ -114,7 +308,8 @@ def execute(name: str, *, warmup_seconds: float, measurement_seconds: float, produced.append(result) return result - monkeypatch.setattr(api, 'run', execute) + monkeypatch.setattr(BenchmarkRunner, 'run', lambda _self, name, **options: execute(name, **options)) + mock_suite(monkeypatch) suite = run_all(warmup_seconds=0.25, measurement_seconds=0.75, on_progress=callback) assert isinstance(suite, SuiteResult) assert calls == list(BENCHMARKS[:2 if status == 'completed' else stop + 1]) @@ -171,6 +366,7 @@ def execute(_runner: BenchmarkRunner, name: str, **_options: object) -> Benchmar return completed(name, 0.5 if name == 'single_core' else 2.0) monkeypatch.setattr(BenchmarkRunner, 'run', execute) + mock_suite(monkeypatch) assert cli.main(['--bench=all', '--warmup=0', '--duration=.01']) == 130 output = capsys.readouterr() assert 'single_core' in output.out diff --git a/tests/test_suite_options.py b/tests/test_suite_options.py new file mode 100644 index 0000000..5b001ec --- /dev/null +++ b/tests/test_suite_options.py @@ -0,0 +1,442 @@ +"""Characterize per-benchmark option conversion through the public suite API.""" + +import signal +from typing import List, Optional, Tuple, Type, Union + +import pytest + +from benchset import BenchmarkRunner, Progress, api, run_all +from tests.helpers import assert_runner_reusable + + +class _Seconds(float): + """Expose when a float-compatible caller supplies each successive value.""" + + def __new__( + cls, _name: str, _values: Tuple[Union[float, BaseException], ...], _events: List[str], + ) -> '_Seconds': + return super().__new__(cls, 0.0) + + def __init__( + self, name: str, values: Tuple[Union[float, BaseException], ...], events: List[str], + ) -> None: + self.name = name + self.values = values + self.events = events + self.conversions = 0 + + def __float__(self) -> float: + self.conversions += 1 + self.events.append(f'{self.name}:{self.conversions}') + value = self.values[self.conversions - 1] + if isinstance(value, BaseException): + raise value + return value + + +def _record_progress(events: List[str], value: Progress) -> None: + """Record child boundaries without depending on progress sampling frequency.""" + started = 'start:' + value.benchmark + if started not in events: + events.append(started) + if value.phase in ('completed', 'failed', 'cancelled'): + events.append(value.phase + ':' + value.benchmark) + + +def test_terminal_sigint_stops_before_another_option_conversion() -> None: + """A signal in the first terminal callback must not inspect the next child's options. + + Legacy run() raised after cleanup before run_all could attempt another child. + The native suite must stop at the same host boundary, even though its first + stored result was already completed when the interrupt arrived. + """ + events: List[str] = [] + warmup = _Seconds('warmup', (0.0, AssertionError('unused warmup')), events) + measurement = _Seconds('measurement', (0.1, AssertionError('unused measurement')), events) + + def progress(value: Progress) -> None: + _record_progress(events, value) + if value.benchmark == 'single_core' and value.phase == 'completed': + signal.raise_signal(signal.SIGINT) + + previous = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + with pytest.raises(KeyboardInterrupt): + run_all(warmup_seconds=warmup, measurement_seconds=measurement, on_progress=progress) + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + finally: + signal.signal(signal.SIGINT, previous) + assert warmup.conversions == measurement.conversions == 1 + assert events == ['warmup:1', 'measurement:1', 'start:single_core', 'completed:single_core'] + + +@pytest.mark.parametrize(('second_warmup', 'second_measurement', 'reason'), [ + (0.02, 0.12, None), + (-1.0, 0.12, 'warmup_seconds must be finite and nonnegative'), + (0.02, 0.0, 'measurement_seconds must be finite and positive'), +]) +def test_each_child_converts_current_windows_in_warmup_then_measurement_order( + second_warmup: float, second_measurement: float, reason: Optional[str], +) -> None: + """Read fresh options after the prior child finishes and before the next starts. + + Both conversions precede validation, even when the new warmup is invalid. + Distinct second values must reach the second snapshot rather than reuse + the first child's converted configuration. + """ + events: List[str] = [] + warmup = _Seconds('warmup', (0.0, second_warmup), events) + measurement = _Seconds('measurement', (0.1, second_measurement), events) + + suite = run_all(warmup_seconds=warmup, measurement_seconds=measurement, + on_progress=lambda value: _record_progress(events, value)) + + assert len(suite.results) == 2 + first, second = suite.results + assert first.benchmark == 'single_core' + assert first.success + assert (first.warmup_seconds, first.measurement_seconds) == (0.0, 0.1) + assert second.benchmark == 'multi_core' + assert (second.warmup_seconds, second.measurement_seconds) == (second_warmup, second_measurement) + assert second.status == ('completed' if reason is None else 'failed') + assert second.reason == reason + assert warmup.conversions == measurement.conversions == 2 + expected = ['warmup:1', 'measurement:1', 'start:single_core', 'completed:single_core', + 'warmup:2', 'measurement:2'] + if reason is None: + expected.extend(['start:multi_core', 'completed:multi_core']) + assert events == expected + assert (suite.aggregated_scores.pi_equivalent is not None) == (reason is None) + + +@pytest.mark.parametrize('failure', ['warmup', 'measurement', 'callback']) +def test_first_child_failure_does_not_convert_options_for_another_child(failure: str) -> None: + """Stop before touching second values after validation or callback failure. + + An invalid first warmup must still convert the first measurement. Raising + second values also detects speculative conversion ahead of suite stopping. + """ + events: List[str] = [] + warmup = _Seconds('warmup', (-1.0 if failure == 'warmup' else 0.0, + AssertionError('unused second warmup')), events) + measurement = _Seconds('measurement', (0.0 if failure == 'measurement' else 0.1, + AssertionError('unused second measurement')), events) + + def progress(value: Progress) -> None: + _record_progress(events, value) + if failure == 'callback': + raise ValueError('first child callback failed') + + suite = run_all(warmup_seconds=warmup, measurement_seconds=measurement, on_progress=progress) + + assert len(suite.results) == 1 + assert suite.results[0].benchmark == 'single_core' + assert suite.results[0].status == 'failed' + assert suite.results[0].reason == { + 'warmup': 'warmup_seconds must be finite and nonnegative', + 'measurement': 'measurement_seconds must be finite and positive', + 'callback': 'progress callback failed: ValueError: first child callback failed', + }[failure] + assert suite.aggregated_scores.pi_equivalent is None + assert warmup.conversions == measurement.conversions == 1 + assert events == ['warmup:1', 'measurement:1'] + (['start:single_core'] if failure == 'callback' else []) + + +@pytest.mark.parametrize('window', ['warmup', 'measurement']) +@pytest.mark.parametrize('error_type', [ + TypeError, ValueError, OSError, RuntimeError, OverflowError, KeyboardInterrupt, SystemExit, GeneratorExit, +]) +def test_second_child_conversion_exceptions_escape_after_cleanup_and_allow_reuse( + monkeypatch: pytest.MonkeyPatch, window: str, error_type: Type[BaseException], +) -> None: + """Preserve the original exception instead of a failed result or callback diagnostic. + + Only runner construction is shared: the public suite and single-run entry + points retain their real execution paths. Reusing that runner after the + exception exposes leaked activity, while signal restoration checks cleanup + for BaseExceptions as well as ordinary numeric conversion errors. + """ + runner = BenchmarkRunner() + monkeypatch.setattr(api, 'BenchmarkRunner', lambda: runner) + previous_handler = signal.getsignal(signal.SIGINT) + events: List[str] = [] + error = error_type('second ' + window + ' conversion failed') + warmup = _Seconds('warmup', (0.0, error if window == 'warmup' else 0.0), events) + measurement = _Seconds('measurement', (0.1, error if window == 'measurement' else 0.1), events) + + with pytest.raises(error_type) as caught: + run_all(warmup_seconds=warmup, measurement_seconds=measurement, + on_progress=lambda value: _record_progress(events, value)) + + assert caught.value is error + assert caught.value.__context__ is None + assert str(caught.value) == 'second ' + window + ' conversion failed' + expected = ['warmup:1', 'measurement:1', 'start:single_core', 'completed:single_core', 'warmup:2'] + if window == 'measurement': + expected.append('measurement:2') + assert events == expected + assert signal.getsignal(signal.SIGINT) is previous_handler + assert runner.run('single_core', warmup_seconds=0, measurement_seconds=0.1).success + + +@pytest.mark.parametrize('window', ['warmup', 'measurement']) +@pytest.mark.parametrize('policy', ['default', 'handled', 'custom', 'ignored']) +def test_sigint_during_second_conversion_interrupts_caller_immediately( + monkeypatch: pytest.MonkeyPatch, window: str, policy: str, +) -> None: + """Legacy child conversions run under the caller's ordinary SIGINT policy. + + A signal raised inside __float__ must stop that method immediately, before + its next statement or another conversion. The native callback catches the + original KeyboardInterrupt until workers and handles are cleaned up, then + returns that same exception without attaching its private transport error. + """ + events: List[str] = [] + raised: List[KeyboardInterrupt] = [] + + class Seconds(_Seconds): + def __float__(self) -> float: + value = super().__float__() + if self.name == window and self.conversions == 2: + assert signal.getsignal(signal.SIGINT) is handler + try: + signal.raise_signal(signal.SIGINT) + except KeyboardInterrupt as error: + raised.append(error) + if policy != 'handled': + raise + events.append('after-signal') + return value + + def custom_handler(_signum: int, _frame: object) -> None: + events.append('custom-handler') + + runner = BenchmarkRunner() + monkeypatch.setattr(api, 'BenchmarkRunner', lambda: runner) + warmup = Seconds('warmup', (0.0, 0.0), events) + measurement = Seconds('measurement', (0.1, 0.1), events) + handler = custom_handler if policy == 'custom' else ( + signal.SIG_IGN if policy == 'ignored' else signal.default_int_handler + ) + previous = signal.signal(signal.SIGINT, handler) + try: + if policy == 'default': + with pytest.raises(KeyboardInterrupt) as caught: + run_all(warmup_seconds=warmup, measurement_seconds=measurement, + on_progress=lambda value: _record_progress(events, value)) + assert len(raised) == 1 + assert raised[0] is caught.value + assert caught.value.__context__ is None + else: + try: + suite = run_all(warmup_seconds=warmup, measurement_seconds=measurement, + on_progress=lambda value: _record_progress(events, value)) + except KeyboardInterrupt: + pytest.fail('the caller handled SIGINT and the suite must continue') + assert len(suite.results) == 2 + assert all(result.success for result in suite.results) + assert len(raised) == (1 if policy == 'handled' else 0) + assert signal.getsignal(signal.SIGINT) is handler + expected = ['warmup:1', 'measurement:1', 'start:single_core', 'completed:single_core', 'warmup:2'] + if window == 'measurement': + expected.append('measurement:2') + if policy != 'default': + if policy == 'custom': + expected.append('custom-handler') + expected.append('after-signal') + if window == 'warmup': + expected.append('measurement:2') + expected.extend(['start:multi_core', 'completed:multi_core']) + assert events == expected + assert runner.run('single_core', warmup_seconds=0, measurement_seconds=0.1).success + finally: + signal.signal(signal.SIGINT, previous) + + +@pytest.mark.parametrize('second_signal', ['after-catching', 'finally']) +@pytest.mark.parametrize('first_no_work', [False, True]) +def test_repeated_converter_sigint_preserves_default_exception_chain( + monkeypatch: pytest.MonkeyPatch, second_signal: str, first_no_work: bool, +) -> None: + """Each user-code SIGINT raises with Python's exception replacement chain. + + Reach the second conversion even if scheduling starves the first window. + Retry only an exact no-work failure before any signal or second conversion; + the nanosecond control exercises this legitimate outcome deterministically. + No missing signal, wrong exception or malformed suite may be retried. + """ + events: List[str] = [] + raised: List[KeyboardInterrupt] = [] + + class Seconds(_Seconds): + def __float__(self) -> float: + value = super().__float__() + if self.conversions == 2: + try: + try: + signal.raise_signal(signal.SIGINT) + except KeyboardInterrupt as error: + raised.append(error) + if second_signal == 'finally': + raise + events.append('after-first-signal') + try: + signal.raise_signal(signal.SIGINT) + except KeyboardInterrupt as error: + raised.append(error) + raise + events.append('after-second-signal') + finally: + if second_signal == 'finally': + events.append('conversion-finally') + try: + signal.raise_signal(signal.SIGINT) + except KeyboardInterrupt as error: + raised.append(error) + raise + events.append('after-finally-signal') + return value + + runner = BenchmarkRunner() + monkeypatch.setattr(api, 'BenchmarkRunner', lambda: runner) + previous = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + caught: Optional[KeyboardInterrupt] = None + for attempt in range(3): + events.clear() + try: + suite = run_all( + warmup_seconds=Seconds('warmup', (0.0, 0.0), events), + measurement_seconds=_Seconds('measurement', + (1e-9 if first_no_work and attempt == 0 else 0.1, + AssertionError('unused measurement')), events), + ) + except KeyboardInterrupt as error: + assert not (first_no_work and attempt == 0), 'The suite continued after a no-work failure' + caught = error + break + assert len(suite.results) == 1, suite + assert suite.results[0].status == 'failed', suite + assert suite.results[0].reason == 'measurement window was too short to complete any work', suite + assert events == ['warmup:1', 'measurement:1'] + assert not raised + assert caught is not None, 'Could not reach the second conversion after three windows' + assert len(raised) == 2 + assert caught is raised[1] + assert raised[0].__context__ is None + assert caught.__context__ is (raised[0] if second_signal == 'finally' else None) + assert caught.__cause__ is None + boundary = 'conversion-finally' if second_signal == 'finally' else 'after-first-signal' + assert events == ['warmup:1', 'measurement:1', 'warmup:2', boundary] + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + assert_runner_reusable(runner) + finally: + signal.signal(signal.SIGINT, previous) + + +def test_suite_configuration_exception_survives_sigint_during_native_cleanup(monkeypatch: pytest.MonkeyPatch) -> None: + """The new shared suite handle defers teardown signals after host conversion already failed. + + HEAD had already freed its first child's handle before this conversion, so + this is an added-wrapper ownership check, not an original-behavior baseline. + Keep the conversion error, complete actual native cleanup and reuse the runner. + """ + from benchset import _native # noqa: PLC0415 + + if not hasattr(_native.Native, 'execute_suite'): + pytest.skip('HEAD has no shared native suite handle to free after a conversion error') + + events: List[str] = [] + raised: List[KeyboardInterrupt] = [] + + class Seconds(_Seconds): + def __float__(self) -> float: + value = super().__float__() + if self.conversions == 2: + try: + signal.raise_signal(signal.SIGINT) + except KeyboardInterrupt as error: + raised.append(error) + raise + return value + + original_free = _native.Native.free + + def free(native: _native.Native, handle: int, suite: bool = False) -> None: + if raised: + events.append('native-cleanup') + signal.raise_signal(signal.SIGINT) + events.append('native-cleanup-done') + original_free(native, handle, True) if suite else original_free(native, handle) + + monkeypatch.setattr(_native.Native, 'free', free) + runner = BenchmarkRunner() + monkeypatch.setattr(api, 'BenchmarkRunner', lambda: runner) + warmup = Seconds('warmup', (0.0, 0.0), events) + measurement = _Seconds('measurement', (0.1, AssertionError('unused measurement')), events) + previous = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + with pytest.raises(KeyboardInterrupt) as caught: + run_all(warmup_seconds=warmup, measurement_seconds=measurement) + assert len(raised) == 1 + assert caught.value is raised[0] + assert caught.value.__context__ is None + assert events == ['warmup:1', 'measurement:1', 'warmup:2', 'native-cleanup', 'native-cleanup-done'] + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + raised.clear() + assert runner.run('single_core', warmup_seconds=0, measurement_seconds=0.1).success + finally: + signal.signal(signal.SIGINT, previous) + + +def test_handled_warmup_sigint_preserves_measurement_interrupt_policy() -> None: + """Handling one converter's interrupt must not consume the next converter's signal.""" + events: List[str] = [] + raised: List[KeyboardInterrupt] = [] + + class Seconds(_Seconds): + def __float__(self) -> float: + value = super().__float__() + if self.conversions == 2: + try: + signal.raise_signal(signal.SIGINT) + except KeyboardInterrupt as error: + raised.append(error) + if self.name == 'measurement': + raise + events.append('handled-warmup') + return value + + previous = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + with pytest.raises(KeyboardInterrupt) as caught: + run_all(warmup_seconds=Seconds('warmup', (0.0, 0.0), events), + measurement_seconds=Seconds('measurement', (0.1, 0.1), events)) + assert len(raised) == 2 + assert caught.value is raised[1] + assert caught.value.__context__ is None + assert events == ['warmup:1', 'measurement:1', 'warmup:2', 'handled-warmup', 'measurement:2'] + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + finally: + signal.signal(signal.SIGINT, previous) + + +def test_background_suite_conversions_preserve_the_callers_signal_policy() -> None: + """A worker thread converts both children without changing process signal handlers.""" + from concurrent.futures import ThreadPoolExecutor # noqa: PLC0415 + + events: List[str] = [] + handler = signal.getsignal(signal.SIGINT) + + class Seconds(_Seconds): + def __float__(self) -> float: + assert signal.getsignal(signal.SIGINT) is handler + return super().__float__() + + with ThreadPoolExecutor(max_workers=1) as pool: + suite = pool.submit(run_all, warmup_seconds=Seconds('warmup', (0.0, 0.0), events), + measurement_seconds=Seconds('measurement', (0.1, 0.1), events)).result(timeout=5) + assert len(suite.results) == 2 + assert all(result.success for result in suite.results) + assert events == ['warmup:1', 'measurement:1', 'warmup:2', 'measurement:2'] + assert signal.getsignal(signal.SIGINT) is handler diff --git a/tests/test_suite_signal_policy.py b/tests/test_suite_signal_policy.py new file mode 100644 index 0000000..1a45d99 --- /dev/null +++ b/tests/test_suite_signal_policy.py @@ -0,0 +1,297 @@ +"""Recheck caller SIGINT policy at each legacy benchmark execution boundary.""" + +import signal +from types import FrameType +from typing import Callable, Dict, List, Optional, Type, Union + +import pytest + +from benchset import ( + BenchmarkResult, + BenchmarkRunner, + Progress, + SuiteResult, + api, + run_all, +) +from benchset._native import Native +from tests.helpers import assert_runner_reusable + +_Handler = Union[int, Callable[[int, Optional[FrameType]], None], None] +_POLICIES = ('default', 'custom', 'ignored') + + +def _handler(policy: str, custom: Callable[[int, Optional[FrameType]], None]) -> _Handler: + handlers: Dict[str, _Handler] = {'default': signal.default_int_handler, 'custom': custom, + 'ignored': signal.SIG_IGN} + return handlers[policy] + + +def _assert_finished_short_measurement(result: BenchmarkResult) -> None: + """Accept completion or the documented no-work failure, never a signal error.""" + if result.success: + assert result.score is not None + assert result.pi_equivalent is not None + else: + assert result.status == 'failed' + assert result.reason == 'measurement window was too short to complete any work' + assert result.score is result.pi_equivalent is None + assert result.iterations == 0 or result.elapsed_seconds == 0 + + +def _assert_execution_handler(actual: _Handler, policy: str, caller_handler: _Handler) -> None: + if policy == 'default': + # run() defers the default exception until cancellation and native cleanup. + assert callable(actual) + assert actual is not signal.default_int_handler + else: + assert actual is caller_handler + + +@pytest.mark.parametrize('initial', _POLICIES) +@pytest.mark.parametrize('target', _POLICIES) +@pytest.mark.parametrize('change_at', ['callback', 'warmup', 'measurement']) +def test_suite_rechecks_signal_policy_after_each_childs_conversions( # noqa: PLR0915 — One scope owns signal restoration and runner reuse. + monkeypatch: pytest.MonkeyPatch, initial: str, target: str, change_at: str, +) -> None: + """A previous callback or either second-child converter may change policy. + + HEAD called run() separately for each child: both float conversions ran + under the caller's current policy before run() decided whether to own + default Ctrl+C. The shared suite must make that same decision for its next + child. Custom and ignored policies retain their exact handler objects; + default SIGINT cancels and escapes as KeyboardInterrupt, never as a failed + progress callback. A fresh call on the same runner checks complete cleanup. + The measurement window allows worker startup on loaded Windows runners; + this test checks signal policy rather than short-window throughput. + """ + events: List[str] = [] + observed: Dict[str, _Handler] = {} + conversion_handlers: Dict[str, _Handler] = {} + conversions = {'warmup': 0, 'measurement': 0} + runner = BenchmarkRunner() + monkeypatch.setattr(api, 'BenchmarkRunner', lambda: runner) + + def initial_custom(_signum: int, _frame: Optional[FrameType]) -> None: + events.append('initial-custom') + + def target_custom(_signum: int, _frame: Optional[FrameType]) -> None: + events.append('target-custom') + + initial_handler = _handler(initial, initial_custom) + target_handler = _handler(target, target_custom) + + class Seconds(float): + def __new__(cls, field: str) -> 'Seconds': + return super().__new__(cls, 0.0 if field == 'warmup' else .1) + + def __init__(self, field: str) -> None: + self.field = field + + def __float__(self) -> float: + conversions[self.field] += 1 + ordinal = conversions[self.field] + conversion_handlers[self.field + ':' + str(ordinal)] = signal.getsignal(signal.SIGINT) + if ordinal == 2 and self.field == change_at: + signal.signal(signal.SIGINT, target_handler) + return float.__float__(self) + + signalled = False + + def progress(value: Progress) -> None: + nonlocal signalled + if value.benchmark == 'single_core' and value.phase == 'completed' and change_at == 'callback': + signal.signal(signal.SIGINT, target_handler) + if value.benchmark == 'multi_core' and value.phase in ('warmup', 'measurement') and not signalled: + signalled = True + observed['second'] = signal.getsignal(signal.SIGINT) + events.append('before-signal') + signal.raise_signal(signal.SIGINT) + events.append('after-signal') + + previous = signal.signal(signal.SIGINT, initial_handler) + suite: Optional[SuiteResult] = None + interrupted = False + try: + try: + suite = run_all(warmup_seconds=Seconds('warmup'), measurement_seconds=Seconds('measurement'), + on_progress=progress) + except KeyboardInterrupt: + interrupted = True + assert signal.getsignal(signal.SIGINT) is target_handler + assert_runner_reusable(runner) + assert signal.getsignal(signal.SIGINT) is target_handler + finally: + signal.signal(signal.SIGINT, previous) + + assert conversions == {'warmup': 2, 'measurement': 2} + assert conversion_handlers['warmup:1'] is initial_handler + assert conversion_handlers['measurement:1'] is initial_handler + assert conversion_handlers['warmup:2'] is (target_handler if change_at == 'callback' else initial_handler) + assert conversion_handlers['measurement:2'] is ( + target_handler if change_at in ('callback', 'warmup') else initial_handler + ) + assert signalled + _assert_execution_handler(observed['second'], target, target_handler) + assert interrupted == (target == 'default'), suite + assert events == ['before-signal'] + (['target-custom'] if target == 'custom' else []) + ['after-signal'] + if target != 'default': + assert suite is not None + assert len(suite.results) == 2 + assert suite.results[0].success + _assert_finished_short_measurement(suite.results[1]) + + +@pytest.mark.parametrize('initial', _POLICIES) +@pytest.mark.parametrize('target', _POLICIES) +def test_single_run_rechecks_signal_policy_after_name_validation(initial: str, target: str) -> None: + """A typed name operator changes policy before native execution owns signals. + + Legacy name validation preceded the SIGINT decision. In particular a custom + or ignored initial policy changed to default must acquire deferred native + cancellation, while a new custom/ignored policy must remain untouched. + """ + events: List[str] = [] + observed: List[_Handler] = [] + comparison_handlers: List[_Handler] = [] + comparisons = 0 + runner = BenchmarkRunner() + + def initial_custom(_signum: int, _frame: Optional[FrameType]) -> None: + events.append('initial-custom') + + def target_custom(_signum: int, _frame: Optional[FrameType]) -> None: + events.append('target-custom') + + initial_handler = _handler(initial, initial_custom) + target_handler = _handler(target, target_custom) + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + nonlocal comparisons + comparisons += 1 + comparison_handlers.append(signal.getsignal(signal.SIGINT)) + if comparisons == 1: + signal.signal(signal.SIGINT, target_handler) + return str.__eq__(self, other) + + signalled = False + + def progress(value: Progress) -> None: + nonlocal signalled + if value.phase in ('warmup', 'measurement') and not signalled: + signalled = True + observed.append(signal.getsignal(signal.SIGINT)) + events.append('before-signal') + signal.raise_signal(signal.SIGINT) + events.append('after-signal') + + previous = signal.signal(signal.SIGINT, initial_handler) + result = None + interrupted = False + try: + try: + result = runner.run(Name('single_core'), warmup_seconds=0, measurement_seconds=.01, + on_progress=progress) + except KeyboardInterrupt: + interrupted = True + assert signal.getsignal(signal.SIGINT) is target_handler + assert_runner_reusable(runner) + assert signal.getsignal(signal.SIGINT) is target_handler + finally: + signal.signal(signal.SIGINT, previous) + + assert comparisons == 2 + assert comparison_handlers[0] is initial_handler + _assert_execution_handler(comparison_handlers[1], target, target_handler) + assert signalled + _assert_execution_handler(observed[0], target, target_handler) + assert interrupted == (target == 'default'), result + assert events == ['before-signal'] + (['target-custom'] if target == 'custom' else []) + ['after-signal'] + if target != 'default': + assert result is not None + _assert_finished_short_measurement(result) + + +@pytest.mark.parametrize('initial', ['custom', 'ignored']) +@pytest.mark.parametrize('window', ['warmup', 'measurement']) +@pytest.mark.parametrize('error_type', [RuntimeError, KeyboardInterrupt]) +@pytest.mark.parametrize('first_no_work', [False, True]) +def test_new_default_policy_protects_cleanup_after_a_converter_raises( + monkeypatch: pytest.MonkeyPatch, initial: str, window: str, error_type: Type[BaseException], first_no_work: bool, +) -> None: + """A converter's newly selected default policy also applies during cleanup. + + HEAD had freed child one before this conversion. A shared suite still owns + its handle and must defer a cleanup SIGINT without replacing the caller's + original error or leaving activity behind. The free hook always completes + real native freeing, including when a broken signal policy raises there. + A nanosecond control pins retries of an exact no-work first result before + the target conversion; no signal, cleanup error or later result is retried. + """ + runner = BenchmarkRunner() + monkeypatch.setattr(api, 'BenchmarkRunner', lambda: runner) + original = error_type('second conversion failed after selecting default') + original_free = Native.free + conversions = {'warmup': 0, 'measurement': 0} + raised = False + cleanup_signals = 0 + + def custom(_signum: int, _frame: Optional[FrameType]) -> None: + pass + + class Seconds(float): + def __new__(cls, field: str) -> 'Seconds': + return super().__new__(cls, 0.0 if field == 'warmup' else + (1e-9 if first_no_work and attempt == 0 else .1)) + + def __init__(self, field: str) -> None: + self.field = field + + def __float__(self) -> float: + nonlocal raised + conversions[self.field] += 1 + if self.field == window and conversions[self.field] == 2: + signal.signal(signal.SIGINT, signal.default_int_handler) + raised = True + raise original + return float.__float__(self) + + def free(native: Native, handle: int, suite: bool = False) -> None: + nonlocal cleanup_signals + try: + if raised: + cleanup_signals += 1 + signal.raise_signal(signal.SIGINT) + finally: + original_free(native, handle, True) if suite else original_free(native, handle) + + monkeypatch.setattr(Native, 'free', free) + previous = signal.signal(signal.SIGINT, _handler(initial, custom)) + caught: Optional[BaseException] = None + try: + for attempt in range(3): + conversions = {'warmup': 0, 'measurement': 0} + try: + suite = run_all(warmup_seconds=Seconds('warmup'), measurement_seconds=Seconds('measurement')) + except BaseException as error: # noqa: BLE001 — Compare every replacement with the exact caller error. + assert not (first_no_work and attempt == 0), 'The suite continued after a no-work failure' + caught = error + break + assert len(suite.results) == 1, suite + assert suite.results[0].status == 'failed', suite + assert suite.results[0].reason == 'measurement window was too short to complete any work', suite + assert conversions == {'warmup': 1, 'measurement': 1} + assert not raised + assert cleanup_signals == 0 + raised = False + assert caught is original, (caught, conversions, cleanup_signals) + assert caught.__context__ is None + assert cleanup_signals == (1 if hasattr(Native, 'execute_suite') else 0) + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + assert_runner_reusable(runner) + finally: + signal.signal(signal.SIGINT, previous) diff --git a/tests/test_validation_lifetime.py b/tests/test_validation_lifetime.py new file mode 100644 index 0000000..7cde0bf --- /dev/null +++ b/tests/test_validation_lifetime.py @@ -0,0 +1,307 @@ +"""Caller-owned name operators run before the reusable runner claims execution.""" + +import signal +from threading import Event, Thread +from typing import List, Optional + +import pytest + +from benchset import BenchmarkResult, BenchmarkRunner, Progress +from tests.helpers import assert_runner_reusable + + +@pytest.mark.parametrize('cancel_at', [1, 2]) +def test_name_operator_cancellation_keeps_validation_before_runner_ownership(cancel_at: int) -> None: + """Validation cancellation is idle; execution cancellation targets the newly claimed run.""" + runner = BenchmarkRunner() + + class Name(str): + __slots__ = ('__dict__',) + __hash__ = str.__hash__ + comparisons = 0 + + def __eq__(self, other: object) -> bool: + self.comparisons += 1 + if self.comparisons == cancel_at: + runner.cancel() + return str.__eq__(self, other) + + name = Name('single_core') + result = runner.run(name, warmup_seconds=0, measurement_seconds=.1) + assert result.status == ('completed' if cancel_at == 1 else 'cancelled') + assert result.benchmark is name + assert name.comparisons == 2 + assert_runner_reusable(runner) + + +def test_name_operator_can_complete_a_nested_run_before_outer_ownership() -> None: + """A typed name may run the idle runner while its outer request is still being validated.""" + runner = BenchmarkRunner() + nested: List[BenchmarkResult] = [] + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + if not nested: + nested.append(runner.run('multi_core', warmup_seconds=0, measurement_seconds=.1)) + return str.__eq__(self, other) + + result = runner.run(Name('single_core'), warmup_seconds=0, measurement_seconds=.1) + assert len(nested) == 1 + assert nested[0].success + assert result.success + assert_runner_reusable(runner) + + +@pytest.mark.parametrize('outcome', ['valid', 'invalid', 'error']) +def test_name_operator_may_publish_another_owner_during_validation(outcome: str) -> None: + """Recheck activity after host comparison and never release the owner it starts. + + Validation may return a valid name, reject it, or raise the caller's own + exception while that other run stays active. Each losing path must leave + cancellation and cleanup in the other run's hands. + """ + runner = BenchmarkRunner() + ready, release = Event(), Event() + results: List[BenchmarkResult] = [] + worker: Optional[Thread] = None + error = ValueError('name comparison failed') + + def progress(_value: Progress) -> None: + ready.set() + assert release.wait(5) + + def execute() -> None: + try: + results.append(runner.run('multi_core', warmup_seconds=0, measurement_seconds=.1, + on_progress=progress)) + finally: + ready.set() + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + nonlocal worker + if worker is None: + worker = Thread(target=execute) + worker.start() + assert ready.wait(5) + if outcome == 'error': + raise error + return outcome == 'valid' and str.__eq__(self, other) + + try: + if outcome == 'error': + with pytest.raises(ValueError, match='name comparison failed') as caught: + runner.run(Name('single_core'), warmup_seconds=0, measurement_seconds=.1) + assert caught.value is error + assert caught.value.__context__ is None + else: + result = runner.run(Name('single_core'), warmup_seconds=0, measurement_seconds=.1) + expected = 'this runner is already running' if outcome == 'valid' else 'unknown benchmark: single_core' + assert result.reason == expected + assert not results + assert runner.run('single_core', warmup_seconds=0, measurement_seconds=.1).reason == 'this runner is already running' + finally: + runner.cancel() + release.set() + if worker is not None: + worker.join(5) + assert worker is not None + assert not worker.is_alive() + assert len(results) == 1 + assert results[0].status == 'cancelled' + assert_runner_reusable(runner) + + +def test_name_operator_can_finish_an_existing_owner_before_claim() -> None: + """A request that starts out busy may claim the runner once its comparison finishes that owner.""" + runner = BenchmarkRunner() + entered, release = Event(), Event() + results: List[BenchmarkResult] = [] + + def progress(_value: Progress) -> None: + entered.set() + assert release.wait(5) + + worker = Thread(target=lambda: results.append(runner.run('multi_core', warmup_seconds=0, + measurement_seconds=.1, on_progress=progress))) + + class Name(str): + __slots__ = ('__dict__',) + __hash__ = str.__hash__ + comparisons = 0 + + def __eq__(self, other: object) -> bool: + self.comparisons += 1 + if self.comparisons == 1: + runner.cancel() + release.set() + worker.join(5) + return str.__eq__(self, other) + + worker.start() + try: + assert entered.wait(5) + result = runner.run(Name('single_core'), warmup_seconds=0, measurement_seconds=.1) + assert result.success + assert len(results) == 1 + assert results[0].status == 'cancelled' + finally: + runner.cancel() + release.set() + worker.join(5) + assert not worker.is_alive() + assert_runner_reusable(runner) + + +@pytest.mark.parametrize('policy', ['default', 'handled', 'custom', 'ignored']) +def test_name_validation_operator_observes_the_callers_sigint_policy(policy: str) -> None: + """A validation signal interrupts its operator immediately, unless the caller handles or ignores it.""" + runner = BenchmarkRunner() + events: List[str] = [] + raised: List[KeyboardInterrupt] = [] + + class Name(str): + __slots__ = ('__dict__',) + __hash__ = str.__hash__ + comparisons = 0 + + def __eq__(self, other: object) -> bool: + self.comparisons += 1 + if self.comparisons == 1: + assert signal.getsignal(signal.SIGINT) is handler + events.append('compare') + try: + signal.raise_signal(signal.SIGINT) + except KeyboardInterrupt as error: + raised.append(error) + if policy != 'handled': + raise + events.append('after-signal') + return str.__eq__(self, other) + + def custom_handler(_signum: int, _frame: object) -> None: + events.append('custom-handler') + + handler = custom_handler if policy == 'custom' else ( + signal.SIG_IGN if policy == 'ignored' else signal.default_int_handler + ) + previous = signal.signal(signal.SIGINT, handler) + try: + if policy == 'default': + with pytest.raises(KeyboardInterrupt) as caught: + runner.run(Name('single_core'), warmup_seconds=0, measurement_seconds=.1) + assert len(raised) == 1 + assert caught.value is raised[0] + assert caught.value.__context__ is None + assert events == ['compare'] + else: + try: + result = runner.run(Name('single_core'), warmup_seconds=0, measurement_seconds=.1) + except KeyboardInterrupt: + pytest.fail('the caller handled or ignored the validation signal') + assert result.success + assert len(raised) == (1 if policy == 'handled' else 0) + expected = ['compare', 'custom-handler', 'after-signal'] if policy == 'custom' else ['compare', 'after-signal'] + assert events == expected + assert signal.getsignal(signal.SIGINT) is handler + assert_runner_reusable(runner) + finally: + signal.signal(signal.SIGINT, previous) + + +@pytest.mark.parametrize('signal_at', ['nested', 'outer']) +def test_nested_run_from_name_validation_keeps_default_sigint_cancellation(signal_at: str) -> None: + """A nested runner still owns default signal cancellation while an outer operator is suspended.""" + runner = BenchmarkRunner() + events: List[str] = [] + + def progress(_value: Progress) -> None: + if signal_at == 'nested' and events[-1] == 'nested': + events.append('nested-progress') + signal.raise_signal(signal.SIGINT) + + def outer_progress(_value: Progress) -> None: + if signal_at == 'outer' and events[-1] == 'after-nested': + events.append('outer-progress') + signal.raise_signal(signal.SIGINT) + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + if not events: + events.append('nested') + runner.run('multi_core', warmup_seconds=0, measurement_seconds=.1, on_progress=progress) + events.append('after-nested') + return str.__eq__(self, other) + + previous = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + with pytest.raises(KeyboardInterrupt): + runner.run(Name('single_core'), warmup_seconds=0, measurement_seconds=.1, on_progress=outer_progress) + expected = ['nested', 'nested-progress'] if signal_at == 'nested' else ['nested', 'after-nested', 'outer-progress'] + assert events == expected + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + assert_runner_reusable(runner) + finally: + signal.signal(signal.SIGINT, previous) + + +@pytest.mark.parametrize('second_signal', ['after-catching', 'finally']) +def test_repeated_name_operator_sigint_preserves_default_exception_chain(second_signal: str) -> None: + """Validation operators keep each default SIGINT and the caller's exception context chain.""" + runner = BenchmarkRunner() + events: List[str] = [] + raised: List[KeyboardInterrupt] = [] + + class Name(str): + __slots__ = () + __hash__ = str.__hash__ + + def __eq__(self, other: object) -> bool: + try: + try: + signal.raise_signal(signal.SIGINT) + except KeyboardInterrupt as error: + raised.append(error) + if second_signal == 'finally': + raise + events.append('after-first-signal') + try: + signal.raise_signal(signal.SIGINT) + except KeyboardInterrupt as error: + raised.append(error) + raise + events.append('after-second-signal') + finally: + if second_signal == 'finally': + events.append('operator-finally') + try: + signal.raise_signal(signal.SIGINT) + except KeyboardInterrupt as error: + raised.append(error) + raise + events.append('after-finally-signal') + return str.__eq__(self, other) + + previous = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + with pytest.raises(KeyboardInterrupt) as caught: + runner.run(Name('single_core'), warmup_seconds=0, measurement_seconds=.1) + assert len(raised) == 2 + assert caught.value is raised[1] + assert raised[0].__context__ is None + assert caught.value.__context__ is (raised[0] if second_signal == 'finally' else None) + assert caught.value.__cause__ is None + assert events == ['operator-finally' if second_signal == 'finally' else 'after-first-signal'] + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + assert_runner_reusable(runner) + finally: + signal.signal(signal.SIGINT, previous) diff --git a/tests/typing/reference_positive.py b/tests/typing/reference_positive.py index ee9e697..2b3d6f6 100644 --- a/tests/typing/reference_positive.py +++ b/tests/typing/reference_positive.py @@ -4,6 +4,7 @@ from typing import Optional, Tuple from benchset import ( + BENCHMARKS, PI_REFERENCE_MACHINE, PI_REFERENCE_RUN, AggregatedScores, @@ -16,6 +17,8 @@ run_all, ) +benchmarks: Tuple[str, str] = BENCHMARKS + def metadata(result: BenchmarkResult) -> Tuple[ReferenceMachine, ReferenceRun]: machine: ReferenceMachine = result.reference_machine diff --git a/tests/typing/test_reference_types.py b/tests/typing/test_reference_types.py index f0125d9..31ac4c6 100644 --- a/tests/typing/test_reference_types.py +++ b/tests/typing/test_reference_types.py @@ -13,7 +13,7 @@ def test_result_reference_public_type_contract( tmp_path: Path, fixture: str, expected_status: int, ) -> None: - """Catch public annotation regressions, including suite tuple and score types.""" + """Catch public annotation regressions, including the fixed benchmark tuple and score types.""" source = Path(__file__).with_name(fixture) result = subprocess.run([ sys.executable, '-m', 'mypy', '--strict', '--disallow-any-expr', @@ -32,15 +32,19 @@ def test_result_reference_public_type_contract( assert any('"reference_machine"' in error and 'read-only' in error and '[misc]' in error for error in errors) -def test_positive_result_reference_consumer_runs_without_native_code() -> None: - """The typed failure/metadata consumer also runs without loading a native library.""" +def test_positive_result_reference_consumer_imports_without_native_code() -> None: + """Typed metadata imports stay native-free; execution may load Rust for validation.""" program = """ import ctypes -import runpy +load_native = ctypes.CDLL def forbidden(*args, **kwargs): raise AssertionError('typed metadata consumer must not load native code') ctypes.CDLL = forbidden -runpy.run_module('tests.typing.reference_positive', run_name='__main__') +from tests.typing import reference_positive +from benchset import BenchmarkResult +reference_positive.metadata(BenchmarkResult('single_core')) +ctypes.CDLL = load_native +reference_positive.main() """ result = subprocess.run([sys.executable, '-c', program], capture_output=True, text=True, check=False) assert result.returncode == 0, result.stderr