From 0d090e3d4d57f113c781b45297dc484d60ce8a08 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Sat, 17 Oct 2026 04:47:55 +0300 Subject: [PATCH 1/2] Pass a shared caller deadline to PBT campaign execution --- usvm-ts-fast-check/scripts/pbt_experiment.py | 47 ++- .../scripts/test_pbt_experiment.py | 272 +++++++++++++++++- 2 files changed, 307 insertions(+), 12 deletions(-) diff --git a/usvm-ts-fast-check/scripts/pbt_experiment.py b/usvm-ts-fast-check/scripts/pbt_experiment.py index 7e3abc97f2..8c343d9425 100644 --- a/usvm-ts-fast-check/scripts/pbt_experiment.py +++ b/usvm-ts-fast-check/scripts/pbt_experiment.py @@ -16,6 +16,7 @@ MODES = ("PBT_ONLY", "SYMBOLIC_ONLY", "SEQUENTIAL", "COVERAGE_FEEDBACK") STOP_CRITERION = "first original fault, PBT successful-run quota, symbolic exhaustion, or deadline" SCRIPT_ROOT = Path(__file__).resolve().parent +PUBLICATION_RESERVE_MILLIS = 1000 def read_json(path): @@ -30,21 +31,29 @@ def digest(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() -def run_process(command, directory, stem, timeout_seconds): - """Retain process failures and overruns; never credit their late native output.""" +def run_process(command, directory, stem, timeout_seconds, *, deadline=None): + """Retain failures and overruns; an absolute deadline also charges process creation.""" started = time.monotonic() process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True) + wait_timeout = max(0.001, timeout_seconds) if deadline is None else max(0, deadline - time.monotonic()) timed_out = False try: - stdout, stderr = process.communicate(timeout=max(0.001, timeout_seconds)) + stdout, stderr = process.communicate(timeout=wait_timeout) except subprocess.TimeoutExpired: timed_out = True - os.killpg(process.pid, signal.SIGKILL) + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass stdout, stderr = process.communicate() + completed = time.monotonic() + if deadline is not None and completed >= deadline: + timed_out = True (directory / f"{stem}.stdout").write_bytes(stdout) (directory / f"{stem}.stderr").write_bytes(stderr) return {"command": [str(part) for part in command], "exitCode": process.returncode, - "timedOut": timed_out, "elapsedMillis": int((time.monotonic() - started) * 1000)} + "timedOut": timed_out, "elapsedMillis": int((time.monotonic() - started) * 1000), + "waitTimeoutMillis": int(wait_timeout * 1000)} def verify_sources(case): @@ -205,19 +214,27 @@ def run_cell(frozen, case, seed, mode, output): directory = output / case["id"] / str(seed) / mode directory.mkdir(parents=True) started = time.monotonic() + started_epoch_millis = int(time.time() * 1000) preparation_cost = case["preparation"]["elapsedMillis"] remaining_budget = max(0, frozen["totalBudgetMillis"] - preparation_cost) deadline = started + remaining_budget / 1000 runner_budget = remaining_budget - frozen["classReplayReserveMillis"] + runner_deadline = deadline - frozen["classReplayReserveMillis"] / 1000 + runner_deadline_epoch_millis = started_epoch_millis + runner_budget summary = {"caseId": case["id"], "project": case["project"], "population": case["population"], "seed": seed, "mode": mode, "stopCriterion": STOP_CRITERION, "preparationChargedMillis": preparation_cost, "declaredTotalBudgetMillis": frozen["totalBudgetMillis"], "pbtRunQuota": frozen["numRuns"], + "runnerDeadlineEpochMillis": runner_deadline_epoch_millis, + "classReplayReserveMillis": frozen["classReplayReserveMillis"], + "publicationReserveMillis": PUBLICATION_RESERVE_MILLIS, "totalStatements": case.get("totalStatements"), "universeSha256": case.get("universeSha256"), "coveredStatementIds": [], "coverageComplete": False, "confirmedViolation": False} try: verify_sources(case) verify_runtime(frozen) - if runner_budget <= frozen["validationReserveMillis"] + frozen["pbtAllocationMillis"]: + required_runner_reserves = frozen["validationReserveMillis"] + frozen["pbtAllocationMillis"] \ + + PUBLICATION_RESERVE_MILLIS + if runner_budget <= required_runner_reserves: raise ValueError("Preparation exhausted the fixed campaign allocation") sources = [{"module": relative, "sha256": sha} for relative, sha in case["sourceSha256"].items()] configuration = {"seed": seed, "numRuns": frozen["numRuns"], "timeoutMillis": runner_budget, @@ -229,15 +246,20 @@ def run_cell(frozen, case, seed, mode, output): request = {"sourceRoot": case["sourceRoot"], "property": case["property"], "mode": mode, "configuration": configuration, "validationReserveMillis": frozen["validationReserveMillis"], "completedInputLimit": 0 if mode == "PBT_ONLY" else frozen["symbolicInputLimit"], - "nodeExecutable": frozen["nodeCommand"], "shrinkWitness": False} + "nodeExecutable": frozen["nodeCommand"], "shrinkWitness": False, + "externalDeadlineEpochMillis": runner_deadline_epoch_millis, + "publicationReserveMillis": PUBLICATION_RESERVE_MILLIS} if mode in ("SEQUENTIAL", "COVERAGE_FEEDBACK"): request["pbtAllocationMillis"] = frozen["pbtAllocationMillis"] write_json(directory / "request.json", request) summary["runnerProcess"] = run_process( [Path(frozen["distribution"]) / "bin/usvm-ts-fast-check", "campaign-file", directory / "request.json", directory / "campaign.json"], directory, "runner", - max(0, deadline - time.monotonic() - frozen["classReplayReserveMillis"] / 1000), + timeout_seconds=runner_budget / 1000, deadline=runner_deadline, ) + runner_process = summary["runnerProcess"] + runner_timely = time.monotonic() < runner_deadline and not runner_process["timedOut"] + runner_process["creditedWithinDeadline"] = False if not (directory / "campaign.json").exists(): raise ValueError("Runner produced no completed artifact; raw process output is retained") artifact = read_json(directory / "campaign.json") @@ -260,12 +282,17 @@ def run_cell(frozen, case, seed, mode, output): stopped_on_pbt_fault = result.get("symbolic") is None and failure.get("kind") == "property" summary["analysisSupported"] = (mode == "PBT_ONLY" or symbolic_supported or stopped_on_pbt_fault) \ and not summary["symbolicDiagnostics"] + runner_completed = runner_process["exitCode"] == 0 \ + or (runner_process["exitCode"] == 1 and summary["confirmedViolation"]) + runner_process["creditedWithinDeadline"] = runner_timely and runner_completed if summary["confirmedViolation"]: summary["findingReturnedByMillis"] = int((time.monotonic() - started) * 1000) + preparation_cost summary["confirmedViolationWithinBudget"] = summary["findingReturnedByMillis"] <= frozen["totalBudgetMillis"] \ - and not summary["runnerProcess"]["timedOut"] + and runner_process["creditedWithinDeadline"] batches, omissions = replay_inputs(artifact) summary["omissions"] = omissions + if not runner_process["creditedWithinDeadline"]: + raise ValueError("Runner did not complete successfully within its deadline; raw artifact and output are retained") prepared = read_json(output / case["id"] / "prepared.json") if not prepared.get("supported"): raise ValueError("Frozen class universe/preflight is unsupported") @@ -284,7 +311,7 @@ def run_cell(frozen, case, seed, mode, output): process = run_process( [frozen["nodeCommand"], SCRIPT_ROOT / "class-coverage-replay.mjs", "replay", frozen["adapter"], directory / f"{stem}.request.json", directory / f"{stem}.replay.json"], - directory, stem, deadline - time.monotonic(), + directory, stem, timeout_seconds=remaining_budget / 1000, deadline=deadline, ) timely = time.monotonic() < deadline and not process["timedOut"] and process["exitCode"] == 0 process["batchId"] = stem diff --git a/usvm-ts-fast-check/scripts/test_pbt_experiment.py b/usvm-ts-fast-check/scripts/test_pbt_experiment.py index ad13b70535..64356a52d1 100644 --- a/usvm-ts-fast-check/scripts/test_pbt_experiment.py +++ b/usvm-ts-fast-check/scripts/test_pbt_experiment.py @@ -1,9 +1,10 @@ import importlib.util -import os from pathlib import Path import struct +import sys import tempfile import unittest +from unittest.mock import Mock, patch ROOT = Path(__file__).resolve().parent @@ -16,10 +17,27 @@ def number(value): return {"kind": "number", "value": "finite", "bits": struct.pack(">d", value).hex()} +class FakeClock: + def __init__(self): + self.now = 100.0 + + def monotonic(self): + return self.now + + def time(self): + return 1_700_000_000 + self.now - 100 + + def advance(self, seconds): + self.now += seconds + + class ExperimentTest(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() self.root = Path(self.temporary.name).resolve() + self.clock = FakeClock() + self.process_calls = [] + self.cell_count = 0 def tearDown(self): self.temporary.cleanup() @@ -51,7 +69,8 @@ def test_frozen_runtime_changes_are_rejected_before_treatment(self): def test_process_timeout_retains_raw_output_and_failure(self): process = driver.run_process( - ["node", "-e", "process.stdout.write('started'); setInterval(() => {}, 1000)"], self.root, "slow", 0.2, + [sys.executable, "-c", "import time; print('started', end='', flush=True); time.sleep(30)"], + self.root, "slow", timeout_seconds=0.2, ) self.assertTrue(process["timedOut"]) @@ -59,6 +78,255 @@ def test_process_timeout_retains_raw_output_and_failure(self): self.assertEqual((self.root / "slow.stdout").read_text(), "started") + def test_absolute_process_deadline_charges_spawn_time(self): + child = Mock(returncode=0) + child.communicate.return_value = (b"completed", b"") + + def spawn(*args, **kwargs): + self.clock.advance(3) + return child + + with patch.object(driver.time, "monotonic", self.clock.monotonic), \ + patch.object(driver.subprocess, "Popen", side_effect=spawn): + process = driver.run_process( + ["runner"], self.root, "spawn", timeout_seconds=20, deadline=105, + ) + + child.communicate.assert_called_once_with(timeout=2) + self.assertEqual(process["waitTimeoutMillis"], 2000) + self.assertEqual(process["elapsedMillis"], 3000) + self.assertFalse(process["timedOut"]) + + + def test_relative_process_timeout_preserves_existing_callers(self): + child = Mock(returncode=0) + child.communicate.return_value = (b"", b"") + + def spawn(*args, **kwargs): + self.clock.advance(3) + return child + + with patch.object(driver.time, "monotonic", self.clock.monotonic), \ + patch.object(driver.subprocess, "Popen", side_effect=spawn): + process = driver.run_process(["runner"], self.root, "relative", timeout_seconds=5) + + child.communicate.assert_called_once_with(timeout=5) + self.assertFalse(process["timedOut"]) + + + def test_spawn_exhausting_deadline_kills_without_a_new_wait_allocation(self): + child = Mock(pid=1234, returncode=-9) + child.communicate.side_effect = [driver.subprocess.TimeoutExpired("runner", 0), (b"partial", b"stopped")] + + def spawn(*args, **kwargs): + self.clock.advance(6) + return child + + with patch.object(driver.time, "monotonic", self.clock.monotonic), \ + patch.object(driver.subprocess, "Popen", side_effect=spawn), \ + patch.object(driver.os, "killpg") as kill: + process = driver.run_process(["runner"], self.root, "expired", timeout_seconds=20, deadline=105) + + self.assertEqual(child.communicate.call_args_list[0].kwargs, {"timeout": 0}) + kill.assert_called_once_with(1234, driver.signal.SIGKILL) + self.assertTrue(process["timedOut"]) + self.assertEqual((self.root / "expired.stdout").read_bytes(), b"partial") + + + def test_late_zero_exit_is_still_a_timeout(self): + child = Mock(returncode=0) + + def communicate(*args, **kwargs): + self.clock.advance(6) + return b"late", b"" + + child.communicate.side_effect = communicate + + with patch.object(driver.time, "monotonic", self.clock.monotonic), \ + patch.object(driver.subprocess, "Popen", return_value=child): + process = driver.run_process(["runner"], self.root, "late", timeout_seconds=20, deadline=105) + + self.assertEqual(process["exitCode"], 0) + self.assertTrue(process["timedOut"]) + + + def campaign(self, *, fault=False): + inputs = [number(1)] + artifact = {"result": {"mode": "COVERAGE_FEEDBACK", "stop": "COMPLETED", + "concrete": {"status": "success", "numRuns": 12, + "observations": {"invocations": [ + {"phase": "generation", "admission": "admitted", "outcome": "holds", + "input": {"status": "captured", "value": {"kind": "array", "elements": inputs}}}, + ]}}, + "symbolic": {"status": "NO_VIOLATION_REACHED", "completedInputs": [inputs]}, + "symbolicInputReplays": [ + {"candidate": {"inputs": inputs}, "completedWithinDeadline": True, + "predictedOutcomes": ["HOLDS"], "replay": {"admission": "ADMITTED", "outcome": "HOLDS"}}, + ]}} + + if fault: + result = artifact["result"] + result["concrete"]["status"] = "failure" + result["concrete"]["failure"] = {"kind": "property"} + result["concrete"]["observations"]["invocations"][0]["outcome"] = "false" + result["symbolic"] = None + result.pop("symbolicInputReplays") + + return artifact + + + def complete_process(self, command, directory, stem, timeout_seconds, *, deadline=None): + self.process_calls.append({"stem": stem, "timeout": timeout_seconds, "deadline": deadline}) + response = self.campaign() if stem == "runner" else {"records": [ + {"verified": True, "creditedStatementIds": [stem]}, + ]} + driver.write_json(command[-1], response) + self.clock.advance(1) + return {"exitCode": 0, "timedOut": False} + + + def invoke_cell(self, process): + self.cell_count += 1 + output = self.root / f"output-{self.cell_count}" + prepared = output / "subject" / "prepared.json" + prepared.parent.mkdir(parents=True) + driver.write_json(prepared, {"supported": True, "verificationBoundary": "SCENARIO_OBSERVATIONS"}) + source = self.root / "Subject.ts" + source.write_text("export function holds(value: number) { return value === value; }", encoding="utf-8") + case = {"id": "subject", "project": "fixture", "population": "correct-control", + "sourceRoot": str(self.root), "sourceSha256": {"Subject.ts": driver.digest(source)}, + "property": {"id": "subject.holds"}, + "preparation": {"elapsedMillis": 2000, "nativeSupported": True, "frontendSupported": True}} + frozen = {"totalBudgetMillis": 30000, "classReplayReserveMillis": 8000, "validationReserveMillis": 1500, + "pbtAllocationMillis": 4000, "numRuns": 12, "pbtInputLimit": 64, "symbolicInputLimit": 16, + "maxArrayElements": 64, "maxObservationBytes": 65536, "nodeCommand": "node", + "distribution": "/distribution", "adapter": "/adapter", "runtimeSha256": {}} + self.cell_directory = output / "subject" / "37" / "COVERAGE_FEEDBACK" + + with patch.object(driver.time, "monotonic", self.clock.monotonic), \ + patch.object(driver.time, "time", self.clock.time), \ + patch.object(driver, "run_process", side_effect=process): + return driver.run_cell(frozen, case, seed=37, mode="COVERAGE_FEEDBACK", output=output) + + + def test_cell_shares_cutoffs_and_passes_reserves_without_rewriting_controls(self): + with patch.object(driver, "verify_runtime", side_effect=lambda frozen: self.clock.advance(3)): + summary = self.invoke_cell(self.complete_process) + + request = driver.read_json(self.cell_directory / "request.json") + self.assertEqual(request["externalDeadlineEpochMillis"], 1_700_000_020_000) + self.assertEqual(request["publicationReserveMillis"], 1000) + self.assertEqual(request["configuration"]["timeoutMillis"], 20000) + self.assertEqual(request["configuration"]["seed"], 37) + self.assertEqual(request["configuration"]["numRuns"], 12) + self.assertEqual(request["validationReserveMillis"], 1500) + self.assertEqual(request["pbtAllocationMillis"], 4000) + self.assertEqual(request["completedInputLimit"], 16) + self.assertFalse(request["shrinkWitness"]) + self.assertEqual([call["deadline"] for call in self.process_calls], [120, 128, 128]) + self.assertEqual([call["timeout"] for call in self.process_calls], [20, 28, 28]) + self.assertTrue(summary["primaryEligible"], summary) + self.assertEqual(summary["coveredStatementIds"], ["pbt", "symbolic"]) + + + def test_confirmed_original_fault_exit_one_remains_eligible(self): + def process(command, directory, stem, timeout_seconds, **kwargs): + result = self.complete_process(command, directory, stem, timeout_seconds, **kwargs) + if stem == "runner": + driver.write_json(command[-1], self.campaign(fault=True)) + result["exitCode"] = 1 + return result + + summary = self.invoke_cell(process) + + self.assertTrue(summary["primaryEligible"], summary) + self.assertTrue(summary["confirmedViolationWithinBudget"]) + + + def test_incomplete_killed_and_late_artifacts_keep_diagnostics_without_credit(self): + for exit_code, timed_out, elapsed in ((2, False, 0), (-9, True, 0), (0, False, 20), (1, False, 20)): + with self.subTest(exit_code=exit_code, timed_out=timed_out, elapsed=elapsed): + self.clock = FakeClock() + self.process_calls = [] + + def process(command, directory, stem, timeout_seconds, **kwargs): + result = self.complete_process(command, directory, stem, timeout_seconds, **kwargs) + artifact = self.campaign(fault=True) + artifact["result"]["diagnostic"] = {"message": "retained reason"} + driver.write_json(command[-1], artifact) + self.clock.advance(elapsed) + return dict(result, exitCode=exit_code, timedOut=timed_out) + + summary = self.invoke_cell(process) + + self.assertEqual(summary["campaignDiagnostic"], {"message": "retained reason"}) + self.assertTrue(summary["confirmedViolation"]) + self.assertFalse(summary["confirmedViolationWithinBudget"]) + self.assertFalse(summary["primaryEligible"]) + self.assertFalse(summary["coverageComplete"]) + self.assertEqual(summary["coveredStatementIds"], []) + self.assertEqual([call["stem"] for call in self.process_calls], ["runner"]) + self.assertTrue((self.cell_directory / "campaign.json").exists()) + + + def test_exit_one_without_confirmed_fault_is_incomplete(self): + def process(command, directory, stem, timeout_seconds, **kwargs): + result = self.complete_process(command, directory, stem, timeout_seconds, **kwargs) + return dict(result, exitCode=1) + + summary = self.invoke_cell(process) + + self.assertFalse(summary["primaryEligible"]) + self.assertFalse(summary["runnerProcess"]["creditedWithinDeadline"]) + self.assertEqual(summary["coveredStatementIds"], []) + + + def test_native_batch_cannot_reset_the_total_cutoff(self): + def process(command, directory, stem, timeout_seconds, **kwargs): + result = self.complete_process(command, directory, stem, timeout_seconds, **kwargs) + if stem == "pbt": + self.clock.advance(26) + return result + + summary = self.invoke_cell(process) + + self.assertEqual([call["deadline"] for call in self.process_calls], [120, 128]) + self.assertFalse(summary["replayProcesses"][0]["creditedWithinDeadline"]) + self.assertFalse(summary["coverageComplete"]) + self.assertEqual(summary["coveredStatementIds"], []) + + + def test_nonzero_native_batch_does_not_earn_coverage(self): + def process(command, directory, stem, timeout_seconds, **kwargs): + result = self.complete_process(command, directory, stem, timeout_seconds, **kwargs) + if stem == "pbt": + result["exitCode"] = 2 + return result + + summary = self.invoke_cell(process) + + self.assertFalse(summary["primaryEligible"]) + self.assertFalse(summary["coverageComplete"]) + self.assertEqual(summary["coveredStatementIds"], []) + + + def test_invalid_runner_or_native_json_does_not_earn_credit(self): + for malformed_stem in ("runner", "pbt"): + with self.subTest(malformed_stem=malformed_stem): + def process(command, directory, stem, timeout_seconds, **kwargs): + result = self.complete_process(command, directory, stem, timeout_seconds, **kwargs) + if stem == malformed_stem: + command[-1].write_text("{invalid", encoding="utf-8") + return result + + summary = self.invoke_cell(process) + + self.assertFalse(summary["primaryEligible"]) + self.assertFalse(summary["coverageComplete"]) + self.assertEqual(summary["coveredStatementIds"], []) + self.assertIn("diagnostic", summary) + + if __name__ == "__main__": unittest.main() From 560b23d494509d726be2a6347e948b45e7dbf1aa Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Sat, 17 Oct 2026 04:52:36 +0300 Subject: [PATCH 2/2] Avoid extending caller deadlines during clock sampling --- usvm-ts-fast-check/scripts/pbt_experiment.py | 6 ++++-- .../scripts/test_pbt_experiment.py | 20 +++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/usvm-ts-fast-check/scripts/pbt_experiment.py b/usvm-ts-fast-check/scripts/pbt_experiment.py index 8c343d9425..4586dbd766 100644 --- a/usvm-ts-fast-check/scripts/pbt_experiment.py +++ b/usvm-ts-fast-check/scripts/pbt_experiment.py @@ -5,6 +5,7 @@ import csv import hashlib import json +import math import os from pathlib import Path import random @@ -214,13 +215,14 @@ def run_cell(frozen, case, seed, mode, output): directory = output / case["id"] / str(seed) / mode directory.mkdir(parents=True) started = time.monotonic() - started_epoch_millis = int(time.time() * 1000) preparation_cost = case["preparation"]["elapsedMillis"] remaining_budget = max(0, frozen["totalBudgetMillis"] - preparation_cost) deadline = started + remaining_budget / 1000 runner_budget = remaining_budget - frozen["classReplayReserveMillis"] runner_deadline = deadline - frozen["classReplayReserveMillis"] / 1000 - runner_deadline_epoch_millis = started_epoch_millis + runner_budget + wall_now = time.time() + monotonic_now = time.monotonic() + runner_deadline_epoch_millis = math.floor((wall_now + runner_deadline - monotonic_now) * 1000) summary = {"caseId": case["id"], "project": case["project"], "population": case["population"], "seed": seed, "mode": mode, "stopCriterion": STOP_CRITERION, "preparationChargedMillis": preparation_cost, "declaredTotalBudgetMillis": frozen["totalBudgetMillis"], "pbtRunQuota": frozen["numRuns"], diff --git a/usvm-ts-fast-check/scripts/test_pbt_experiment.py b/usvm-ts-fast-check/scripts/test_pbt_experiment.py index 64356a52d1..b071a53d07 100644 --- a/usvm-ts-fast-check/scripts/test_pbt_experiment.py +++ b/usvm-ts-fast-check/scripts/test_pbt_experiment.py @@ -185,7 +185,7 @@ def complete_process(self, command, directory, stem, timeout_seconds, *, deadlin return {"exitCode": 0, "timedOut": False} - def invoke_cell(self, process): + def invoke_cell(self, process, *, wall_time=None): self.cell_count += 1 output = self.root / f"output-{self.cell_count}" prepared = output / "subject" / "prepared.json" @@ -202,9 +202,10 @@ def invoke_cell(self, process): "maxArrayElements": 64, "maxObservationBytes": 65536, "nodeCommand": "node", "distribution": "/distribution", "adapter": "/adapter", "runtimeSha256": {}} self.cell_directory = output / "subject" / "37" / "COVERAGE_FEEDBACK" + wall_clock = self.clock.time if wall_time is None else wall_time with patch.object(driver.time, "monotonic", self.clock.monotonic), \ - patch.object(driver.time, "time", self.clock.time), \ + patch.object(driver.time, "time", wall_clock), \ patch.object(driver, "run_process", side_effect=process): return driver.run_cell(frozen, case, seed=37, mode="COVERAGE_FEEDBACK", output=output) @@ -229,6 +230,21 @@ def test_cell_shares_cutoffs_and_passes_reserves_without_rewriting_controls(self self.assertEqual(summary["coveredStatementIds"], ["pbt", "symbolic"]) + def test_clock_sampling_delay_cannot_extend_the_monotonic_cutoff(self): + def delayed_wall_time(): + self.clock.advance(3) + sampled_wall = self.clock.time() + self.clock.advance(2) + return sampled_wall + + summary = self.invoke_cell(self.complete_process, wall_time=delayed_wall_time) + + request = driver.read_json(self.cell_directory / "request.json") + self.assertEqual(request["externalDeadlineEpochMillis"], 1_700_000_018_000) + self.assertEqual([call["deadline"] for call in self.process_calls], [120, 128, 128]) + self.assertTrue(summary["primaryEligible"], summary) + + def test_confirmed_original_fault_exit_one_remains_eligible(self): def process(command, directory, stem, timeout_seconds, **kwargs): result = self.complete_process(command, directory, stem, timeout_seconds, **kwargs)