From a87e05e6a4a9183675933e56aec413888391432c Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Wed, 14 Oct 2026 18:39:05 +0300 Subject: [PATCH 1/2] Rebase class coverage measurement onto extracted engine changes --- buildSrc/src/main/kotlin/Dependencies.kt | 2 +- usvm-ts-calls/scripts/run-class-experiment.py | 481 ++++++++++++ .../scripts/test_run_class_experiment.py | 447 ++++++++++++ .../usvm/ts/calls/CallsClassCoverageEngine.kt | 491 +++++++++++++ .../org/usvm/ts/calls/CallsClassExperiment.kt | 513 +++++++++++++ .../usvm/ts/calls/CallsClassExperimentCli.kt | 553 ++++++++++++++ .../org/usvm/ts/calls/CallsClassHarness.kt | 415 +++++++++++ .../org/usvm/ts/calls/CallsClassOracle.kt | 683 ++++++++++++++++++ .../calls/CallsClassSourceInstrumentation.kt | 129 ++++ .../usvm/ts/calls/CallsClassSourceUniverse.kt | 57 ++ .../org/usvm/ts/calls/CallsCoverageCell.kt | 35 +- .../org/usvm/ts/calls/CallsCoverageSearch.kt | 163 ++++- .../org/usvm/ts/calls/CallsExperimentCli.kt | 8 + .../org/usvm/ts/calls/CallsSymbolicInputs.kt | 5 +- .../ts/calls/CallsUnknownCallTelemetry.kt | 4 +- .../ts/calls/CurrentTsCallsSymbolicEngine.kt | 22 +- .../usvm/ts/calls/CallsClassAdmissionTest.kt | 403 +++++++++++ .../ts/calls/CallsClassCoverageSourceTest.kt | 171 +++++ .../calls/CallsClassDataRegistryOracleTest.kt | 388 ++++++++++ .../usvm/ts/calls/CallsClassExperimentTest.kt | 289 ++++++++ .../ts/calls/CallsClassHarnessOracleTest.kt | 473 ++++++++++++ .../calls/CallsClassNativeDiagnosticTest.kt | 120 +++ .../calls/CallsClassUnknownCallJournalTest.kt | 323 +++++++++ .../fast-check-adapter/package.json | 2 +- .../src/class-coverage-source.ts | 671 +++++++++++++++++ .../src/local-source-closure.ts | 4 +- .../src/source-inspector-cli.ts | 7 +- .../test/class-coverage-source.test.ts | 396 ++++++++++ .../test/local-source-closure.test.ts | 33 +- .../fastcheck/TypeScriptSourceInspector.kt | 66 ++ 30 files changed, 7310 insertions(+), 44 deletions(-) create mode 100644 usvm-ts-calls/scripts/run-class-experiment.py create mode 100644 usvm-ts-calls/scripts/test_run_class_experiment.py create mode 100644 usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassCoverageEngine.kt create mode 100644 usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperiment.kt create mode 100644 usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperimentCli.kt create mode 100644 usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassHarness.kt create mode 100644 usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassOracle.kt create mode 100644 usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassSourceInstrumentation.kt create mode 100644 usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassSourceUniverse.kt create mode 100644 usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassAdmissionTest.kt create mode 100644 usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassCoverageSourceTest.kt create mode 100644 usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassDataRegistryOracleTest.kt create mode 100644 usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassExperimentTest.kt create mode 100644 usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassHarnessOracleTest.kt create mode 100644 usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassNativeDiagnosticTest.kt create mode 100644 usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassUnknownCallJournalTest.kt create mode 100644 usvm-ts-fast-check/fast-check-adapter/src/class-coverage-source.ts create mode 100644 usvm-ts-fast-check/fast-check-adapter/test/class-coverage-source.test.ts diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt index e7f7c7423f..28c2c00af8 100644 --- a/buildSrc/src/main/kotlin/Dependencies.kt +++ b/buildSrc/src/main/kotlin/Dependencies.kt @@ -6,7 +6,7 @@ object Versions { const val clikt = "5.0.0" const val detekt = "1.23.7" const val ini4j = "0.5.4" - const val jacodb = "23199c1843" + const val jacodb = "37327563ab" const val juliet = "1.3.2" const val junit = "5.9.3" const val kotlin = "2.1.0" diff --git a/usvm-ts-calls/scripts/run-class-experiment.py b/usvm-ts-calls/scripts/run-class-experiment.py new file mode 100644 index 0000000000..645f4b4ffe --- /dev/null +++ b/usvm-ts-calls/scripts/run-class-experiment.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +"""Sequential paired class campaign with profile-independent source/native admission.""" +import argparse +import csv +import hashlib +import json +import subprocess +from pathlib import Path + +PROFILES = ('EMPTY_STOP', 'EMPTY_FRESH', 'FROZEN_STOP', 'FROZEN_FRESH') +BUDGETS = (30_000, 120_000) +COVERAGE_POLICY = 'native-original-instrumented-complete-v2' +SUSPENDED_COVERAGE_POLICY = 'reached-prefix-and-final-frontier-inputs-v1' + + +def validate_measurement_protocol(identity): + if (identity.get('schemaVersion') != 2 or identity.get('coveragePolicy') != COVERAGE_POLICY + or identity.get('suspendedCoveragePolicy') != SUSPENDED_COVERAGE_POLICY): + raise ValueError('A new schema-2 native coverage and reached-prefix acquisition is required') + + +def read(path): + return json.loads(path.read_text(encoding='utf-8')) + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def freeze(path, value): + if path.exists(): + if read(path) != value: + raise ValueError(f'Persisted campaign identity changed: {path}') + return + + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + '.tmp') + temporary.write_text(json.dumps(value, indent=2) + '\n', encoding='utf-8') + temporary.replace(path) + + +def invoke(calls, arguments, log, timeout): + log.parent.mkdir(parents=True, exist_ok=True) + with log.open('w', encoding='utf-8') as output: + try: + result = subprocess.run([str(calls), *map(str, arguments)], stdout=output, + stderr=subprocess.STDOUT, timeout=timeout, check=False) + return {'exitCode': result.returncode, 'log': str(log)} + except subprocess.TimeoutExpired: + return {'exitCode': None, 'timeoutSeconds': timeout, 'log': str(log)} + + +def select_classes(ledger, eligible): + if ledger is None: + return sorted(eligible), [] + + decisions = ledger['decisions'] + chosen = [] + missing = [] + used = set() + for selected in sorted((item for item in decisions if item['origin'] == 'selected'), + key=lambda item: item['order']): + alternatives = [selected] + sorted( + (item for item in decisions if item['origin'] == 'reserve' + and item['projectId'] == selected['projectId'] and item['stratum'] == selected['stratum']), + key=lambda item: item['order']) + replacement = next((item for item in alternatives + if item.get('classId') in eligible and item['classId'] not in used), None) + if replacement is None: + missing.append({'identity': selected['identity'], 'projectId': selected['projectId'], + 'stratum': selected['stratum'], 'reason': 'No eligible preordered same-stratum reserve'}) + else: + used.add(replacement['classId']) + chosen.append(replacement['classId']) + + return chosen, missing + + +def admits_primary_comparison(preflight): + if preflight.get('admissionVersion') != 1: + raise ValueError('Legacy profile-dependent preflight cannot admit a new scientific comparison') + + return preflight['eligible'] and preflight['primaryInferenceEligible'] is True + + +def require_native_acquisition_health(output, stage, records): + if (output / 'acquisition-health-failure.json').exists(): + raise RuntimeError('Saved native acquisition failure invalidates the entire acquisition attempt') + + failures = [{'classId': record.get('classId', record.get('identity', {}).get('classId')), + 'nativeInfrastructureFailures': record['nativeInfrastructureFailures']} + for record in records if record.get('nativeInfrastructureFailures')] + if failures: + freeze(output / 'acquisition-health-failure.json', {'stage': stage, 'failures': failures}) + raise RuntimeError('Native instrumentation corruption invalidates the entire acquisition attempt') + + +def native_replay_failure(event, statement_ids): + original = event.get('original') + instrumented = event.get('instrumented') + if instrumented is not None and statement_ids is not None: + foreign = set(instrumented.get('coveredStatementKeys', [])) - statement_ids + if foreign: + return f'Instrumentation reported foreign statement IDs: {sorted(foreign)}' + if original is None or instrumented is None: + return None + if original['inputs'] != instrumented['inputs']: + return 'Instrumentation changed the frozen native input transport' + + original_observation = original['observation'] + instrumented_observation = instrumented['observation'] + if original_observation['completion'] != instrumented_observation['completion']: + return 'Instrumentation changed native completion' + original_representable = original_observation.get('representable', True) + instrumented_representable = instrumented_observation.get('representable', True) + if original_representable != instrumented_representable: + return 'Instrumentation changed native observability or observation schema' + if original_representable and original_observation != instrumented_observation: + return 'Instrumentation changed native returned values or receiver state' + + return None + + +def saved_replay_failures(output): + journals = [(path, path.parent.parent.name) + for path in sorted((output / 'prepared').glob('*/preflight/replays.jsonl'))] + journals.extend((path, path.relative_to(output / 'cells').parts[0]) + for path in sorted((output / 'cells').rglob('replays.jsonl'))) + universes = {} + failures = [] + for path, class_id in journals: + if class_id not in universes: + record = output / 'prepared' / class_id / 'class.json' + universes[class_id] = ({statement['id'] for statement in + read(record)['instrumentation']['universe']['statements']} + if record.exists() else None) + for line in path.read_text(encoding='utf-8').splitlines(): + if not line.strip(): + continue + event = json.loads(line) + failure = native_replay_failure(event, universes[class_id]) + if failure is not None: + failures.append({'classId': class_id, 'nativeInfrastructureFailures': [failure], + 'journal': str(path), 'eventIndex': event['index']}) + + return failures + + +def require_saved_acquisition_health(output): + require_native_acquisition_health(output, 'saved', []) + records = [read(path) for path in sorted((output / 'prepared').glob('*/preflight.json'))] + records.extend(read(path) for path in sorted((output / 'cells').rglob('result.json'))) + eligibility = output / 'eligibility.json' + if eligibility.exists(): + records.extend(read(eligibility)) + records.extend(saved_replay_failures(output)) + + require_native_acquisition_health(output, 'saved', records) + + +def scheduled_cells(plan, budgets=BUDGETS): + if plan['budgetsMillis'] != list(BUDGETS) or any(budget not in BUDGETS for budget in budgets): + raise ValueError('A phase must preserve the full frozen 30/120-second protocol') + + for budget in budgets: + for class_index, class_id in enumerate(plan['classIds']): + for seed in plan['seeds']: + rotation = (class_index + seed) % len(PROFILES) + profiles = PROFILES[rotation:] + PROFILES[:rotation] + for profile in profiles: + yield budget, class_id, seed, profile + + +def preserves_input_values(prepared, declared): + """Kotlin writes domain defaults; every explicitly frozen value must still agree.""" + if isinstance(declared, dict): + if not isinstance(prepared, dict): + return False + return all(preserves_input_values(prepared.get(key), value) for key, value in declared.items()) + if isinstance(declared, list): + if not isinstance(prepared, list) or len(prepared) != len(declared): + return False + return all(preserves_input_values(actual, expected) for actual, expected in zip(prepared, declared)) + return prepared == declared + + +def cell_directory(output, class_id, seed, budget, profile): + return output / 'cells' / class_id / str(seed) / str(budget) / profile + + +def validate_result(result, expected, prepared, universe): + if result.get('nativeInfrastructureFailures'): + raise ValueError('Native instrumentation corruption invalidates the acquisition attempt') + + identity = result['identity'] + validate_measurement_protocol(identity) + for key, value in expected.items(): + if identity.get(key) != value: + raise ValueError(f'Paired cell identity differs for {key}') + + if identity['preparedSha256'] != sha(prepared / identity['classId'] / 'class.json'): + raise ValueError('Prepared class identity differs') + if identity['preflightSha256'] != sha(prepared / identity['classId'] / 'preflight.json'): + raise ValueError('Frozen preflight identity differs') + if identity['universeSha256'] != universe['universeSha256']: + raise ValueError('Statement universe identity differs') + + ids = {statement['id'] for statement in universe['statements']} + previous = set() + previous_strict = set() + candidates = result['search']['search']['candidates'] + for candidate in candidates: + origin = candidate['origin'] + if origin not in ('TERMINATED', 'SUSPENDED') or candidate['emittedAtMillis'] < 0: + raise ValueError('Invalid candidate origin or emission time') + if origin == 'SUSPENDED' and (candidate.get('completion') is not None + or candidate.get('observation') is not None): + raise ValueError('Suspended input contains an invented terminal observation') + if origin == 'TERMINATED' and (candidate.get('completion') not in ('RETURNED', 'THREW') + or candidate.get('suspendedAtMachineStop', False)): + raise ValueError('Invalid terminal candidate provenance') + for point in result['checkpoints']: + covered = set(point['coveredStatementIds']) + uncovered = set(point['uncoveredStatementIds']) + if point['totalStatements'] != len(ids) or covered & uncovered or covered | uncovered != ids: + raise ValueError('Invalid exact statement numerator/denominator') + if not previous <= covered: + raise ValueError('Coverage history is not monotone') + if point['searchBudgetMillis'] > identity['budgetMillis']: + raise ValueError('Coverage checkpoint exceeds cell budget') + strict = set(point['strictCoveredStatementIds']) + strict_uncovered = set(point['strictUncoveredStatementIds']) + if strict & strict_uncovered or strict | strict_uncovered != ids or not strict <= covered: + raise ValueError('Invalid strict secondary statement coverage') + if not previous_strict <= strict: + raise ValueError('Strict coverage history is not monotone') + final = point['searchBudgetMillis'] == identity['budgetMillis'] + selected = [candidate for candidate in candidates + if (candidate['origin'] == 'SUSPENDED' and candidate.get('suspendedAtMachineStop', False) and final) + or (not candidate.get('suspendedAtMachineStop', False) + and candidate['emittedAtMillis'] <= point['searchBudgetMillis'])] + post_stop = sum(candidate.get('suspendedAtMachineStop', False) for candidate in selected) + suspended = sum(candidate['origin'] == 'SUSPENDED' for candidate in selected) + if (point['emittedCandidates'] != len(selected) or point['suspendedCandidates'] != suspended + or point['postStopSuspendedCandidates'] != post_stop + or point['includesSuspendedExtraction'] != final): + raise ValueError('Suspended extraction checkpoint attribution differs') + if point['acceptedCandidates'] + point['rejectedCandidates'] != len(selected): + raise ValueError('Native candidate accounting differs') + if (point['strictAcceptedCandidates'] + point['strictRejectedCandidates'] + + point['strictOracleNotApplicable'] != len(selected) + or point['strictOracleNotApplicable'] != suspended): + raise ValueError('Strict secondary candidate accounting differs') + previous = covered + previous_strict = strict + + if not result['checkpoints'] or result['checkpoints'][-1]['searchBudgetMillis'] != identity['budgetMillis']: + raise ValueError('Final requested-budget coverage checkpoint is absent') + + +def summarize(output, prepared, plan, scheduled_budgets=BUDGETS): + require_saved_acquisition_health(output) + manifest = read(prepared / 'manifest.json') + validate_measurement_protocol(manifest) + rows = [] + failures = [] + pending = [] + pairs = [] + for class_id in plan['classIds']: + record = read(prepared / class_id / 'class.json') + universe = record['instrumentation']['universe'] + for seed in manifest['seeds']: + for budget in BUDGETS: + pair = [] + for profile in PROFILES: + path = cell_directory(output, class_id, seed, budget, profile) / 'result.json' + if not path.exists(): + missing = {'classId': class_id, 'seed': seed, 'budgetMillis': budget, + 'profile': profile, 'reason': 'Missing completed cell'} + if budget in scheduled_budgets: + failures.append(missing) + else: + pending.append(missing) + continue + + result = read(path) + expected = {'experimentId': manifest['experimentId'], 'toolRevision': manifest['toolRevision'], + 'nativeFrontendRevision': manifest['nativeFrontendRevision'], + 'manifestSha256': sha(prepared / 'manifest.json'), + 'classId': class_id, 'seed': seed, 'budgetMillis': budget, 'profile': profile} + try: + validate_result(result, expected, prepared, universe) + except (KeyError, ValueError) as error: + failures.append({**expected, 'reason': str(error), 'path': str(path)}) + continue + + search = result['search']['search'] + final = result['checkpoints'][-1] + row = {'classId': class_id, 'profile': profile, 'seed': seed, 'budgetMillis': budget, + 'covered': len(final['coveredStatementIds']), 'total': final['totalStatements'], + 'fraction': len(final['coveredStatementIds']) / final['totalStatements'], + 'status': search['status'], 'acceptedCandidates': final['acceptedCandidates'], + 'machineStopReason': search.get('machineStopReason'), + 'rejectedCandidates': final['rejectedCandidates'], + 'strictCovered': len(final['strictCoveredStatementIds']), + 'strictAcceptedCandidates': final['strictAcceptedCandidates'], + 'strictRejectedCandidates': final['strictRejectedCandidates'], + 'strictOracleNotApplicable': final['strictOracleNotApplicable'], + 'suspendedCandidates': final['suspendedCandidates'], + 'postStopSuspendedCandidates': final['postStopSuspendedCandidates'], + 'suspendedExtractionElapsedMillis': search['suspendedExtractionElapsedMillis'], + 'frontierFlushElapsedMillis': search['frontierFlushElapsedMillis'], + 'searchElapsedMillis': search['searchElapsedMillis'], + 'setupElapsedMillis': search['preparationElapsedMillis'] + search['machineSetupElapsedMillis'], + 'replayElapsedMillis': result['replayElapsedMillis'], + 'solverCalls': result['search'].get('solverCalls'), + 'solverElapsedMillis': result['search'].get('solverElapsedMillis'), + 'runtimeLimited': search['runtimeLimited'], 'engineFailed': search['engineFailed'], + 'candidateCapReached': search['candidateCapReached'], + 'replayErrors': result['replayErrors'], 'oracleRejections': result['oracleRejections'], + 'nativeCoverageRejections': result['nativeCoverageRejections']} + rows.append(row) + pair.append(row) + + if len(pair) == len(PROFILES): + pairs.append({'classId': class_id, 'seed': seed, 'budgetMillis': budget, + 'coveredByProfile': {row['profile']: row['covered'] for row in pair}, + 'strictCoveredByProfile': {row['profile']: row['strictCovered'] for row in pair}, + 'total': pair[0]['total'], + 'measurementFailures': any(row['engineFailed'] or row['runtimeLimited'] + or row['candidateCapReached'] or row['replayErrors'] + or row['nativeCoverageRejections'] for row in pair), + 'symbolicOracleDisagreement': any(row['oracleRejections'] for row in pair)}) + + summary = {'schemaVersion': 2, 'coveragePolicy': COVERAGE_POLICY, + 'suspendedCoveragePolicy': SUSPENDED_COVERAGE_POLICY, 'experimentId': manifest['experimentId'], + 'classCount': len(plan['classIds']), 'completePairedGroups': len(pairs), + 'incompleteOrInvalidCells': failures, 'missingStrata': plan['missingStrata'], + 'scheduledBudgetsMillis': list(scheduled_budgets), 'pendingOtherPhaseCells': pending, + 'complete': bool(plan['classIds']) and not failures and not pending, + 'rows': rows, 'pairs': pairs} + (output / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n', encoding='utf-8') + if rows: + with (output / 'coverage.csv').open('w', encoding='utf-8', newline='') as stream: + writer = csv.DictWriter(stream, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + lines = [f"Experiment: {manifest['experimentId']}", '', + 'Primary coverage requires complete agreeing native original/instrumented replay. Strict symbolic agreement is reported separately.', + 'In-search reached-prefix inputs use their actual emission time; post-stop frontier inputs appear only in the final budget checkpoint.', + f"Classes: {len(plan['classIds'])}; complete paired groups: {len(pairs)}; missing strata: {len(plan['missingStrata'])}.", + f"Complete acquisition: {summary['complete']}; pending cells in the other phase: {len(pending)}; invalid or missing scheduled cells: {len(failures)}.", + '', '| Class | Seed | Budget (s) | EMPTY_STOP | EMPTY_FRESH | FROZEN_STOP | FROZEN_FRESH |', + '|---|---:|---:|---:|---:|---:|---:|'] + for pair in pairs: + values = [f"{pair['coveredByProfile'][profile]}/{pair['total']}" for profile in PROFILES] + lines.append(f"| {pair['classId']} | {pair['seed']} | {pair['budgetMillis'] // 1000} | " + ' | '.join(values) + ' |') + lines.extend(['', '| Strict secondary class | Seed | Budget (s) | EMPTY_STOP | EMPTY_FRESH | FROZEN_STOP | FROZEN_FRESH |', + '|---|---:|---:|---:|---:|---:|---:|']) + for pair in pairs: + values = [f"{pair['strictCoveredByProfile'][profile]}/{pair['total']}" for profile in PROFILES] + lines.append(f"| {pair['classId']} | {pair['seed']} | {pair['budgetMillis'] // 1000} | " + ' | '.join(values) + ' |') + lines.extend(['', 'These are paired observations under the frozen domains. Unsupported candidates and missing strata remain in the eligibility ledger.', + 'Engine-suited datasets must be reported separately from the random/stratified corpus. No statistical superiority is inferred from one seed.']) + (output / 'RESULTS.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--calls', type=Path, required=True) + parser.add_argument('--manifest', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--selection-ledger', type=Path) + parser.add_argument('--preflight-ms', type=int, default=30_000) + parser.add_argument('--preflight-only', action='store_true') + parser.add_argument('--summarize-only', action='store_true') + parser.add_argument('--budget-ms', type=int, choices=BUDGETS, + help='Acquire or check one phase while retaining both budgets in the frozen plan') + args = parser.parse_args() + scheduled_budgets = (args.budget_ms,) if args.budget_ms is not None else BUDGETS + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + require_saved_acquisition_health(output) + prepared = output / 'prepared' + calls = args.calls.resolve() + ledger = read(args.selection_ledger) if args.selection_ledger else None + input_manifest = read(args.manifest) + validate_measurement_protocol(input_manifest) + if input_manifest['budgetsMillis'] != list(BUDGETS): + raise ValueError('Class budgets must be exactly 30 and 120 seconds') + freeze(output / 'campaign-input.json', {'manifest': input_manifest, 'selectionLedger': ledger, + 'preflightMillis': args.preflight_ms}) + if not (prepared / 'manifest.json').exists(): + outcome = invoke(calls, ['class-prepare', args.manifest.resolve(), prepared], + output / 'prepare.log', timeout=600) + freeze(output / 'prepare-process.json', outcome) + if outcome['exitCode'] != 0: + raise RuntimeError(f'Class preparation failed: {outcome}') + + manifest = read(prepared / 'manifest.json') + if not preserves_input_values(manifest, input_manifest): + raise ValueError('Prepared manifest differs from campaign input') + plan_path = output / 'run-plan.json' + if args.summarize_only: + summary = summarize(output, prepared, read(plan_path), scheduled_budgets=scheduled_budgets) + print(json.dumps(summary, indent=2)) + if summary['incompleteOrInvalidCells']: + raise RuntimeError('Incomplete or invalid scheduled campaign phase; see summary.json') + if not summary['classCount']: + raise RuntimeError('No primary admitted classes; no representative comparison was run') + return + + eligibility = [] + eligible = set() + for case in manifest['classes']: + class_id = case['classId'] + directory = prepared / class_id + failed = directory / 'preparation-failure.json' + if failed.exists(): + eligibility.append({'classId': class_id, 'eligible': False, 'preparationFailure': read(failed)}) + continue + preflight = directory / 'preflight.json' + if not preflight.exists(): + print(f'Preflight {class_id}', flush=True) + process = invoke(calls, ['class-preflight', prepared, class_id, args.preflight_ms], + output / 'logs' / f'{class_id}-preflight.log', timeout=600 + args.preflight_ms // 1000) + freeze(output / 'processes' / f'{class_id}-preflight.json', process) + if preflight.exists(): + outcome = read(preflight) + primary_admitted = admits_primary_comparison(outcome) + eligibility.append({'classId': class_id, 'eligible': outcome['eligible'], + 'primaryInferenceEligible': primary_admitted, + 'diagnosticReady': outcome['diagnosticReady'], + 'nativeSmoke': outcome.get('nativeSmoke'), + 'sourceAdmissionFailures': outcome['sourceAdmissionFailures'], + 'nativeInfrastructureFailures': outcome['nativeInfrastructureFailures'], + 'correctnessFrontierFailures': case.get('correctnessFrontierFailures', []), + 'missingRequiredScenarios': outcome['missingRequiredScenarios'], + 'reasons': outcome['reasons'], 'preflightSha256': sha(preflight)}) + if primary_admitted: + eligible.add(class_id) + else: + eligibility.append({'classId': class_id, 'eligible': False, 'reasons': ['Preflight failed before final record; see process log']}) + + chosen, missing = select_classes(ledger, eligible) + freeze(output / 'eligibility.json', eligibility) + require_native_acquisition_health(output, 'preflight', eligibility) + plan = {'schemaVersion': 2, 'coveragePolicy': COVERAGE_POLICY, + 'suspendedCoveragePolicy': SUSPENDED_COVERAGE_POLICY, 'admission': 'profile-independent-source-native-v1', + 'manifestSha256': sha(prepared / 'manifest.json'), 'classIds': chosen, + 'missingStrata': missing, 'profiles': list(PROFILES), 'budgetsMillis': list(BUDGETS), + 'seeds': manifest['seeds'], 'order': 'all 30s cells then all 120s; seeded rotation of profiles; sequential processes'} + freeze(plan_path, plan) + print(f'Frozen source/native admission: {len(chosen)} primary classes; {len(missing)} missing strata.', flush=True) + if args.preflight_only: + return + + for budget, class_id, seed, profile in scheduled_cells(plan, budgets=scheduled_budgets): + directory = cell_directory(output, class_id, seed, budget, profile) + if (directory / 'result.json').exists(): + require_native_acquisition_health(output, 'measurement', [read(directory / 'result.json')]) + continue + print(f'Measure {class_id} {profile} seed={seed} budget={budget}ms', flush=True) + process = invoke(calls, ['class-cell', prepared, class_id, profile, seed, budget, directory], + output / 'logs' / f'{class_id}-{seed}-{budget}-{profile}.log', timeout=600 + budget // 1000) + freeze(output / 'processes' / f'{class_id}-{seed}-{budget}-{profile}.json', process) + if (directory / 'result.json').exists(): + require_native_acquisition_health(output, 'measurement', [read(directory / 'result.json')]) + + summary = summarize(output, prepared, plan, scheduled_budgets=scheduled_budgets) + print(f"Finished: {len(summary['rows'])} cells, {summary['completePairedGroups']} complete paired groups.", flush=True) + if summary['incompleteOrInvalidCells']: + raise RuntimeError('Incomplete or invalid scheduled campaign phase; see summary.json') + if not chosen: + raise RuntimeError('No primary admitted classes; no representative comparison was run') + + +if __name__ == '__main__': + main() diff --git a/usvm-ts-calls/scripts/test_run_class_experiment.py b/usvm-ts-calls/scripts/test_run_class_experiment.py new file mode 100644 index 0000000000..994bb4929d --- /dev/null +++ b/usvm-ts-calls/scripts/test_run_class_experiment.py @@ -0,0 +1,447 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location('campaign', Path(__file__).with_name('run-class-experiment.py')) +campaign = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(campaign) + + +class CampaignIntegrityTest(unittest.TestCase): + def make_phase_fixture(self, output): + prepared = output / 'prepared' + directory = prepared / 'class' + directory.mkdir(parents=True) + manifest = {'experimentId': 'phase-test', 'toolRevision': 'revision', + 'schemaVersion': 2, 'coveragePolicy': campaign.COVERAGE_POLICY, + 'suspendedCoveragePolicy': campaign.SUSPENDED_COVERAGE_POLICY, + 'nativeFrontendRevision': 'frontend', 'budgetsMillis': list(campaign.BUDGETS), + 'seeds': [17], 'classes': [{'classId': 'class'}]} + universe = {'universeSha256': 'universe', 'statements': [{'id': 'original'}]} + campaign.freeze(prepared / 'manifest.json', manifest) + campaign.freeze(directory / 'class.json', {'instrumentation': {'universe': universe}}) + campaign.freeze(directory / 'preflight.json', { + 'admissionVersion': 1, 'classId': 'class', 'eligible': True, + 'primaryInferenceEligible': True, 'diagnosticReady': False, + 'nativeSmoke': {'status': 'VERIFIED'}, 'sourceAdmissionFailures': [], + 'nativeInfrastructureFailures': [], 'missingRequiredScenarios': ['slow'], + 'reasons': ['Symbolic pilot reached the time budget'], + }) + plan = {'classIds': ['class'], 'seeds': [17], 'budgetsMillis': list(campaign.BUDGETS), + 'missingStrata': []} + + return prepared, manifest, plan + + def write_phase_result(self, output, prepared, manifest, budget, profile): + directory = campaign.cell_directory(output, 'class', 17, budget, profile) + identity = {'experimentId': manifest['experimentId'], 'toolRevision': manifest['toolRevision'], + 'schemaVersion': 2, 'coveragePolicy': campaign.COVERAGE_POLICY, + 'suspendedCoveragePolicy': campaign.SUSPENDED_COVERAGE_POLICY, + 'nativeFrontendRevision': manifest['nativeFrontendRevision'], + 'manifestSha256': campaign.sha(prepared / 'manifest.json'), + 'preparedSha256': campaign.sha(prepared / 'class' / 'class.json'), + 'preflightSha256': campaign.sha(prepared / 'class' / 'preflight.json'), + 'universeSha256': 'universe', 'classId': 'class', 'seed': 17, + 'budgetMillis': budget, 'profile': profile} + result = {'identity': identity, 'nativeInfrastructureFailures': [], + 'search': {'search': {'status': 'TIMEOUT', 'searchElapsedMillis': budget, + 'candidates': [], 'suspendedExtractionElapsedMillis': 0, 'frontierFlushElapsedMillis': 0, + 'preparationElapsedMillis': 0, 'machineSetupElapsedMillis': 0, + 'runtimeLimited': False, 'engineFailed': False, 'candidateCapReached': False}}, + 'checkpoints': [{'searchBudgetMillis': budget, 'totalStatements': 1, + 'coveredStatementIds': [], 'uncoveredStatementIds': ['original'], + 'strictCoveredStatementIds': [], 'strictUncoveredStatementIds': ['original'], + 'strictAcceptedCandidates': 0, 'strictRejectedCandidates': 0, + 'strictOracleNotApplicable': 0, 'emittedCandidates': 0, + 'suspendedCandidates': 0, 'postStopSuspendedCandidates': 0, + 'includesSuspendedExtraction': True, + 'acceptedCandidates': 0, 'rejectedCandidates': 0}], + 'replayElapsedMillis': 0, 'replayErrors': 0, 'oracleRejections': 0, 'nativeCoverageRejections': 0} + + campaign.freeze(directory / 'result.json', result) + + def test_phase_schedule_preserves_full_protocol_and_profile_rotation(self): + plan = {'classIds': ['first', 'second'], 'seeds': [17, 29], + 'budgetsMillis': list(campaign.BUDGETS)} + frozen = json.dumps(plan, sort_keys=True) + + full = list(campaign.scheduled_cells(plan)) + short = list(campaign.scheduled_cells(plan, budgets=(30_000,))) + long = list(campaign.scheduled_cells(plan, budgets=(120_000,))) + + self.assertEqual(full, short + long) + self.assertEqual(16, len(short)) + self.assertEqual(16, len(long)) + self.assertEqual(30_000, short[0][0]) + self.assertEqual(campaign.PROFILES[1], short[0][3]) + self.assertEqual(frozen, json.dumps(plan, sort_keys=True)) + with self.assertRaisesRegex(ValueError, 'full frozen'): + list(campaign.scheduled_cells(plan, budgets=(60_000,))) + + def test_short_phase_retains_timeouts_and_reports_pending_long_phase(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, manifest, plan = self.make_phase_fixture(output) + for profile in campaign.PROFILES: + self.write_phase_result(output, prepared, manifest, budget=30_000, profile=profile) + + summary = campaign.summarize(output, prepared, plan, scheduled_budgets=(30_000,)) + + self.assertEqual(4, len(summary['rows'])) + self.assertTrue(all(row['status'] == 'TIMEOUT' for row in summary['rows'])) + self.assertEqual([], summary['incompleteOrInvalidCells']) + self.assertEqual(4, len(summary['pendingOtherPhaseCells'])) + self.assertEqual({120_000}, {cell['budgetMillis'] for cell in summary['pendingOtherPhaseCells']}) + self.assertFalse(summary['complete']) + + for profile in campaign.PROFILES: + self.write_phase_result(output, prepared, manifest, budget=120_000, profile=profile) + complete = campaign.summarize(output, prepared, plan) + + self.assertEqual(8, len(complete['rows'])) + self.assertEqual(2, complete['completePairedGroups']) + self.assertEqual([], complete['pendingOtherPhaseCells']) + self.assertTrue(complete['complete']) + + def test_scheduled_missing_cell_is_a_failure_not_a_pending_other_phase(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, manifest, plan = self.make_phase_fixture(output) + for profile in campaign.PROFILES[:-1]: + self.write_phase_result(output, prepared, manifest, budget=30_000, profile=profile) + + summary = campaign.summarize(output, prepared, plan, scheduled_budgets=(30_000,)) + + self.assertEqual(1, len(summary['incompleteOrInvalidCells'])) + self.assertEqual(30_000, summary['incompleteOrInvalidCells'][0]['budgetMillis']) + self.assertEqual(4, len(summary['pendingOtherPhaseCells'])) + self.assertFalse(summary['complete']) + + def test_phase_resume_keeps_full_frozen_identity_and_launches_only_requested_budget(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, manifest, _ = self.make_phase_fixture(output) + manifest_path = output / 'input-manifest.json' + campaign.freeze(manifest_path, manifest) + + def fake_invoke(calls, arguments, log, timeout): + self.assertEqual('class-cell', arguments[0]) + self.write_phase_result(output, prepared, manifest, budget=arguments[5], profile=arguments[3]) + return {'exitCode': 0} + + base_args = ['campaign', '--calls', str(output / 'calls'), '--manifest', str(manifest_path), + '--output', str(output)] + with patch('sys.argv', [*base_args, '--budget-ms', '30000']), patch.object( + campaign, 'invoke', side_effect=fake_invoke) as short_calls, patch('builtins.print'): + campaign.main() + frozen_input = (output / 'campaign-input.json').read_bytes() + frozen_plan = (output / 'run-plan.json').read_bytes() + + with patch('sys.argv', [*base_args, '--budget-ms', '120000']), patch.object( + campaign, 'invoke', side_effect=fake_invoke) as long_calls, patch('builtins.print'): + campaign.main() + + self.assertEqual(4, short_calls.call_count) + self.assertEqual({30_000}, {call.args[1][5] for call in short_calls.call_args_list}) + self.assertEqual(4, long_calls.call_count) + self.assertEqual({120_000}, {call.args[1][5] for call in long_calls.call_args_list}) + self.assertEqual(frozen_input, (output / 'campaign-input.json').read_bytes()) + self.assertEqual(frozen_plan, (output / 'run-plan.json').read_bytes()) + self.assertEqual(list(campaign.BUDGETS), campaign.read(output / 'run-plan.json')['budgetsMillis']) + self.assertTrue(campaign.read(output / 'summary.json')['complete']) + + def test_source_rejected_preflight_may_omit_null_native_smoke(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, manifest, _ = self.make_phase_fixture(output) + manifest_path = output / 'input-manifest.json' + campaign.freeze(manifest_path, manifest) + preflight_path = prepared / 'class' / 'preflight.json' + preflight = campaign.read(preflight_path) + preflight.pop('nativeSmoke') + preflight.update(eligible=False, primaryInferenceEligible=False, + sourceAdmissionFailures=['TOP_LEVEL_INITIALIZATION_UNSUPPORTED: offset 0']) + preflight_path.write_text(json.dumps(preflight) + '\n') + args = ['campaign', '--calls', str(output / 'calls'), '--manifest', str(manifest_path), + '--output', str(output), '--preflight-only'] + + with patch('sys.argv', args), patch.object(campaign, 'invoke') as invoke, patch('builtins.print'): + campaign.main() + + invoke.assert_not_called() + eligibility = campaign.read(output / 'eligibility.json') + self.assertIsNone(eligibility[0]['nativeSmoke']) + self.assertFalse(eligibility[0]['primaryInferenceEligible']) + self.assertEqual([], campaign.read(output / 'run-plan.json')['classIds']) + self.assertFalse((output / 'summary.json').exists()) + + def test_saved_corruption_in_later_unselected_class_blocks_any_resume_launch(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + _, manifest, _ = self.make_phase_fixture(output) + manifest_path = output / 'input-manifest.json' + campaign.freeze(manifest_path, manifest) + later = output / 'cells' / 'unselected' / '29' / '120000' / 'FROZEN_FRESH' / 'result.json' + campaign.freeze(later, {'identity': {'classId': 'unselected'}, + 'nativeInfrastructureFailures': ['Native completion changed']}) + args = ['campaign', '--calls', str(output / 'calls'), '--manifest', str(manifest_path), + '--output', str(output), '--budget-ms', '30000'] + + with patch('sys.argv', args), patch.object(campaign, 'invoke') as invoke: + with self.assertRaisesRegex(RuntimeError, 'entire acquisition attempt'): + campaign.main() + + invoke.assert_not_called() + self.assertFalse((output / 'run-plan.json').exists()) + self.assertEqual('unselected', campaign.read(output / 'acquisition-health-failure.json')['failures'][0]['classId']) + + def test_summarize_refuses_corruption_outside_plan_before_writing_results(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, _, plan = self.make_phase_fixture(output) + campaign.freeze(prepared / 'excluded' / 'preflight.json', { + 'classId': 'excluded', 'nativeInfrastructureFailures': ['Native receiver changed'], + }) + + with self.assertRaisesRegex(RuntimeError, 'entire acquisition attempt'): + campaign.summarize(output, prepared, plan, scheduled_budgets=(30_000,)) + + self.assertFalse((output / 'summary.json').exists()) + self.assertFalse((output / 'RESULTS.md').exists()) + + def test_sticky_health_receipt_blocks_summarize_only_before_any_launch(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + campaign.freeze(output / 'acquisition-health-failure.json', {'stage': 'previous', 'failures': []}) + args = ['campaign', '--calls', str(output / 'calls'), '--manifest', str(output / 'missing.json'), + '--output', str(output), '--summarize-only'] + + with patch('sys.argv', args), patch.object(campaign, 'invoke') as invoke: + with self.assertRaisesRegex(RuntimeError, 'Saved native acquisition failure'): + campaign.main() + + invoke.assert_not_called() + self.assertFalse((output / 'summary.json').exists()) + self.assertFalse((output / 'campaign-input.json').exists()) + + def test_interrupted_native_replay_mismatch_blocks_resume_before_result(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, manifest, _ = self.make_phase_fixture(output) + manifest_path = output / 'input-manifest.json' + campaign.freeze(manifest_path, manifest) + journal = output / 'cells' / 'class' / '17' / '120000' / 'FROZEN_FRESH' / 'replays.jsonl' + journal.parent.mkdir(parents=True) + original = {'inputs': [], 'observation': {'completion': 'RETURNED', 'representable': True, + 'values': {'state': {'kind': 'boolean', 'value': True}}}} + instrumented = {'inputs': [], 'observation': {'completion': 'RETURNED', 'representable': True, + 'values': {'state': {'kind': 'boolean', 'value': False}}}, + 'coveredStatementKeys': ['original']} + event = {'index': 0, 'original': original, 'instrumented': instrumented, + 'oracle': {'accepted': False}} + journal.write_text(json.dumps(event) + '\n') + args = ['campaign', '--calls', str(output / 'calls'), '--manifest', str(manifest_path), + '--output', str(output), '--budget-ms', '30000'] + + with patch('sys.argv', args), patch.object(campaign, 'invoke') as invoke: + with self.assertRaisesRegex(RuntimeError, 'entire acquisition attempt'): + campaign.main() + + invoke.assert_not_called() + self.assertFalse((journal.parent / 'result.json').exists()) + self.assertEqual('class', campaign.read(output / 'acquisition-health-failure.json')['failures'][0]['classId']) + with self.assertRaisesRegex(RuntimeError, 'Saved native acquisition failure'): + campaign.summarize(output, prepared, {'classIds': []}) + + def test_interrupted_foreign_statement_is_corruption_even_without_original_replay(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, _, plan = self.make_phase_fixture(output) + journal = prepared / 'class' / 'preflight' / 'replays.jsonl' + journal.parent.mkdir() + event = {'index': 0, 'instrumented': {'coveredStatementKeys': ['foreign']}, + 'oracle': {'accepted': False}, 'error': 'original timed out'} + journal.write_text(json.dumps(event) + '\n') + + with self.assertRaisesRegex(RuntimeError, 'entire acquisition attempt'): + campaign.summarize(output, prepared, plan) + + self.assertFalse((output / 'summary.json').exists()) + receipt = campaign.read(output / 'acquisition-health-failure.json') + self.assertIn('foreign statement IDs', receipt['failures'][0]['nativeInfrastructureFailures'][0]) + + def test_lawful_incomplete_or_missing_native_replay_does_not_prove_corruption(self): + original = {'inputs': [], 'observation': {'completion': 'THREW', 'representable': False, + 'values': {}, 'diagnostic': 'original exception'}} + instrumented = {'inputs': [], 'observation': {'completion': 'THREW', 'representable': False, + 'values': {}, 'diagnostic': 'instrumented exception'}, 'coveredStatementKeys': ['original']} + + self.assertIsNone(campaign.native_replay_failure( + {'original': original, 'instrumented': instrumented}, statement_ids={'original'})) + self.assertIsNone(campaign.native_replay_failure( + {'instrumented': instrumented, 'error': 'original timed out'}, statement_ids={'original'})) + + def test_reserve_replacement_cannot_cross_project_or_stratum(self): + selected = {'origin': 'selected', 'order': 0, 'identity': 'selected', 'projectId': 'p', + 'stratum': 'complex', 'classId': 'original'} + reserves = [dict(selected, origin='reserve', order=0, classId='foreign', projectId='q'), + dict(selected, origin='reserve', order=1, classId='simple', stratum='moderate'), + dict(selected, origin='reserve', order=2, classId='second'), + dict(selected, origin='reserve', order=3, classId='third')] + + chosen, missing = campaign.select_classes({'decisions': [selected, *reserves]}, + {'foreign', 'simple', 'second', 'third'}) + + self.assertEqual(['second'], chosen) + self.assertEqual([], missing) + chosen, missing = campaign.select_classes({'decisions': [selected, *reserves]}, {'foreign', 'simple'}) + self.assertEqual([], chosen) + self.assertEqual('selected', missing[0]['identity']) + + def test_slow_or_incomplete_pilot_does_not_select_the_primary_population(self): + slow = {'admissionVersion': 1, 'eligible': True, 'primaryInferenceEligible': True, + 'diagnosticReady': False, 'missingRequiredScenarios': ['one', 'two'], + 'search': {'status': 'TIMEOUT'}} + broken = dict(slow, primaryInferenceEligible=False, + correctnessFrontierFailures=['Known lowering defect']) + selected = {'origin': 'selected', 'order': 0, 'identity': 'slow', 'projectId': 'p', + 'stratum': 'complex', 'classId': 'slow'} + + admitted = {class_id for class_id, preflight in [('slow', slow), ('broken', broken)] + if campaign.admits_primary_comparison(preflight)} + chosen, missing = campaign.select_classes({'decisions': [selected]}, admitted) + + self.assertEqual({'slow'}, admitted) + self.assertEqual(['slow'], chosen) + self.assertEqual([], missing) + + def test_native_corruption_stops_the_whole_attempt_instead_of_dropping_one_class(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + slow = {'classId': 'slow', 'missingRequiredScenarios': ['method'], + 'nativeInfrastructureFailures': [], 'replayErrors': 1} + broken = {'classId': 'broken', 'nativeInfrastructureFailures': ['Receiver state changed']} + + campaign.require_native_acquisition_health(output, 'preflight', [slow]) + self.assertFalse((output / 'acquisition-health-failure.json').exists()) + + with self.assertRaisesRegex(RuntimeError, 'entire acquisition attempt'): + campaign.require_native_acquisition_health(output, 'preflight', [slow, broken]) + + receipt = campaign.read(output / 'acquisition-health-failure.json') + self.assertEqual('preflight', receipt['stage']) + self.assertEqual('broken', receipt['failures'][0]['classId']) + self.assertFalse((output / 'run-plan.json').exists()) + + def test_legacy_pilot_success_cannot_silently_change_the_new_admission_protocol(self): + with self.assertRaisesRegex(ValueError, 'profile-dependent'): + campaign.admits_primary_comparison({'eligible': True, 'missingRequiredScenarios': []}) + + def test_same_reserve_cannot_fill_two_selected_slots(self): + selected = {'origin': 'selected', 'order': 0, 'identity': 'one', 'projectId': 'p', 'stratum': 'complex'} + other = dict(selected, order=1, identity='two') + reserve = dict(selected, origin='reserve', classId='reserve') + + chosen, missing = campaign.select_classes({'decisions': [selected, other, reserve]}, {'reserve'}) + + self.assertEqual(['reserve'], chosen) + self.assertEqual('two', missing[0]['identity']) + + def test_serializer_defaults_preserve_all_explicit_frozen_values(self): + declared = {'classes': [{'inputs': [{'domain': {'kind': 'string', 'maxLength': 8}}]}], 'seeds': [17]} + prepared = {'classes': [{'inputs': [{'domain': {'kind': 'string', 'minLength': 0, 'maxLength': 8}}]}], + 'seeds': [17], 'schemaVersion': 1} + + self.assertTrue(campaign.preserves_input_values(prepared, declared)) + prepared['classes'][0]['inputs'][0]['domain']['maxLength'] = 9 + self.assertFalse(campaign.preserves_input_values(prepared, declared)) + self.assertFalse(campaign.preserves_input_values({'seeds': [18]}, declared)) + + def test_saved_plan_rejects_changed_runtime_or_eligibility(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / 'plan.json' + campaign.freeze(path, {'classIds': ['first'], 'revision': 'one'}) + + with self.assertRaisesRegex(ValueError, 'identity changed'): + campaign.freeze(path, {'classIds': ['second'], 'revision': 'one'}) + + def test_coverage_cannot_admit_a_foreign_statement(self): + with tempfile.TemporaryDirectory() as temporary: + prepared = Path(temporary) + directory = prepared / 'class' + directory.mkdir() + (directory / 'class.json').write_text('{}') + (directory / 'preflight.json').write_text('{}') + identity = {'classId': 'class', 'budgetMillis': 30_000, 'universeSha256': 'universe', + 'schemaVersion': 2, 'coveragePolicy': campaign.COVERAGE_POLICY, + 'suspendedCoveragePolicy': campaign.SUSPENDED_COVERAGE_POLICY, + 'preparedSha256': campaign.sha(directory / 'class.json'), + 'preflightSha256': campaign.sha(directory / 'preflight.json')} + result = {'identity': identity, 'search': {'search': {'candidates': []}}, + 'checkpoints': [{'searchBudgetMillis': 30_000, + 'totalStatements': 1, 'coveredStatementIds': ['foreign'], 'uncoveredStatementIds': []}]} + universe = {'universeSha256': 'universe', 'statements': [{'id': 'original'}]} + + with self.assertRaisesRegex(ValueError, 'numerator/denominator'): + campaign.validate_result(result, {'classId': 'class'}, prepared, universe) + + def test_new_measurement_cannot_reinterpret_sealed_legacy_results(self): + with self.assertRaisesRegex(ValueError, 'schema-2'): + campaign.validate_measurement_protocol({'schemaVersion': 1}) + + def test_native_primary_and_strict_secondary_are_reported_separately(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, manifest, plan = self.make_phase_fixture(output) + for profile in campaign.PROFILES: + self.write_phase_result(output, prepared, manifest, budget=30_000, profile=profile) + path = campaign.cell_directory(output, 'class', 17, 30_000, profile) / 'result.json' + result = campaign.read(path) + result['search']['search']['candidates'] = [ + {'origin': 'TERMINATED', 'completion': 'RETURNED', 'emittedAtMillis': 5}, + {'origin': 'SUSPENDED', 'suspendedAtMachineStop': True, 'emittedAtMillis': 30_050}, + {'origin': 'SUSPENDED', 'suspendedAtMachineStop': False, 'emittedAtMillis': 30_051}, + ] + final = result['checkpoints'][-1] + final.update(coveredStatementIds=['original'], uncoveredStatementIds=[], + acceptedCandidates=2, emittedCandidates=2, strictRejectedCandidates=1, + strictOracleNotApplicable=1, suspendedCandidates=1, postStopSuspendedCandidates=1) + result['oracleRejections'] = 1 + path.write_text(json.dumps(result) + '\n') + + summary = campaign.summarize(output, prepared, plan, scheduled_budgets=(30_000,)) + + self.assertEqual([], summary['incompleteOrInvalidCells']) + self.assertEqual({1}, {row['covered'] for row in summary['rows']}) + self.assertEqual({0}, {row['strictCovered'] for row in summary['rows']}) + self.assertFalse(summary['pairs'][0]['measurementFailures']) + self.assertTrue(summary['pairs'][0]['symbolicOracleDisagreement']) + self.assertEqual({1}, {row['postStopSuspendedCandidates'] for row in summary['rows']}) + + def test_post_stop_extraction_cannot_be_backdated_into_intermediate_checkpoint(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, manifest, _ = self.make_phase_fixture(output) + profile = campaign.PROFILES[0] + self.write_phase_result(output, prepared, manifest, budget=30_000, profile=profile) + path = campaign.cell_directory(output, 'class', 17, 30_000, profile) / 'result.json' + result = campaign.read(path) + result['search']['search']['candidates'] = [ + {'origin': 'SUSPENDED', 'suspendedAtMachineStop': True, 'emittedAtMillis': 10}, + ] + intermediate = dict(result['checkpoints'][0], searchBudgetMillis=1_000, + includesSuspendedExtraction=False, emittedCandidates=1, + acceptedCandidates=1, suspendedCandidates=1, postStopSuspendedCandidates=1, + strictOracleNotApplicable=1) + result['checkpoints'].insert(0, intermediate) + universe = campaign.read(prepared / 'class' / 'class.json')['instrumentation']['universe'] + + with self.assertRaisesRegex(ValueError, 'checkpoint attribution'): + campaign.validate_result(result, {}, prepared, universe) + + +if __name__ == '__main__': + unittest.main() diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassCoverageEngine.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassCoverageEngine.kt new file mode 100644 index 0000000000..9b7bfd632a --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassCoverageEngine.kt @@ -0,0 +1,491 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.Serializable +import org.jacodb.ets.model.EtsClassSignature +import org.jacodb.ets.model.EtsMethod +import org.usvm.CoverageZone +import org.usvm.PathSelectorFairnessStrategy +import org.usvm.SolverType +import org.usvm.StateCollectionStrategy +import org.usvm.UMachineOptions +import org.usvm.machine.TsAnalysisStopReason +import org.usvm.machine.TsEntryPointTimeQuantumEvent +import org.usvm.machine.TsInterpreterObserver +import org.usvm.machine.TsMachine +import org.usvm.machine.TsOptions +import org.usvm.machine.TsRuntimeFeatureLimitationEvent +import org.usvm.machine.TsSolverQueryEvent +import org.usvm.machine.call.TsUnknownCallDecision +import org.usvm.machine.call.TsUnknownCallEvent +import org.usvm.machine.call.TsUnknownCallModelSelection +import org.usvm.machine.interpreter.TsInitialStateSolverUnknownException +import org.usvm.machine.state.TsState +import org.usvm.statistics.CompositeUMachineObserver +import org.usvm.statistics.UMachineObserver +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.usvm.PlainObjectInputShapes +import kotlin.time.Duration +import kotlin.time.TimeSource + +internal data class CallsClassSearchRequest( + val harness: CallsClassPreparedHarness, + val profile: CallsExperimentProfile, + val frozenModelIds: Set, + val expectedNativeFrontendRevision: String, + val seed: Long, + val budget: Duration, + val timeQuantum: Duration, + val solverQueryLimit: Duration, + val solverType: SolverType, + val candidateCap: Int, + val requireEntryPointWitness: Boolean = false, + val onCandidate: (CallsCoverageCandidate) -> Unit = {}, +) { + init { + require(budget > Duration.ZERO && timeQuantum > Duration.ZERO && timeQuantum <= budget) + require(solverQueryLimit > Duration.ZERO && solverQueryLimit <= budget) + require(candidateCap > 0) + } +} + +@Serializable +internal data class CallsClassScenarioMetrics( + val scenarioId: String, + val executedSteps: Long, + val stepWallTimeMillis: Long, + val completedCandidates: Int, + val suspendedCandidates: Int = 0, +) + +@Serializable +internal data class CallsClassMethodMetrics( + val methodSignature: String, + val executedSteps: Long, + val stepWallTimeNanos: Long, +) + +/** Consecutive visits to an entry point; a single remaining entry can span multiple selector quanta. */ +@Serializable +internal data class CallsClassSchedulingVisit( + val scenarioId: String, + val startedAtMillis: Long, + val elapsedMillis: Long, + val requestedQuantumMillis: Long, +) + +@Serializable +internal data class CallsClassTimeQuantum( + val scenarioId: String, + val startedAtMillis: Long, + val requestedQuantumMillis: Long, + val elapsedNanos: Long, +) + +@Serializable +internal data class CallsClassSolverMetrics( + val phase: String, + val scenarioId: String?, + val methodSignature: String?, + val calls: Long, + val elapsedNanos: Long, + val failures: Long, + val statusCounts: Map, +) + +/** Ordered observer notifications for the entire search, without implying ancestry of any candidate. */ +@Serializable +internal data class CallsClassUnknownCallEvent( + val eventIndex: Int, + val callSite: CallsUnknownCallSite, + val callerMethodSignature: String, + val callee: CallsUnknownCallCallee, + val failureReason: String, + val decision: CallsUnknownCallDecisionKind, + val outcome: String, + val modelId: String? = null, + val residualPolicy: String? = null, +) { + init { + require(eventIndex >= 1) { "Unknown-call event index must be positive" } + require((decision == CallsUnknownCallDecisionKind.MODEL_APPLIED) == (modelId != null)) { + "Exactly a model-applied decision must carry a model ID" + } + require((decision == CallsUnknownCallDecisionKind.RESIDUAL_FALLBACK) == (residualPolicy != null)) { + "Exactly a residual-fallback decision must carry a residual policy" + } + } +} + +internal fun TsUnknownCallEvent.toCallsClassUnknownCallEvent(eventIndex: Int): CallsClassUnknownCallEvent { + val selectedDecision = decision + + return CallsClassUnknownCallEvent( + eventIndex = eventIndex, + callSite = callSite.toCallsUnknownCallSite(), + callerMethodSignature = callSite.location.method.signature.toString(), + callee = callee.toCallsUnknownCallCallee(), + failureReason = failureReason.name, + decision = when (selectedDecision) { + is TsUnknownCallDecision.ModelApplied -> CallsUnknownCallDecisionKind.MODEL_APPLIED + is TsUnknownCallDecision.ResidualFallback -> CallsUnknownCallDecisionKind.RESIDUAL_FALLBACK + }, + outcome = outcome.name, + modelId = (selectedDecision as? TsUnknownCallDecision.ModelApplied)?.modelId, + residualPolicy = (selectedDecision as? TsUnknownCallDecision.ResidualFallback)?.policy?.name, + ) +} + +@Serializable +internal data class CallsClassSearchResult( + val search: CallsCoverageSearchResult, + val scenarios: List, + val visits: List, + val unknownCallEvents: Int, + val methods: List = emptyList(), + val timeQuanta: List = emptyList(), + val solverQueries: List = emptyList(), + val solverCalls: Long? = null, + val solverElapsedMillis: Long? = null, + val unavailableMetrics: List = emptyList(), + val unknownCalls: List = emptyList(), +) + +/** A class owns one machine and one timeout; scenarios retain their path selectors between bounded visits. */ +internal class CurrentTsCallsClassCoverageEngine { + @Suppress("LongMethod") + fun search(request: CallsClassSearchRequest): CallsClassSearchResult { + val preparationStarted = TimeSource.Monotonic.markNow() + val sourceEngine = CurrentTsCallsSymbolicEngine() + val harness = request.harness + val preparations = harness.functions.map { function -> + sourceEngine.prepareFunction( + CallsFunctionPreflightRequest( + sourceRoot = harness.sourceRoot, + project = harness.project, + function = function, + expectedNativeFrontendRevision = request.expectedNativeFrontendRevision, + verifySourceCheckout = false, + ), + ) + } + val rejection = preparations.filterIsInstance().firstOrNull() + if (rejection != null) { + return rejectedSearch( + diagnostic = "${rejection.reasonCode}: ${rejection.diagnostic}", + preparationElapsedMillis = preparationStarted.elapsedNow().inWholeMilliseconds, + status = if (rejection.status == CallsSymbolicStatus.TOOL_ERROR) { + CallsCoverageSearchStatus.TOOL_ERROR + } else { + CallsCoverageSearchStatus.UNSUPPORTED + }, + ) + } + val eligible = preparations.filterIsInstance() + val first = eligible.first() + require(eligible.all { it.scene === first.scene }) { "Class scenarios must share one source scene" } + val scope = resolveCallsSymbolicCoverageScope( + preparation = first, + target = CallsSymbolicCoverageClass(harness.surface.sourceFile, harness.surface.exportName), + ) + if (scope is CallsSymbolicCoverageScope.Rejected) { + return rejectedSearch( + diagnostic = scope.diagnostic, + preparationElapsedMillis = preparationStarted.elapsedNow().inWholeMilliseconds, + status = scope.status, + ) + } + scope as CallsSymbolicCoverageScope.Selected + val objectShapes = PlainObjectInputShapes( + sourceScene = first.scene, + domains = harness.functions.flatMap { function -> function.inputs.map(PropertyInput::domain) }, + ) + val contexts = eligible.mapIndexed { index, preparation -> + val function = harness.functions[index] + CallsCoverageCandidateContext( + method = preparation.method, + inputDomains = function.inputs, + symbolicInputs = CallsSymbolicInputs( + inputs = function.inputs, + bindings = preparation.inputBindings, + lexicalEnvironment = preparation.lexicalEnvironment, + captures = preparation.captures, + sourceScene = first.scene, + objectShapes = objectShapes, + ), + entryPointId = harness.scenarios[index].id, + observationFields = harness.observations[index], + ) + } + val preparationElapsedMillis = preparationStarted.elapsedNow().inWholeMilliseconds + val byMethod = contexts.associateBy(CallsCoverageCandidateContext::method) + val candidates = mutableListOf() + val extractionFailures = mutableListOf() + val machineCallStarted = TimeSource.Monotonic.markNow() + val candidateObserver = CompletedCoverageCandidateObserver( + contexts = contexts, + coverageStatements = scope.statements, + candidateCap = request.candidateCap, + budgetMillis = request.budget.inWholeMilliseconds, + machineCallStarted = machineCallStarted, + onCandidate = { candidate -> + request.onCandidate(candidate) + candidates += candidate + }, + onExtractionFailure = extractionFailures::add, + requireEntryPointWitness = request.requireEntryPointWitness, + emitSuspendedCandidates = true, + ) + val schedulingObserver = ClassSchedulingObserver( + scenarioIds = contexts.associate { it.method to requireNotNull(it.entryPointId) }, + requestedQuantumMillis = request.timeQuantum.inWholeMilliseconds, + targetClass = requireNotNull(scope.classSignature), + ) + val limitations = linkedSetOf() + val unknownCalls = mutableListOf() + val interpreterObserver = object : TsInterpreterObserver { + override fun onRuntimeFeatureLimitation(event: TsRuntimeFeatureLimitationEvent) { + limitations += "${event.reason.name}: ${event.detail}" + } + + override fun onUnknownCall(event: TsUnknownCallEvent) { + unknownCalls += event.toCallsClassUnknownCallEvent(eventIndex = unknownCalls.size + 1) + } + + override fun onSolverQuery(event: TsSolverQueryEvent) { + schedulingObserver.recordSolverQuery(event) + } + + override fun onEntryPointTimeQuantum(event: TsEntryPointTimeQuantumEvent) { + schedulingObserver.recordTimeQuantum(event) + } + } + val modelIds = if (request.profile.usesFrozenModels) request.frozenModelIds else emptySet() + val inputCapacity = contexts.flatMap { it.inputDomains }.maxOfOrNull { it.domain.callsInputCapacity() } ?: 0 + val machineResult = runCatching { + TsMachine( + scene = objectShapes.scene, + options = UMachineOptions( + pathSelectionStrategies = listOf(CALLS_PATH_SELECTION_STRATEGY), + pathSelectorFairnessStrategy = PathSelectorFairnessStrategy.CONSTANT_TIME, + stateCollectionStrategy = StateCollectionStrategy.COVERED_NEW, + randomSeed = request.seed, + timeout = request.budget, + solverTimeout = request.solverQueryLimit, + solverType = request.solverType, + maxEntryPointTimeQuantum = request.timeQuantum, + coverageZone = CoverageZone.CLASS, + removeCoveredEntryPoints = false, + stopOnCoverage = CALLS_STOP_ON_COVERAGE, + stopOnTargetsReached = false, + throwExceptionOnStepFailure = true, + emitRemainingStatesOnStop = true, + ), + tsOptions = TsOptions( + maxArraySize = maxOf(TsOptions().maxArraySize, inputCapacity), + unknownCallModelSelection = TsUnknownCallModelSelection.Only(modelIds), + unknownCallFallback = request.profile.fallback, + classCoverageTargets = setOf(requireNotNull(scope.classSignature)), + eraseTypeAssertions = true, + ), + machineObserver = CompositeUMachineObserver(candidateObserver, schedulingObserver), + observer = interpreterObserver, + ).use { machine -> + machine.analyzeWithMetadata( + methods = contexts.map(CallsCoverageCandidateContext::method), + configureInitialState = { method, state -> + byMethod.getValue(method).symbolicInputs.initialize(state) + }, + ) + } + } + candidateObserver.finishTiming() + schedulingObserver.finishTiming() + val elapsedMillis = machineCallStarted.elapsedNow().inWholeMilliseconds + val outcome = machineResult.getOrNull() + val status = when { + machineResult.exceptionOrNull() is TsInitialStateSolverUnknownException -> CallsCoverageSearchStatus.UNKNOWN + machineResult.isFailure -> CallsCoverageSearchStatus.TOOL_ERROR + outcome?.timedOut == true -> CallsCoverageSearchStatus.TIMEOUT + outcome?.stopReason == TsAnalysisStopReason.EXHAUSTED -> CallsCoverageSearchStatus.EXHAUSTED + else -> CallsCoverageSearchStatus.STOPPED + } + val search = CallsCoverageSearchResult( + status = status, + candidates = candidates, + selectedStates = candidateObserver.selectedStates, + executedSteps = candidateObserver.executedSteps, + stepsWithinBudget = candidateObserver.stepsWithinBudget, + extractionFailures = extractionFailures, + candidateCapReached = candidateObserver.capReached, + preparationElapsedMillis = preparationElapsedMillis, + machineSetupElapsedMillis = candidateObserver.machineSetupElapsedMillis, + searchElapsedMillis = candidateObserver.searchElapsedMillis, + machineTeardownElapsedMillis = ( + elapsedMillis - candidateObserver.machineSetupElapsedMillis - candidateObserver.searchElapsedMillis + ).coerceAtLeast(0), + unsupportedCall = outcome?.unsupportedCall == true, + engineFailed = outcome?.engineFailed == true, + runtimeLimited = outcome?.runtimeLimited == true || limitations.isNotEmpty(), + runtimeLimitationReasons = (limitations + outcome?.unsupportedPaths.orEmpty()).toList(), + diagnostic = machineResult.exceptionOrNull()?.let { it.message ?: it::class.java.name }, + suspendedStates = candidateObserver.suspendedStates, + suspendedExtractionElapsedMillis = candidateObserver.suspendedExtractionElapsedMillis, + frontierFlushElapsedMillis = candidateObserver.frontierFlushElapsedMillis, + machineStopReason = outcome?.stopReason?.name, + ) + + return CallsClassSearchResult( + search = search, + scenarios = schedulingObserver.metrics(candidates), + visits = schedulingObserver.visits, + unknownCallEvents = unknownCalls.size, + unknownCalls = unknownCalls.toList(), + methods = schedulingObserver.methodMetrics(), + timeQuanta = schedulingObserver.timeQuanta, + solverQueries = schedulingObserver.solverMetrics(), + solverCalls = schedulingObserver.solverMetrics().sumOf { it.calls }, + solverElapsedMillis = schedulingObserver.solverMetrics().sumOf { it.elapsedNanos } / 1_000_000, + ) + } +} + +private class ClassSchedulingObserver( + private val scenarioIds: Map, + private val requestedQuantumMillis: Long, + private val targetClass: EtsClassSignature, +) : UMachineObserver { + private var machineStarted = TimeSource.Monotonic.markNow() + private var stepStarted = TimeSource.Monotonic.markNow() + private var currentMethod: EtsMethod? = null + private var visitStartedAtMillis = 0L + private val steps = mutableMapOf() + private val timesNanos = mutableMapOf() + private var executingMethod: EtsMethod? = null + private val methodSteps = mutableMapOf() + private val methodTimesNanos = mutableMapOf() + val visits = mutableListOf() + val timeQuanta = mutableListOf() + private var solverPhase = "SETUP" + private val solverEvents = linkedMapOf, MutableList>() + + fun recordTimeQuantum(event: TsEntryPointTimeQuantumEvent) { + timeQuanta += CallsClassTimeQuantum( + scenarioId = scenarioIds.getValue(event.entrypoint), + startedAtMillis = (machineStarted.elapsedNow() - event.elapsed).inWholeMilliseconds.coerceAtLeast(0), + requestedQuantumMillis = event.requested.inWholeMilliseconds, + elapsedNanos = event.elapsed.inWholeNanoseconds, + ) + } + + fun recordSolverQuery(event: TsSolverQueryEvent) { + val scenarioId = currentMethod?.let(scenarioIds::getValue) + val methodSignature = executingMethod?.signature?.toString() + solverEvents.getOrPut(Triple(solverPhase, scenarioId, methodSignature)) { mutableListOf() } += event + } + + fun solverMetrics(): List = solverEvents.map { (key, events) -> + CallsClassSolverMetrics( + phase = key.first, + scenarioId = key.second, + methodSignature = key.third, + calls = events.size.toLong(), + elapsedNanos = events.sumOf { it.elapsedNanos }, + failures = events.count { it.failure != null }.toLong(), + statusCounts = events.groupingBy { it.status?.name ?: "ERROR" }.eachCount().mapValues { it.value.toLong() }, + ) + } + + override fun onMachineStarted() { + machineStarted = TimeSource.Monotonic.markNow() + solverPhase = "SEARCH" + } + + override fun onStatePeeked(state: TsState) { + val elapsed = machineStarted.elapsedNow().inWholeMilliseconds + if (currentMethod != state.entrypoint) { + finishVisit(elapsed) + currentMethod = state.entrypoint + visitStartedAtMillis = elapsed + } + val activeMethod = if (state.callStack.isNotEmpty()) state.callStack.lastMethod() else null + executingMethod = activeMethod?.takeIf { it.signature.enclosingClass == targetClass } + stepStarted = TimeSource.Monotonic.markNow() + } + + override fun onState(parent: TsState, forks: Sequence) { + val elapsedNanos = stepStarted.elapsedNow().inWholeNanoseconds + steps.merge(parent.entrypoint, 1L, Long::plus) + timesNanos.merge(parent.entrypoint, elapsedNanos, Long::plus) + executingMethod?.let { method -> + methodSteps.merge(method, 1L, Long::plus) + methodTimesNanos.merge(method, elapsedNanos, Long::plus) + } + } + + override fun onMachineStopped() = finishTiming() + + fun finishTiming() { + finishVisit(machineStarted.elapsedNow().inWholeMilliseconds) + currentMethod = null + executingMethod = null + solverPhase = "TEARDOWN" + } + + fun methodMetrics(): List = methodSteps.map { (method, executedSteps) -> + CallsClassMethodMetrics( + methodSignature = method.signature.toString(), + executedSteps = executedSteps, + stepWallTimeNanos = methodTimesNanos[method] ?: 0, + ) + } + + fun metrics(candidates: List): List = + scenarioIds.map { (method, id) -> + CallsClassScenarioMetrics( + scenarioId = id, + executedSteps = steps[method] ?: 0, + stepWallTimeMillis = (timesNanos[method] ?: 0) / 1_000_000, + completedCandidates = candidates.count { + it.entryPointId == id && it.origin == CallsCoverageCandidateOrigin.TERMINATED + }, + suspendedCandidates = candidates.count { + it.entryPointId == id && it.origin == CallsCoverageCandidateOrigin.SUSPENDED + }, + ) + } + + private fun finishVisit(elapsed: Long) { + val method = currentMethod ?: return + visits += CallsClassSchedulingVisit( + scenarioId = scenarioIds.getValue(method), + startedAtMillis = visitStartedAtMillis, + elapsedMillis = elapsed - visitStartedAtMillis, + requestedQuantumMillis = requestedQuantumMillis, + ) + } +} + +private fun rejectedSearch( + diagnostic: String, + preparationElapsedMillis: Long, + status: CallsCoverageSearchStatus, +): CallsClassSearchResult = CallsClassSearchResult( + search = CallsCoverageSearchResult( + status = status, + candidates = emptyList(), + selectedStates = 0, + executedSteps = 0, + stepsWithinBudget = 0, + extractionFailures = emptyList(), + candidateCapReached = false, + preparationElapsedMillis = preparationElapsedMillis, + machineSetupElapsedMillis = 0, + searchElapsedMillis = 0, + machineTeardownElapsedMillis = 0, + diagnostic = diagnostic, + ), + scenarios = emptyList(), + visits = emptyList(), + unknownCallEvents = 0, + unavailableMetrics = listOf("solverCalls", "solverElapsedMillis", "exactSelectorQuantumBoundaries"), +) diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperiment.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperiment.kt new file mode 100644 index 0000000000..9da422ff6c --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperiment.kt @@ -0,0 +1,513 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.Serializable +import org.usvm.machine.call.TsBuiltInUnknownCallModels +import org.usvm.ts.pbt.model.PropertyDomain +import java.nio.file.Files +import java.nio.file.Path + +/** Frozen external correctness evidence, independent of the outcome of a treatment's symbolic pilot. */ +@Serializable +internal data class CallsClassCorrectnessFrontierFailure(val reason: String, val provenance: String) { + init { + require(reason.isNotBlank() && provenance.isNotBlank()) { + "Correctness limitations require evidence provenance" + } + } +} + +@Serializable +internal data class CallsClassCase( + val classId: String, + val projectId: String, + val revision: String, + val sourceRoot: String, + val sourceSha256: Map, + val surface: CallsClassSurface, + val correctnessFrontierFailures: List = emptyList(), +) { + init { + require(classId.matches(Regex("[A-Za-z0-9_-]+"))) { "Class ID must be a safe directory name" } + require(revision.matches(Regex("[0-9a-f]{40}"))) { "Source revision must be a Git commit" } + require(surface.sourceFile in sourceSha256) { "Source closure omits the class source" } + require(sourceSha256.values.all { it.matches(Regex("[0-9a-f]{64}")) }) { "Invalid source hashes" } + require( + sourceSha256.keys.all { path -> + val value = Path.of(path) + !value.isAbsolute && value.normalize() == value && !value.startsWith("..") + }, + ) { "Source closure contains an unsafe path" } + } + + fun originalRoot(manifestDirectory: Path): Path = manifestDirectory.resolve(sourceRoot).normalize().toRealPath() + + fun verifySources(root: Path, verifyCheckout: Boolean) { + val canonicalRoot = root.toRealPath() + if (verifyCheckout) verifyCallsGitCheckout(canonicalRoot, revision) + sourceSha256.forEach { (relative, expected) -> + val path = canonicalRoot.resolve(relative).normalize().toRealPath() + require(path.startsWith(canonicalRoot)) { "Source closure escapes its root: $relative" } + require(callsClassSourceSha256(Files.readAllBytes(path)) == expected) { + "Frozen class source changed: $relative" + } + } + } +} + +@Serializable +internal data class CallsClassExperimentManifest( + val schemaVersion: Int = 2, + val experimentId: String, + val toolRevision: String, + val nativeFrontendRevision: String, + val solver: String = "YICES", + val modelSet: CallsModelSetIdentity, + val seeds: List, + val budgetsMillis: List = CLASS_SEARCH_BUDGETS_MILLIS, + val maxSequenceLength: Int = 2, + val timeQuantumMillis: Long = 250, + val solverLimitMillis: Long = 1_000, + val candidateCap: Int = 10_000, + val classes: List, + val coveragePolicy: String = CLASS_NATIVE_COVERAGE_POLICY, + val suspendedCoveragePolicy: String = CLASS_SUSPENDED_COVERAGE_POLICY, +) { + init { + require(schemaVersion == 2) { "Class measurement requires the native coverage and suspended-input protocol" } + require(coveragePolicy == CLASS_NATIVE_COVERAGE_POLICY) + require(suspendedCoveragePolicy == CLASS_SUSPENDED_COVERAGE_POLICY) + require(experimentId.isNotBlank()) + require(toolRevision.matches(Regex("[0-9a-f]{40}"))) { "Tool revision must be clean" } + require(modelSet.toolRevision == toolRevision) { "Frozen model revision differs" } + require(solver == "YICES" || solver == "Z3") + require(seeds.isNotEmpty() && seeds.distinct().size == seeds.size) + require(budgetsMillis == CLASS_SEARCH_BUDGETS_MILLIS) { "Class treatments are exactly 30 and 120 seconds" } + require(maxSequenceLength in 1..2) + require(timeQuantumMillis in 1..budgetsMillis.first()) + require(solverLimitMillis in 1..budgetsMillis.first()) + require(candidateCap > 0) + require(classes.isNotEmpty() && classes.map { it.classId }.distinct().size == classes.size) + } + + fun verifyBuild(toolRevision: String, nativeFrontendRevision: String) { + require(this.toolRevision == toolRevision) { "Class manifest and running tool revision differ" } + require(this.nativeFrontendRevision == nativeFrontendRevision) { "Class manifest and frontend revision differ" } + val availableModels = TsBuiltInUnknownCallModels.catalog().modelIds.toSet() + require(availableModels.containsAll(modelSet.ids)) { "Frozen models are absent from this build" } + require(System.getenv("ETS_FRONTEND_SCRIPT") == null) { "ETS_FRONTEND_SCRIPT must be unset" } + } +} + +@Serializable +internal data class CallsClassPreparedRecord( + val classCase: CallsClassCase, + val originalSourceRoot: String, + val scenarios: List, + val functions: List, + val observations: List>, + val harnessFile: String, + val harnessSha256: String, + val instrumentation: CallsClassSourceInstrumentation, + val preparationElapsedMillis: Long = 0, +) { + fun harness(classDirectory: Path): CallsClassPreparedHarness { + val root = classDirectory.resolve("source").toRealPath() + classCase.verifySources(root, verifyCheckout = false) + require(callsClassSourceSha256(Files.readAllBytes(root.resolve(harnessFile))) == harnessSha256) { + "Generated class wrapper changed" + } + require(functions.map { it.functionId } == scenarios.map { it.id }) { "Scenario schedule differs" } + require(functions.all { it.sourceFile == harnessFile }) { "Scenarios do not share one wrapper module" } + require(observations.size == functions.size) + instrumentation.validateSurface(classCase.surface) + instrumentation.universe.verifyUnchanged(root) + + return CallsClassPreparedHarness( + surface = classCase.surface, + scenarios = scenarios, + sourceRoot = root, + project = CallsProjectCase( + projectId = classCase.projectId, + revision = classCase.revision, + sourceRoot = root.toString(), + development = true, + functions = functions, + ), + functions = functions, + observations = observations, + ) + } + + fun verifyOriginal() = classCase.verifySources(Path.of(originalSourceRoot), verifyCheckout = true) +} + +@Serializable +internal data class CallsClassPreparationFailure( + val classId: String, + val diagnostic: String, + val preparationElapsedMillis: Long = 0, +) + +@Serializable +internal enum class CallsClassNativeSmokeStatus { VERIFIED, INVALID, INCONCLUSIVE } + +@Serializable +internal data class CallsClassNativeSmokeResult( + val seed: Long, + val scenarioId: String, + val status: CallsClassNativeSmokeStatus, + val original: CallsClassReplayObservation? = null, + val instrumented: CallsClassReplayObservation? = null, + val diagnostic: String? = null, +) + +/** Check the native observation mechanism independently of symbolic policies and their selected candidates. */ +internal fun callsClassNativeConstructorSmoke( + harness: CallsClassPreparedHarness, + instrumentation: CallsClassSourceInstrumentation, + seed: Long, + timeoutMillis: Long, +): CallsClassNativeSmokeResult { + val index = harness.scenarios.indexOfFirst { it.memberIds.isEmpty() } + require(index >= 0) { "Class schedule omits the constructor observation scenario" } + val function = harness.functions[index] + val fields = harness.observations[index] + val scenarioId = harness.scenarios[index].id + val original = runCatching { + OriginalTypeScriptClassReplayer(harness.sourceRoot, function, fields).use { + it.sample(seed = seed, timeoutMillis = timeoutMillis) + } + } + val originalValue = original.getOrNull() + if (originalValue == null) { + val failure = requireNotNull(original.exceptionOrNull()) + return CallsClassNativeSmokeResult( + seed = seed, + scenarioId = scenarioId, + status = CallsClassNativeSmokeStatus.INCONCLUSIVE, + diagnostic = "Native original sampling did not complete: ${failure.message ?: failure::class.java.name}", + ) + } + + val instrumented = runCatching { + OriginalTypeScriptClassReplayer( + sourceRoot = harness.sourceRoot, + function = function, + observationFields = fields, + sourceOverride = instrumentation.instrumentedSource, + overrideSourceFile = harness.surface.sourceFile, + counterGlobalName = instrumentation.hitGlobalName, + ).use { it.replay(inputs = originalValue.inputs, timeoutMillis = timeoutMillis) } + } + val instrumentedValue = instrumented.getOrNull() + if (instrumentedValue == null) { + val failure = requireNotNull(instrumented.exceptionOrNull()) + return CallsClassNativeSmokeResult( + seed = seed, + scenarioId = scenarioId, + status = CallsClassNativeSmokeStatus.INCONCLUSIVE, + original = originalValue, + diagnostic = "Native instrumented replay did not complete: ${failure.message ?: failure::class.java.name}", + ) + } + + val invalid = callsClassNativeInfrastructureFailure(originalValue, instrumentedValue) + val checkedIds = runCatching { instrumentation.universe.checkedCoveredIds(instrumentedValue.coveredStatementKeys) } + val invalidDiagnostic = invalid ?: checkedIds.exceptionOrNull()?.let { it.message ?: it::class.java.name } + val representable = originalValue.observation.representable && instrumentedValue.observation.representable + val status = when { + invalidDiagnostic != null -> CallsClassNativeSmokeStatus.INVALID + representable -> CallsClassNativeSmokeStatus.VERIFIED + else -> CallsClassNativeSmokeStatus.INCONCLUSIVE + } + + val diagnostic = invalidDiagnostic ?: if (representable) { + null + } else { + "Native observation is incomplete at the sampled input" + } + return CallsClassNativeSmokeResult( + seed = seed, + scenarioId = scenarioId, + status = status, + original = originalValue, + instrumented = instrumentedValue, + diagnostic = diagnostic, + ) +} + +/** An incomplete lawful observation is not evidence of an instrumentation defect. */ +internal fun callsClassNativeInfrastructureFailure( + original: CallsClassReplayObservation, + instrumented: CallsClassReplayObservation, +): String? { + if (original.inputs != instrumented.inputs) return "Instrumentation changed the frozen native input transport" + if (original.observation.completion != instrumented.observation.completion) { + return "Instrumentation changed native completion" + } + if (original.observation.representable != instrumented.observation.representable) { + return "Instrumentation changed native observability or observation schema" + } + val completeObservations = original.observation.representable && instrumented.observation.representable + if (completeObservations && original.observation != instrumented.observation) { + return "Instrumentation changed native returned values or receiver state" + } + return null +} + +@Serializable +internal data class CallsClassPreflightResult( + val eligible: Boolean, + val classId: String, + val preparedSha256: String, + val toolRevision: String, + val search: CallsClassSearchResult, + val replays: List, + val missingRequiredScenarios: List, + val reasons: List, + val admissionVersion: Int = 0, + val sourceAdmissionFailures: List = emptyList(), + val nativeInfrastructureFailures: List = emptyList(), + val nativeSmoke: CallsClassNativeSmokeResult? = null, + val diagnosticReady: Boolean? = null, + val primaryInferenceEligible: Boolean? = null, + val correctnessFrontierFailures: List = emptyList(), +) { + init { + require(admissionVersion in 0..1) { "Unsupported class admission version" } + if (admissionVersion == 1) { + require(diagnosticReady != null && primaryInferenceEligible != null) { + "Version-one admission must distinguish pilot readiness and primary inference" + } + require(sourceAdmissionFailures.isNotEmpty() || nativeSmoke != null) { + "Source-ready admission requires a native probe" + } + } + } +} + +internal fun callsClassPreflightResult( + classCase: CallsClassCase, + preparedSha256: String, + toolRevision: String, + search: CallsClassSearchResult, + replays: List, + missingScenarios: List, + pilotDiagnostics: List, + sourceAdmissionFailures: List, + nativeSmoke: CallsClassNativeSmokeResult?, + nativeInfrastructureFailures: List, +): CallsClassPreflightResult { + require(sourceAdmissionFailures.isNotEmpty() || nativeSmoke != null) { + "Source-ready admission requires a native probe" + } + val invalidSmoke = nativeSmoke?.status == CallsClassNativeSmokeStatus.INVALID + val eligible = sourceAdmissionFailures.isEmpty() && !invalidSmoke + val primaryInferenceEligible = eligible && nativeSmoke?.status == CallsClassNativeSmokeStatus.VERIFIED && + classCase.correctnessFrontierFailures.isEmpty() + val admissionDiagnostics = sourceAdmissionFailures + nativeInfrastructureFailures + + listOfNotNull(nativeSmoke?.diagnostic) + + classCase.correctnessFrontierFailures.map { "${it.reason} (${it.provenance})" } + + return CallsClassPreflightResult( + eligible = eligible, + classId = classCase.classId, + preparedSha256 = preparedSha256, + toolRevision = toolRevision, + search = search, + replays = replays, + missingRequiredScenarios = missingScenarios, + reasons = admissionDiagnostics + pilotDiagnostics, + admissionVersion = 1, + sourceAdmissionFailures = sourceAdmissionFailures, + nativeInfrastructureFailures = nativeInfrastructureFailures, + nativeSmoke = nativeSmoke, + diagnosticReady = pilotDiagnostics.isEmpty() && nativeInfrastructureFailures.isEmpty(), + primaryInferenceEligible = primaryInferenceEligible, + correctnessFrontierFailures = classCase.correctnessFrontierFailures, + ) +} + +@Serializable +internal data class CallsClassReplayEvent( + val index: Int, + val scenarioId: String, + val original: CallsClassReplayObservation? = null, + val instrumented: CallsClassReplayObservation? = null, + val oracle: CallsClassOracleResult, + val error: String? = null, + val replayElapsedMillis: Long = 0, + val nativeCoverage: CallsClassOracleResult, + val origin: CallsCoverageCandidateOrigin = CallsCoverageCandidateOrigin.TERMINATED, + val strictOracleApplicable: Boolean = true, +) + +/** Primary coverage checks the native measurement, independently of symbolic execution predictions. */ +internal fun callsClassNativeCoverageAcceptance( + original: CallsClassReplayObservation?, + instrumented: CallsClassReplayObservation?, + universe: CallsClassSourceUniverse, + error: String? = null, +): CallsClassOracleResult { + if (error != null || original == null || instrumented == null) { + return CallsClassOracleResult(accepted = false, diagnostic = error ?: "Concrete replay missing") + } + + val foreignIds = runCatching { universe.checkedCoveredIds(instrumented.coveredStatementKeys) }.exceptionOrNull() + val failure = callsClassNativeInfrastructureFailure(original, instrumented) + ?: foreignIds?.let { it.message ?: it::class.java.name } + if (failure != null) return CallsClassOracleResult(accepted = false, diagnostic = failure) + + val complete = original.observation.representable && instrumented.observation.representable + if (!complete) { + return CallsClassOracleResult(accepted = false, diagnostic = "Native observation is incomplete") + } + + return CallsClassOracleResult(accepted = true) +} + +@Serializable +internal data class CallsClassCellIdentity( + val schemaVersion: Int = 2, + val experimentId: String, + val toolRevision: String, + val nativeFrontendRevision: String, + val manifestSha256: String, + val preparedSha256: String, + val preflightSha256: String, + val universeSha256: String, + val classId: String, + val profile: CallsExperimentProfile, + val seed: Long, + val budgetMillis: Long, + val coveragePolicy: String = CLASS_NATIVE_COVERAGE_POLICY, + val suspendedCoveragePolicy: String = CLASS_SUSPENDED_COVERAGE_POLICY, +) + +@Serializable +internal data class CallsClassCoverageCheckpoint( + val searchBudgetMillis: Long, + val emittedCandidates: Int, + val acceptedCandidates: Int, + val rejectedCandidates: Int, + val coveredStatementIds: Set, + val uncoveredStatementIds: Set, + val totalStatements: Int, + val strictAcceptedCandidates: Int, + val strictRejectedCandidates: Int, + val strictOracleNotApplicable: Int, + val strictCoveredStatementIds: Set, + val strictUncoveredStatementIds: Set, + val suspendedCandidates: Int, + val includesSuspendedExtraction: Boolean, + val postStopSuspendedCandidates: Int, +) + +@Serializable +internal data class CallsClassCellResult( + val identity: CallsClassCellIdentity, + val search: CallsClassSearchResult, + val checkpoints: List, + val replayErrors: Int, + val oracleRejections: Int, + val replayElapsedMillis: Long, + val cellElapsedMillis: Long, + val primaryInferenceEligible: Boolean? = null, + val correctnessFrontierFailures: List = emptyList(), + val nativeInfrastructureFailures: List = emptyList(), + val nativeCoverageRejections: Int = 0, + val strictOracleNotApplicable: Int = 0, +) + +internal fun callsClassCoverageCheckpoints( + search: CallsClassSearchResult, + events: List, + universe: CallsClassSourceUniverse, + budgetMillis: Long, +): List { + require(events.size == search.search.candidates.size) { "Incomplete class replay journal" } + val checkpoints = (CLASS_COVERAGE_CHECKPOINTS_MILLIS + budgetMillis) + .filter { it <= budgetMillis }.distinct().sorted() + + return checkpoints.map { checkpoint -> + val indices = search.search.candidates.indices.filter { index -> + val candidate = search.search.candidates[index] + when (candidate.origin) { + CallsCoverageCandidateOrigin.TERMINATED -> + candidate.emittedAtMillis <= checkpoint + CallsCoverageCandidateOrigin.SUSPENDED -> if (candidate.suspendedAtMachineStop) { + checkpoint == budgetMillis + } else { + candidate.emittedAtMillis <= checkpoint + } + } + } + val selectedEvents = indices.map { events[it] } + indices.forEach { index -> + val event = events[index] + val candidate = search.search.candidates[index] + val matchesCandidate = event.index == index && + event.scenarioId == candidate.entryPointId && event.origin == candidate.origin + require(matchesCandidate) { + "Class replay event differs from its candidate" + } + require(event.strictOracleApplicable == (candidate.origin == CallsCoverageCandidateOrigin.TERMINATED)) { + "Class replay symbolic oracle applicability differs from its candidate" + } + val expectedNative = callsClassNativeCoverageAcceptance( + original = event.original, + instrumented = event.instrumented, + universe = universe, + error = event.error, + ) + require(event.nativeCoverage == expectedNative) { "Class replay native coverage decision differs" } + } + val accepted = selectedEvents.filter { it.nativeCoverage.accepted && it.error == null } + val covered = universe.checkedCoveredIds( + accepted.flatMap { event -> requireNotNull(event.instrumented).coveredStatementKeys }, + ) + val strictAccepted = selectedEvents.filter { + it.strictOracleApplicable && it.oracle.accepted && it.error == null + } + val strictCovered = universe.checkedCoveredIds( + strictAccepted.flatMap { event -> requireNotNull(event.instrumented).coveredStatementKeys }, + ) + val strictNotApplicable = selectedEvents.count { !it.strictOracleApplicable } + CallsClassCoverageCheckpoint( + searchBudgetMillis = checkpoint, + emittedCandidates = indices.size, + acceptedCandidates = accepted.size, + rejectedCandidates = indices.size - accepted.size, + coveredStatementIds = covered, + uncoveredStatementIds = universe.statementIds - covered, + totalStatements = universe.statementIds.size, + strictAcceptedCandidates = strictAccepted.size, + strictRejectedCandidates = indices.size - strictAccepted.size - strictNotApplicable, + strictOracleNotApplicable = strictNotApplicable, + strictCoveredStatementIds = strictCovered, + strictUncoveredStatementIds = universe.statementIds - strictCovered, + suspendedCandidates = indices.count { + search.search.candidates[it].origin == CallsCoverageCandidateOrigin.SUSPENDED + }, + includesSuspendedExtraction = checkpoint == budgetMillis, + postStopSuspendedCandidates = indices.count { search.search.candidates[it].suspendedAtMachineStop }, + ) + } +} + +internal const val CLASS_NATIVE_COVERAGE_POLICY = "native-original-instrumented-complete-v2" +internal const val CLASS_SUSPENDED_COVERAGE_POLICY = "reached-prefix-and-final-frontier-inputs-v1" + +private const val SHORT_CLASS_BUDGET_MILLIS = 30_000L +private const val LONG_CLASS_BUDGET_MILLIS = 120_000L +private const val FIRST_CLASS_CHECKPOINT_MILLIS = 1_000L +private const val SECOND_CLASS_CHECKPOINT_MILLIS = 5_000L +private const val THIRD_CLASS_CHECKPOINT_MILLIS = 10_000L +private val CLASS_SEARCH_BUDGETS_MILLIS = listOf(SHORT_CLASS_BUDGET_MILLIS, LONG_CLASS_BUDGET_MILLIS) +private val CLASS_COVERAGE_CHECKPOINTS_MILLIS = listOf( + FIRST_CLASS_CHECKPOINT_MILLIS, + SECOND_CLASS_CHECKPOINT_MILLIS, + THIRD_CLASS_CHECKPOINT_MILLIS, + SHORT_CLASS_BUDGET_MILLIS, + LONG_CLASS_BUDGET_MILLIS, +) diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperimentCli.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperimentCli.kt new file mode 100644 index 0000000000..203cb4eaac --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperimentCli.kt @@ -0,0 +1,553 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.decodeFromString +import org.usvm.SolverType +import org.usvm.machine.call.TsBuiltInUnknownCallModels +import java.nio.file.Files +import java.nio.file.Path +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.TimeSource + +internal fun runClassCatalog(args: List) { + require(args.isEmpty()) { "class-catalog takes no arguments" } + val identity = CallsModelSetIdentity( + ids = TsBuiltInUnknownCallModels.catalog().modelIds.sorted().toSet(), + toolRevision = CallsBuildIdentity.toolRevision, + ) + System.out.appendLine(encodeCallsArtifact(identity)) +} + +internal fun runClassPrepare(args: List) { + require(args.size == 2) { "class-prepare requires " } + val manifestPath = Path.of(args[0]).toRealPath() + val manifest = decodeClassArtifact(manifestPath) + manifest.verifyBuild(CallsBuildIdentity.toolRevision, CallsBuildIdentity.nativeFrontendRevision) + val output = Path.of(args[1]).toAbsolutePath().normalize() + Files.createDirectories(output) + require(Files.list(output).use { it.findAny().isEmpty }) { "Preparation output must be empty" } + writeCallsArtifactAtomically(output.resolve("manifest.json"), encodeCallsArtifact(manifest)) + for (classCase in manifest.classes) { + val classDirectory = output.resolve(classCase.classId) + Files.createDirectories(classDirectory) + val preparationStarted = TimeSource.Monotonic.markNow() + val preparation = runCatching { + val originalRoot = classCase.originalRoot(requireNotNull(manifestPath.parent)) + classCase.verifySources(originalRoot, verifyCheckout = true) + val overlay = classDirectory.resolve("source") + Files.createDirectories(overlay) + classCase.sourceSha256.keys.forEach { relative -> + val copied = overlay.resolve(relative) + Files.createDirectories(copied.parent) + Files.copy(originalRoot.resolve(relative), copied) + } + val instrumentation = CallsClassSourceInstrumentation.create( + sourceRoot = originalRoot, + sourceFile = classCase.surface.sourceFile, + className = classCase.surface.exportName, + ) + instrumentation.validateSurface(classCase.surface) + val project = CallsProjectCase( + projectId = classCase.projectId, + revision = classCase.revision, + sourceRoot = overlay.toString(), + development = true, + functions = emptyList(), + ) + val harness = prepareCallsClassHarness( + sourceRoot = overlay, + project = project, + surface = classCase.surface, + maxSequenceLength = manifest.maxSequenceLength, + ) + val harnessFile = harness.functions.first().sourceFile + classCase.verifySources(originalRoot, verifyCheckout = true) + CallsClassPreparedRecord( + classCase = classCase, + originalSourceRoot = originalRoot.toString(), + scenarios = harness.scenarios, + functions = harness.functions, + observations = harness.observations, + harnessFile = harnessFile, + harnessSha256 = callsClassSourceSha256(Files.readAllBytes(overlay.resolve(harnessFile))), + instrumentation = instrumentation, + preparationElapsedMillis = preparationStarted.elapsedNow().inWholeMilliseconds, + ) + } + val prepared = preparation.getOrNull() + if (prepared != null) { + writeCallsArtifactAtomically(classDirectory.resolve("class.json"), encodeCallsArtifact(prepared)) + val statementCount = prepared.instrumentation.universe.statements.size + System.out.appendLine("Prepared ${classCase.classId}: $statementCount statements") + } else { + val error = requireNotNull(preparation.exceptionOrNull()) + val failure = CallsClassPreparationFailure( + classId = classCase.classId, + diagnostic = error.message ?: error::class.java.name, + preparationElapsedMillis = preparationStarted.elapsedNow().inWholeMilliseconds, + ) + writeCallsArtifactAtomically( + classDirectory.resolve("preparation-failure.json"), + encodeCallsArtifact(failure), + ) + System.out.appendLine(encodeCallsArtifact(failure)) + } + } +} + +/** Source/native admission is independent of the diagnostic pilot and any treatment completion rate. */ +internal fun runClassPreflight( + args: List, + searchFunction: (CallsClassSearchRequest) -> CallsClassSearchResult = CurrentTsCallsClassCoverageEngine()::search, +) { + require(args.size == CLASS_PREFLIGHT_ARGUMENT_COUNT) { + "class-preflight requires " + } + val source = loadPreparedClass(Path.of(args[0]), classId = args[1]) + val budgetMillis = args[2].toLong() + require(budgetMillis >= source.manifest.solverLimitMillis && budgetMillis >= source.manifest.timeQuantumMillis) + val output = source.directory.resolve("preflight") + Files.createDirectories(output) + require(Files.list(output).use { it.findAny().isEmpty }) { "Class preflight already started" } + val sourceAdmissionFailures = callsClassSourceAdmissionFailures( + harness = source.harness, + expectedNativeFrontendRevision = source.manifest.nativeFrontendRevision, + ) + val nativeSmoke = if (sourceAdmissionFailures.isEmpty()) { + callsClassNativeConstructorSmoke( + harness = source.harness, + instrumentation = source.record.instrumentation, + seed = source.manifest.seeds.first(), + timeoutMillis = CLASS_REPLAY_TIMEOUT_MILLIS, + ) + } else { + null + } + writeCallsArtifactAtomically(output.resolve("native-smoke.json"), encodeCallsArtifact(nativeSmoke)) + val candidatesPath = output.resolve("candidates.jsonl") + val search = searchFunction( + source.searchRequest( + profile = CallsExperimentProfile.FROZEN_FRESH, + seed = source.manifest.seeds.first(), + budgetMillis = budgetMillis, + requireEntryPointWitness = true, + onCandidate = { appendCallsJournalDurably(candidatesPath, encodeCallsArtifact(it)) }, + ), + ) + source.verify() + require(readCallsJournal(candidatesPath) == search.search.candidates) { + "Preflight candidate journal differs" + } + writeCallsArtifactAtomically(output.resolve("search.json"), encodeCallsArtifact(search)) + val events = replayClassCandidates(source, search, output) + val acceptedIds = events.filter { it.oracle.accepted && it.error == null }.mapTo(hashSetOf()) { it.scenarioId } + val missing = source.record.scenarios.map { it.id }.filter { it !in acceptedIds } + val pilotDiagnostics = callsClassPilotDiagnostics(search, events, missing) + val nativeInfrastructureFailures = callsClassPilotNativeFailures(events, source.record.instrumentation.universe) + val result = callsClassPreflightResult( + classCase = source.record.classCase, + preparedSha256 = source.preparedSha256, + toolRevision = source.manifest.toolRevision, + search = search, + replays = events, + missingScenarios = missing, + pilotDiagnostics = pilotDiagnostics, + sourceAdmissionFailures = sourceAdmissionFailures, + nativeSmoke = nativeSmoke, + nativeInfrastructureFailures = nativeInfrastructureFailures, + ) + source.verify() + writeCallsArtifactAtomically(source.directory.resolve("preflight.json"), encodeCallsArtifact(result)) + System.out.appendLine(encodeCallsArtifact(result)) +} + +private fun callsClassPilotDiagnostics( + search: CallsClassSearchResult, + events: List, + missing: List, +): List = buildList { + if (missing.isNotEmpty()) add("No exact returned-value/state witness for scenarios: $missing") + if (search.search.engineFailed) add("Symbolic engine failed") + if (search.search.unsupportedCall) add("Technical preflight stopped an unsupported call") + if (search.search.runtimeLimited) addAll(search.search.runtimeLimitationReasons) + if (search.search.extractionFailures.isNotEmpty()) addAll(search.search.extractionFailures) + if (search.search.candidateCapReached) add("Technical preflight reached candidate cap") + if (search.search.status == CallsCoverageSearchStatus.TIMEOUT) add("Symbolic pilot reached the time budget") + if (events.any { it.error != null }) add("Concrete preflight replay failed") + if (events.any { it.strictOracleApplicable && !it.oracle.accepted }) { + add("Technical preflight observation oracle rejected a candidate") + } + val terminalFailure = search.search.status in setOf( + CallsCoverageSearchStatus.UNSUPPORTED, + CallsCoverageSearchStatus.UNMAPPED, + CallsCoverageSearchStatus.AMBIGUOUS, + CallsCoverageSearchStatus.UNKNOWN, + CallsCoverageSearchStatus.TOOL_ERROR, + ) + if (terminalFailure) add(search.search.diagnostic ?: search.search.status.name) +} + +internal fun callsClassPilotNativeFailures( + events: List, + universe: CallsClassSourceUniverse, +): List = buildList { + for (event in events) { + val original = event.original + val instrumented = event.instrumented + if (original != null && instrumented != null) { + val failure = callsClassNativeInfrastructureFailure(original, instrumented) + if (failure != null) add("${event.scenarioId}: $failure") + } + if (instrumented != null) { + val checkedIds = runCatching { universe.checkedCoveredIds(instrumented.coveredStatementKeys) } + checkedIds.exceptionOrNull()?.let { add(it.message ?: it::class.java.name) } + } + } +} + +internal fun runClassCell( + args: List, + searchFunction: (CallsClassSearchRequest) -> CallsClassSearchResult = CurrentTsCallsClassCoverageEngine()::search, +) { + require(args.size == CLASS_CELL_ARGUMENT_COUNT) { + "class-cell requires " + } + val cellStarted = TimeSource.Monotonic.markNow() + val arguments = args.iterator() + val preparedDirectory = Path.of(arguments.next()) + val classId = arguments.next() + val source = loadPreparedClass(preparedDirectory, classId = classId) + val profile = CallsExperimentProfile.valueOf(arguments.next()) + val seed = arguments.next().toLong() + val budgetMillis = arguments.next().toLong() + require(seed in source.manifest.seeds && budgetMillis in source.manifest.budgetsMillis) { + "Cell is outside the frozen paired seed/budget schedule" + } + val preflightPath = source.directory.resolve("preflight.json") + val preflight = decodeClassArtifact(preflightPath) + source.validatePreflightAdmission(preflight) + val output = Path.of(arguments.next()).toAbsolutePath().normalize() + Files.createDirectories(output) + val identity = CallsClassCellIdentity( + experimentId = source.manifest.experimentId, + toolRevision = source.manifest.toolRevision, + nativeFrontendRevision = source.manifest.nativeFrontendRevision, + manifestSha256 = source.manifestSha256, + preparedSha256 = source.preparedSha256, + preflightSha256 = callsArtifactSha256(Files.readAllBytes(preflightPath)), + universeSha256 = source.record.instrumentation.universe.universeSha256, + classId = source.record.classCase.classId, + profile = profile, + seed = seed, + budgetMillis = budgetMillis, + ) + val identityPath = output.resolve("identity.json") + if (Files.exists(identityPath)) { + require(decodeClassArtifact(identityPath) == identity) { "Class cell identity changed" } + } else { + require(Files.list(output).use { it.findAny().isEmpty }) { "Cell output is not empty" } + writeCallsArtifactAtomically(identityPath, encodeCallsArtifact(identity)) + } + require(!Files.exists(output.resolve("result.json"))) { "Class cell is already complete" } + val candidatesPath = output.resolve("candidates.jsonl") + val searchPath = output.resolve("search.json") + val search = if (Files.exists(searchPath)) { + decodeClassArtifact(searchPath) + } else { + require(!Files.exists(candidatesPath)) { "Interrupted class search requires investigation before restart" } + val result = searchFunction( + source.searchRequest( + profile = profile, + seed = seed, + budgetMillis = budgetMillis, + requireEntryPointWitness = false, + onCandidate = { appendCallsJournalDurably(candidatesPath, encodeCallsArtifact(it)) }, + ), + ) + source.verify() + require(readCallsJournal(candidatesPath) == result.search.candidates) { + "Class candidate journal differs from search result" + } + writeCallsArtifactAtomically(searchPath, encodeCallsArtifact(result)) + result + } + require(readCallsJournal(candidatesPath) == search.search.candidates) { + "Persisted class search and candidate journal differ" + } + val events = replayClassCandidates(source, search, output) + source.verify() + val result = CallsClassCellResult( + identity = identity, + search = search, + checkpoints = callsClassCoverageCheckpoints( + search = search, + events = events, + universe = source.record.instrumentation.universe, + budgetMillis = budgetMillis, + ), + replayErrors = events.count { it.error != null }, + oracleRejections = events.count { it.strictOracleApplicable && !it.oracle.accepted }, + replayElapsedMillis = events.sumOf { it.replayElapsedMillis }, + cellElapsedMillis = cellStarted.elapsedNow().inWholeMilliseconds, + primaryInferenceEligible = preflight.primaryInferenceEligible, + correctnessFrontierFailures = preflight.correctnessFrontierFailures, + nativeInfrastructureFailures = callsClassPilotNativeFailures(events, source.record.instrumentation.universe), + nativeCoverageRejections = events.count { !it.nativeCoverage.accepted }, + strictOracleNotApplicable = events.count { !it.strictOracleApplicable }, + ) + writeCallsArtifactAtomically(output.resolve("result.json"), encodeCallsArtifact(result)) + System.out.appendLine(encodeCallsArtifact(result)) +} + +private fun PreparedClassSource.validatePreflightAdmission(preflight: CallsClassPreflightResult) { + require(preflight.admissionVersion == 1) { "Class requires a new profile-independent admission check" } + require(preflight.eligible) { "Class source/native admission failed: ${preflight.reasons}" } + require(preflight.correctnessFrontierFailures == record.classCase.correctnessFrontierFailures) { + "Frozen correctness-frontier evidence differs from the prepared class" + } + val expectedPrimaryEligibility = preflight.eligible && + preflight.nativeSmoke?.status == CallsClassNativeSmokeStatus.VERIFIED && + preflight.correctnessFrontierFailures.isEmpty() + require(preflight.primaryInferenceEligible == expectedPrimaryEligibility) { "Primary admission evidence differs" } + require(preflight.classId == record.classCase.classId) + val preflightIdentityMatches = preflight.preparedSha256 == preparedSha256 && + preflight.toolRevision == manifest.toolRevision + require(preflightIdentityMatches) { "Class preflight identity differs" } +} + +/** Parsing, domain binding and source coverage selection run without a symbolic treatment or machine. */ +internal fun callsClassSourceAdmissionFailures( + harness: CallsClassPreparedHarness, + expectedNativeFrontendRevision: String, +): List = runCatching { + val engine = CurrentTsCallsSymbolicEngine() + val preparations = harness.functions.map { function -> + engine.prepareFunction( + CallsFunctionPreflightRequest( + sourceRoot = harness.sourceRoot, + project = harness.project, + function = function, + expectedNativeFrontendRevision = expectedNativeFrontendRevision, + verifySourceCheckout = false, + ), + ) + } + val failures = preparations.filterIsInstance().map { + "${it.reasonCode}: ${it.diagnostic}" + } + if (failures.isNotEmpty()) { + failures + } else { + val first = preparations.first() as CallsFunctionPreparation.Eligible + val scope = resolveCallsSymbolicCoverageScope( + preparation = first, + target = CallsSymbolicCoverageClass(harness.surface.sourceFile, harness.surface.exportName), + ) + if (scope is CallsSymbolicCoverageScope.Rejected) listOf(scope.diagnostic) else emptyList() + } +}.getOrElse { listOf("Source preparation failed: ${it.message ?: it::class.java.name}") } + +private class PreparedClassSource( + val manifest: CallsClassExperimentManifest, + val manifestSha256: String, + val directory: Path, + val record: CallsClassPreparedRecord, + val preparedSha256: String, + val harness: CallsClassPreparedHarness, +) { + fun verify() { + record.verifyOriginal() + record.harness(directory) + } + + fun searchRequest( + profile: CallsExperimentProfile, + seed: Long, + budgetMillis: Long, + requireEntryPointWitness: Boolean, + onCandidate: (CallsCoverageCandidate) -> Unit, + ): CallsClassSearchRequest = CallsClassSearchRequest( + harness = harness, + profile = profile, + frozenModelIds = manifest.modelSet.ids, + expectedNativeFrontendRevision = manifest.nativeFrontendRevision, + seed = seed, + budget = budgetMillis.milliseconds, + timeQuantum = manifest.timeQuantumMillis.milliseconds, + solverQueryLimit = manifest.solverLimitMillis.milliseconds, + solverType = SolverType.valueOf(manifest.solver), + candidateCap = manifest.candidateCap, + requireEntryPointWitness = requireEntryPointWitness, + onCandidate = onCandidate, + ) +} + +private fun loadPreparedClass(preparedDirectory: Path, classId: String): PreparedClassSource { + val root = preparedDirectory.toRealPath() + val manifestPath = root.resolve("manifest.json") + val manifest = decodeClassArtifact(manifestPath) + manifest.verifyBuild(CallsBuildIdentity.toolRevision, CallsBuildIdentity.nativeFrontendRevision) + val classCase = manifest.classes.single { it.classId == classId } + val directory = root.resolve(classId).toRealPath() + require(directory.startsWith(root)) + val recordPath = directory.resolve("class.json") + val record = decodeClassArtifact(recordPath) + require(record.classCase == classCase) { "Prepared class differs from manifest" } + record.verifyOriginal() + + return PreparedClassSource( + manifest = manifest, + manifestSha256 = callsArtifactSha256(Files.readAllBytes(manifestPath)), + directory = directory, + record = record, + preparedSha256 = callsArtifactSha256(Files.readAllBytes(recordPath)), + harness = record.harness(directory), + ) +} + +private fun replayClassCandidates( + source: PreparedClassSource, + search: CallsClassSearchResult, + output: Path, +): List { + val path = output.resolve("replays.jsonl") + val events = readCallsJournal(path).toMutableList() + require(events.map { it.index } == events.indices.toList() && events.size <= search.search.candidates.size) { + "Class replay journal has missing, duplicate, or excess indices" + } + events.forEachIndexed { index, event -> + validateCallsClassReplayEvent( + candidate = search.search.candidates[index], + index = index, + event = event, + universe = source.record.instrumentation.universe, + ) + } + val scenarios = source.record.scenarios.withIndex().associate { it.value.id to it.index } + for (index in search.search.candidates.indices.drop(events.size)) { + source.verify() + val candidate = search.search.candidates[index] + val scenarioId = requireNotNull(candidate.entryPointId) + val scenarioIndex = scenarios.getValue(scenarioId) + val function = source.record.functions[scenarioIndex] + val observations = source.record.observations[scenarioIndex] + val replayStarted = TimeSource.Monotonic.markNow() + val original = runCatching { + OriginalTypeScriptClassReplayer( + sourceRoot = source.harness.sourceRoot, + function = function, + observationFields = observations, + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = CLASS_REPLAY_TIMEOUT_MILLIS) } + } + val instrumentation = source.record.instrumentation + val instrumented = runCatching { + OriginalTypeScriptClassReplayer( + sourceRoot = source.harness.sourceRoot, + function = function, + observationFields = observations, + sourceOverride = instrumentation.instrumentedSource, + overrideSourceFile = source.record.classCase.surface.sourceFile, + counterGlobalName = instrumentation.hitGlobalName, + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = CLASS_REPLAY_TIMEOUT_MILLIS) } + } + source.verify() + val originalValue = original.getOrNull() + val instrumentedValue = instrumented.getOrNull() + val validation = runCatching { + require(originalValue != null && instrumentedValue != null) { "Concrete replay failed" } + require(originalValue.inputs == candidate.inputs && instrumentedValue.inputs == candidate.inputs) { + "Class replay input snapshot differs from candidate" + } + instrumentation.universe.checkedCoveredIds(instrumentedValue.coveredStatementKeys) + classCandidateStrictOracle(candidate, originalValue, instrumentedValue) + } + val failures = listOfNotNull( + original.exceptionOrNull()?.let { "original: ${it.message ?: it::class.java.name}" }, + instrumented.exceptionOrNull()?.let { "instrumented: ${it.message ?: it::class.java.name}" }, + validation.exceptionOrNull()?.let { "validation: ${it.message ?: it::class.java.name}" }, + ) + val replayError = failures.takeIf { it.isNotEmpty() }?.joinToString(separator = "; ") + val event = CallsClassReplayEvent( + index = index, + scenarioId = scenarioId, + original = originalValue, + instrumented = instrumentedValue, + oracle = validation.getOrNull() ?: CallsClassOracleResult( + accepted = false, + diagnostic = "Concrete replay failed", + ), + error = replayError, + replayElapsedMillis = replayStarted.elapsedNow().inWholeMilliseconds, + nativeCoverage = callsClassNativeCoverageAcceptance( + original = originalValue, + instrumented = instrumentedValue, + universe = instrumentation.universe, + error = replayError, + ), + origin = candidate.origin, + strictOracleApplicable = candidate.origin == CallsCoverageCandidateOrigin.TERMINATED, + ) + appendCallsJournalDurably(path, encodeCallsArtifact(event)) + events += event + } + + return events +} + +internal fun validateCallsClassReplayEvent( + candidate: CallsCoverageCandidate, + index: Int, + event: CallsClassReplayEvent, + universe: CallsClassSourceUniverse, +) { + require(event.index == index && event.scenarioId == candidate.entryPointId) { + "Persisted replay index or scenario differs from candidate" + } + require(event.origin == candidate.origin) { "Persisted replay candidate origin differs" } + require(event.strictOracleApplicable == (candidate.origin == CallsCoverageCandidateOrigin.TERMINATED)) { + "Persisted symbolic oracle applicability differs from candidate origin" + } + if (event.error == null) { + val original = requireNotNull(event.original) + val instrumented = requireNotNull(event.instrumented) + require(original.inputs == candidate.inputs && instrumented.inputs == candidate.inputs) { + "Persisted replay input snapshot differs" + } + universe.checkedCoveredIds(instrumented.coveredStatementKeys) + val expectedOracle = classCandidateStrictOracle(candidate, original, instrumented) + require(event.oracle == expectedOracle) { "Persisted replay oracle decision differs" } + } else { + require(!event.oracle.accepted) { "Failed persisted replay is marked accepted" } + } + + val expectedNative = callsClassNativeCoverageAcceptance( + original = event.original, + instrumented = event.instrumented, + universe = universe, + error = event.error, + ) + require(event.nativeCoverage == expectedNative) { "Persisted native coverage decision differs" } +} + +private fun classCandidateStrictOracle( + candidate: CallsCoverageCandidate, + original: CallsClassReplayObservation, + instrumented: CallsClassReplayObservation, +): CallsClassOracleResult = when (candidate.origin) { + CallsCoverageCandidateOrigin.TERMINATED -> compareCallsClassObservations( + symbolic = candidate.observation, + original = original.observation, + instrumented = instrumented.observation, + ) + CallsCoverageCandidateOrigin.SUSPENDED -> CallsClassOracleResult( + accepted = false, + diagnostic = "Not applicable: suspended input has no symbolic completion or observation", + ) +} + +private inline fun decodeClassArtifact(path: Path): T = + CallsExperimentJson.json.decodeFromString(Files.readString(path)) + +private const val CLASS_REPLAY_TIMEOUT_MILLIS = 20_000L + +private const val CLASS_PREFLIGHT_ARGUMENT_COUNT = 3 +private const val CLASS_CELL_ARGUMENT_COUNT = 6 diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassHarness.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassHarness.kt new file mode 100644 index 0000000000..7cde0af0c1 --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassHarness.kt @@ -0,0 +1,415 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import org.usvm.ts.pbt.model.ArrayDomain +import org.usvm.ts.pbt.model.BooleanDomain +import org.usvm.ts.pbt.model.ConstantDomain +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.NumberDomain +import org.usvm.ts.pbt.model.ObjectDomain +import org.usvm.ts.pbt.model.OptionalDomain +import org.usvm.ts.pbt.model.PropertyDomain +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.model.StringDomain +import org.usvm.ts.pbt.model.TupleDomain +import org.usvm.ts.pbt.model.TypeScriptEntryPoint +import org.usvm.ts.pbt.model.encodeToUtf8SafeString +import java.nio.file.Files +import java.nio.file.Path + +@Serializable +internal enum class CallsClassMemberKind { METHOD, GETTER, SETTER, FUNCTION_FIELD } + +@Serializable +internal data class CallsClassMember( + val id: String, + val name: String, + val kind: CallsClassMemberKind, + val inputs: List, + val resultDomain: PropertyDomain, + val isStatic: Boolean = false, + val resultClassName: String? = null, +) + +/** A same-module exported class whose complete state consists only of these own data fields. */ +@Serializable +internal data class CallsClassDataClass( + val id: String, + val exportName: String, + val fields: Map, +) + +/** A complete source-inventoried callable surface and receiver-state shape, frozen before comparing policies. */ +@Serializable +internal data class CallsClassSurface( + val sourceFile: String, + val exportName: String, + val constructorInputs: List, + val members: List, + val stateFields: Map, + val dataClasses: List = emptyList(), +) { + init { + CallsSymbolicCoverageClass(sourceFile = sourceFile, className = exportName) + require(exportName.matches(CLASS_IDENTIFIER)) { "Class export must be an identifier" } + require(members.map(CallsClassMember::id).distinct().size == members.size) { "Member IDs must be unique" } + require(members.all { it.id.isNotBlank() && it.name.matches(CLASS_IDENTIFIER) }) { + "Every callable member needs a stable ID and an identifier name" + } + require(dataClasses.map(CallsClassDataClass::id).distinct().size == dataClasses.size) { + "Data-class IDs must be unique" + } + require(dataClasses.map(CallsClassDataClass::exportName).distinct().size == dataClasses.size) { + "Data-class exports must be unique" + } + for (dataClass in dataClasses) { + require(dataClass.id.matches(CLASS_IDENTIFIER) && dataClass.id !in setOf("plain", "target")) { + "Data-class IDs must be non-reserved identifiers" + } + require(dataClass.exportName.matches(CLASS_IDENTIFIER) && dataClass.exportName != exportName) { + "Data classes must name distinct same-module exports" + } + require(dataClass.fields.keys.all { it.matches(CLASS_IDENTIFIER) }) { + "Data-class fields must be identifiers" + } + require(dataClass.fields.values.all { it.isSupportedCallsSymbolicDomain() }) { + "Unsupported data-class observation domain" + } + } + val domains = stateFields.values + constructorInputs.map(PropertyInput::domain) + + members.flatMap { member -> member.inputs.map(PropertyInput::domain) + member.resultDomain } + + dataClasses.flatMap { it.fields.values } + if (dataClasses.isNotEmpty()) { + require(domains.none { it.hasClassIdentityField() } && CLASS_IDENTITY_FIELD !in stateFields) { + "Observation fields collide with the reserved class-identity field" + } + require(dataClasses.none { CLASS_IDENTITY_FIELD in it.fields }) { + "Data-class fields collide with the reserved class-identity field" + } + } + require(stateFields.keys.all { it.matches(CLASS_IDENTIFIER) }) { "State fields must be identifiers" } + for (member in members) { + require(member.resultClassName == null || member.resultClassName == exportName) { + "Only the selected concrete class can be an explicitly declared class result" + } + when (member.kind) { + CallsClassMemberKind.GETTER -> require(member.inputs.isEmpty()) { "Getters have no inputs" } + CallsClassMemberKind.SETTER -> require(member.inputs.size == 1) { "Setters have one input" } + else -> Unit + } + } + require(constructorInputs.map(PropertyInput::name).distinct().size == constructorInputs.size) + require(callsSymbolicInputPreflight(constructorInputs) == null) { "Unsupported constructor input domain" } + require(members.all { callsSymbolicInputPreflight(it.inputs) == null }) { "Unsupported callable input domain" } + require( + stateFields.values.all { it.isSupportedCallsSymbolicDomain() } + ) { "Unsupported receiver observation domain" } + require( + members.all { it.resultDomain.isSupportedCallsSymbolicDomain() } + ) { "Unsupported returned-value observation domain" } + } +} + +@Serializable +internal data class CallsClassScenario(val id: String, val memberIds: List) + +internal data class CallsClassPreparedHarness( + val surface: CallsClassSurface, + val scenarios: List, + val sourceRoot: Path, + val project: CallsProjectCase, + val functions: List, + val observations: List>, +) + +/** Each scenario constructs a fresh receiver; every invocation inside that scenario uses the same receiver. */ +internal fun prepareCallsClassHarness( + sourceRoot: Path, + project: CallsProjectCase, + surface: CallsClassSurface, + maxSequenceLength: Int = 2, +): CallsClassPreparedHarness { + require(maxSequenceLength in 1..2) { "Class scenarios support a frozen sequence bound of one or two" } + val root = sourceRoot.toRealPath() + require(Files.isRegularFile(root.resolve(surface.sourceFile))) { "Class source is missing" } + val sequences = mutableListOf(emptyList()) + sequences += surface.members.map { member -> listOf(member.id) } + if (maxSequenceLength == 2) { + sequences += surface.members.flatMap { first -> surface.members.map { second -> listOf(first.id, second.id) } } + } + val scenarios = sequences.mapIndexed { index, ids -> CallsClassScenario(id = "scenario-$index", memberIds = ids) } + val fileName = ".usvm-calls-class-harness.ts" + require(!Files.exists(root.resolve(fileName))) { "A generated class harness already exists in the overlay" } + val source = StringBuilder() + val moduleName = CallsExperimentJson.json.encodeToUtf8SafeString("./${surface.sourceFile}") + val imports = listOf("${surface.exportName} as CallsTargetClass") + + surface.dataClasses.mapIndexed { index, dataClass -> "${dataClass.exportName} as CallsDataClass$index" } + source.appendLine("import { ${imports.joinToString()} } from $moduleName;") + if (surface.dataClasses.isNotEmpty()) { + source.appendLine("// __usvm_class_registry ${CallsExperimentJson.json.encodeToString(surface.dataClasses)}") + } + val functions = mutableListOf() + val observations = mutableListOf>() + for ((index, scenario) in scenarios.withIndex()) { + val generated = ClassScenarioSource(surface, scenario, index).generate() + source.appendLine(generated.source) + functions += CallsFunctionCase( + functionId = scenario.id, + sourceFile = fileName, + entryPoint = TypeScriptEntryPoint(module = fileName, exportName = generated.exportName), + inputs = generated.inputs, + targets = emptyList(), + symbolicCoverageClass = CallsSymbolicCoverageClass(surface.sourceFile, surface.exportName), + ) + observations += generated.observationFields + } + Files.writeString(root.resolve(fileName), source) + + return CallsClassPreparedHarness( + surface = surface, + scenarios = scenarios, + sourceRoot = root, + project = project.copy(functions = functions), + functions = functions, + observations = observations, + ) +} + +@Serializable +internal data class CallsClassRawShapeCheck( + val expression: String, + val domain: PropertyDomain, + val skipCallableProperties: Boolean, + val allowTargetClass: Boolean = false, + val callablePropertyNames: Set = emptySet(), + val dataClasses: List = emptyList(), +) + +private data class GeneratedClassScenario( + val source: String, + val exportName: String, + val inputs: List, + val observationFields: Map, +) + +private class ClassScenarioSource( + private val surface: CallsClassSurface, + private val scenario: CallsClassScenario, + private val index: Int, +) { + private val inputs = mutableListOf() + private val fields = linkedMapOf() + private val declarations = StringBuilder() + private var snapshotIndex = 0 + + fun generate(): GeneratedClassScenario { + val constructorArgs = addInputs(surface.constructorInputs) + declarations.appendLine(" const receiver = new CallsTargetClass($constructorArgs);") + observeState(step = 0) + val membersById = surface.members.associateBy(CallsClassMember::id) + scenario.memberIds.forEachIndexed { callIndex, id -> + val member = membersById.getValue(id) + val args = addInputs(member.inputs) + val owner = if (member.isStatic) "CallsTargetClass" else "receiver" + val access = "$owner.${member.name}" + val expression = when (member.kind) { + CallsClassMemberKind.GETTER -> { access } + CallsClassMemberKind.SETTER -> { + declarations.appendLine(" $access = $args;") + "undefined" + } + CallsClassMemberKind.METHOD, CallsClassMemberKind.FUNCTION_FIELD -> { "$access($args)" } + } + declarations.appendLine(" const value_result_$callIndex = $expression;") + observe( + name = "result_$callIndex", + expression = "value_result_$callIndex", + domain = member.resultDomain, + allowTargetClass = member.resultClassName == surface.exportName, + ) + observeState(step = callIndex + 1) + } + val observationType = fields.entries.joinToString(separator = "; ") { (name, domain) -> + "$name: ${domain.classTypeScriptType()}" + } + val parameters = inputs.joinToString { input -> "${input.name}: ${input.domain.classTypeScriptType()}" } + val exportName = "callsClassScenario$index" + val source = """ + interface CallsObservation$index { $observationType } + export function $exportName($parameters): CallsObservation$index { + $declarations + return { ${fields.keys.joinToString()} }; + } + """.trimIndent() + + return GeneratedClassScenario(source, exportName, inputs.toList(), fields.toMap()) + } + + private fun addInputs(declared: List): String = declared.joinToString { input -> + val name = "input_${inputs.size}" + inputs += input.copy(name = name) + name + } + + private fun observeState(step: Int) { + rawShapeCheck( + expression = "receiver", + domain = ObjectDomain(surface.stateFields), + skipCallableProperties = true, + allowTargetClass = true, + ) + if (surface.dataClasses.isNotEmpty()) { + observe( + name = "state_${step}_$CLASS_IDENTITY_FIELD", + expression = classIdentity("receiver"), + domain = classIdentityDomain(), + ) + } + surface.stateFields.forEach { (name, domain) -> + val type = domain.classTypeScriptType() + observe( + name = "state_${step}_$name", + expression = "(receiver as unknown as { $name: $type }).$name", + domain = domain, + ) + } + } + + private fun observe( + name: String, + expression: String, + domain: PropertyDomain, + allowTargetClass: Boolean = false, + ) { + rawShapeCheck( + expression = expression, + domain = domain, + skipCallableProperties = true, + allowTargetClass = allowTargetClass, + ) + val snapshot = snapshot(expression, domain) + declarations.appendLine(" const $name = $snapshot;") + fields[name] = observationDomain(domain) + } + + private fun rawShapeCheck( + expression: String, + domain: PropertyDomain, + skipCallableProperties: Boolean, + allowTargetClass: Boolean = false, + ) { + val callablePropertyNames = surface.members.filter { member -> + member.kind == CallsClassMemberKind.FUNCTION_FIELD && !member.isStatic + }.mapTo(linkedSetOf()) { it.name } + val check = CallsClassRawShapeCheck( + expression = expression, + domain = domain, + skipCallableProperties = skipCallableProperties, + allowTargetClass = allowTargetClass, + callablePropertyNames = callablePropertyNames, + dataClasses = surface.dataClasses, + ) + val encoded = CallsExperimentJson.json.encodeToString(check) + declarations.appendLine(" // __usvm_class_observe $encoded") + } + + private fun classIdentityDomain(): StringDomain = StringDomain( + maxLength = maxOf("target".length, "plain".length, surface.dataClasses.maxOf { it.id.length }), + ) + + private fun classIdentity(expression: String): String { + val registered = surface.dataClasses.mapIndexed { index, dataClass -> + val id = CallsExperimentJson.json.encodeToUtf8SafeString(dataClass.id) + "($expression) instanceof CallsDataClass$index ? $id : " + }.joinToString(separator = "") + return "(($expression) instanceof CallsTargetClass ? \"target\" : ${registered}\"plain\")" + } + + private fun observationDomain(domain: PropertyDomain): PropertyDomain { + if (surface.dataClasses.isEmpty()) return domain + + return when (domain) { + is ObjectDomain -> ObjectDomain( + fields = domain.fields.mapValues { (_, value) -> observationDomain(value) } + + (CLASS_IDENTITY_FIELD to classIdentityDomain()), + ) + is OptionalDomain -> domain.copy(value = observationDomain(domain.value)) + is ArrayDomain -> domain.copy(element = observationDomain(domain.element)) + else -> domain + } + } + + private fun snapshot(expression: String, domain: PropertyDomain): String = when (domain) { + is ObjectDomain -> { + val entries = domain.fields.map { (name, nested) -> + require(name.matches(CLASS_IDENTIFIER)) { "Observed object fields must be identifiers" } + "$name: ${snapshot("($expression).$name", nested)}" + }.toMutableList() + if (surface.dataClasses.isNotEmpty()) { + entries += "$CLASS_IDENTITY_FIELD: ${classIdentity(expression)}" + } + entries.joinToString(prefix = "{ ", postfix = " }") + } + is OptionalDomain -> { + val nil = if (domain.nil == JsConcreteValue.Null) "null" else "undefined" + val temporary = "snapshot_${snapshotIndex++}" + val original = "snapshot_${snapshotIndex++}" + declarations.appendLine(" const $original = $expression;") + declarations.appendLine(" let $temporary: ${observationDomain(domain).classTypeScriptType()};") + declarations.appendLine(" if ($original === $nil) {") + declarations.appendLine(" $temporary = $nil;") + declarations.appendLine(" } else {") + val nested = snapshot(original, domain.value) + declarations.appendLine(" $temporary = $nested;") + declarations.appendLine(" }") + temporary + } + is ArrayDomain -> { + val temporary = "snapshot_${snapshotIndex++}" + val variable = "snapshot_index_${snapshotIndex++}" + declarations.appendLine(" const $temporary: ${observationDomain(domain).classTypeScriptType()} = [];") + declarations.appendLine(" for (let $variable = 0; $variable < ($expression).length; $variable++) {") + val element = snapshot("($expression)[$variable]", domain.element) + declarations.appendLine(" $temporary[$variable] = $element;") + declarations.appendLine(" }") + temporary + } + is TupleDomain -> { error("Tuple class observations are unsupported") } + else -> { expression } + } +} + +internal fun PropertyDomain.classTypeScriptType(): String = when (this) { + BooleanDomain -> "boolean" + is IntegerDomain, is NumberDomain -> "number" + is StringDomain -> "string" + is ConstantDomain -> when (value) { + JsConcreteValue.Null -> "null" + JsConcreteValue.Undefined -> "undefined" + is JsConcreteValue.Boolean -> "boolean" + is JsConcreteValue.Number -> "number" + is JsConcreteValue.String -> "string" + else -> error("Structured constants are unsupported class domains") + } + is OptionalDomain -> "(${value.classTypeScriptType()} | ${ConstantDomain(nil).classTypeScriptType()})" + is ArrayDomain -> "(${element.classTypeScriptType()})[]" + is TupleDomain -> elements.joinToString(prefix = "[", postfix = "]") { it.classTypeScriptType() } + is ObjectDomain -> fields.entries.joinToString(prefix = "{ ", postfix = " }") { (name, domain) -> + val key = CallsExperimentJson.json.encodeToUtf8SafeString(name) + "$key: ${domain.classTypeScriptType()}" + } +} + +private fun PropertyDomain.hasClassIdentityField(): Boolean = when (this) { + is ObjectDomain -> CLASS_IDENTITY_FIELD in fields || fields.values.any { it.hasClassIdentityField() } + is OptionalDomain -> value.hasClassIdentityField() + is ArrayDomain -> element.hasClassIdentityField() + is TupleDomain -> elements.any { it.hasClassIdentityField() } + else -> false +} + +internal const val CLASS_IDENTITY_FIELD = "__usvmCallsClassIdentity" + +private val CLASS_IDENTIFIER = Regex("[A-Za-z_$][A-Za-z0-9_$]*") diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassOracle.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassOracle.kt new file mode 100644 index 0000000000..d2aa0e5d23 --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassOracle.kt @@ -0,0 +1,683 @@ +package org.usvm.ts.calls + +import io.ksmt.utils.asExpr +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import org.jacodb.ets.model.EtsArrayType +import org.jacodb.ets.model.EtsMethod +import org.jacodb.ets.model.EtsType +import org.jacodb.ets.model.EtsUnionType +import org.usvm.UExpr +import org.usvm.USort +import org.usvm.isTrue +import org.usvm.machine.expr.TsUnresolvedSort +import org.usvm.machine.expr.extractDouble +import org.usvm.machine.expr.extractInt +import org.usvm.machine.expr.toConcreteBoolValue +import org.usvm.machine.state.TsMethodResult +import org.usvm.machine.state.TsState +import org.usvm.model.UModelBase +import org.usvm.ts.pbt.backend.PropertyRunConfiguration +import org.usvm.ts.pbt.backend.PropertyRunStatus +import org.usvm.ts.pbt.fastcheck.FastCheckBackend +import org.usvm.ts.pbt.fastcheck.FastCheckRuntime +import org.usvm.ts.pbt.model.ArrayDomain +import org.usvm.ts.pbt.model.BooleanDomain +import org.usvm.ts.pbt.model.ConstantDomain +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.NumberDomain +import org.usvm.ts.pbt.model.ObjectDomain +import org.usvm.ts.pbt.model.OptionalDomain +import org.usvm.ts.pbt.model.PropertyDefinition +import org.usvm.ts.pbt.model.PropertyDomain +import org.usvm.ts.pbt.model.PropertyId +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.model.StringDomain +import org.usvm.ts.pbt.model.TupleDomain +import org.usvm.ts.pbt.model.TypeScriptEntryPoint +import org.usvm.ts.pbt.model.contains +import org.usvm.ts.pbt.model.encodeToUtf8SafeString +import org.usvm.util.EtsHierarchy +import org.usvm.util.componentType +import org.usvm.util.declaredInstanceProperty +import org.usvm.util.mkArrayIndexLValue +import org.usvm.util.mkArrayLengthLValue +import org.usvm.util.mkFieldLValue +import org.usvm.util.resolveStringValue +import java.nio.file.Files +import java.nio.file.Path +import java.util.UUID +import kotlin.time.TimeSource + +/** Unrepresentable observations remain evidence, never a successful equivalence check. */ +@Serializable +internal data class CallsClassObservation( + val completion: CallsCoverageCompletion, + val values: Map = emptyMap(), + val representable: Boolean = true, + val diagnostic: String? = null, +) + +@Serializable +internal data class CallsClassReplayObservation( + val inputs: List, + val observation: CallsClassObservation, + val coveredStatementKeys: Set = emptySet(), + val replayElapsedMillis: Long = 0, +) + +@Serializable +internal data class CallsClassOracleResult(val accepted: Boolean, val diagnostic: String? = null) + +internal fun compareCallsClassObservations( + symbolic: CallsClassObservation?, + original: CallsClassObservation, + instrumented: CallsClassObservation, +): CallsClassOracleResult { + if (symbolic == null) return CallsClassOracleResult(accepted = false, diagnostic = "Symbolic observation missing") + val observations = listOf(symbolic, original, instrumented) + val unrepresentable = observations.firstOrNull { !it.representable } + if (unrepresentable != null) { + return CallsClassOracleResult( + accepted = false, + diagnostic = unrepresentable.diagnostic ?: "Unrepresentable observation" + ) + } + if (original != instrumented) { + return CallsClassOracleResult( + accepted = false, + diagnostic = "Instrumentation changed returned values or receiver state" + ) + } + if (symbolic != original) { + return CallsClassOracleResult( + accepted = false, + diagnostic = "Symbolic and original returned values or receiver state disagree" + ) + } + + return CallsClassOracleResult(accepted = true) +} + +/** Uses the same first model as CallsSymbolicInputs.resolve; no solver query changes candidate selection. */ +internal fun extractCallsClassObservation( + state: TsState, + method: EtsMethod, + fields: Map, +): CallsClassObservation { + val methodResult = state.methodResult + if (methodResult is TsMethodResult.TsException) { + return CallsClassObservation( + completion = CallsCoverageCompletion.THREW, + representable = false, + diagnostic = "Thrown class scenarios have no complete return/state snapshot; " + + "exception type=${methodResult.type}, value=${methodResult.value}", + ) + } + + val completion = CallsCoverageCompletion.RETURNED + return runCatching { + val result = methodResult as? TsMethodResult.Success.RegularCall + ?: error("Class scenario did not produce a regular returned snapshot") + require(result.method == method) { "Class scenario did not complete its wrapper" } + val value = ClassObservationValueResolver(state, state.models.first()).resolve( + domain = ObjectDomain(fields), + type = method.returnType, + expression = result.value, + ) as JsConcreteValue.Object + CallsClassObservation(completion = completion, values = value.fields) + }.getOrElse { error -> + CallsClassObservation( + completion = completion, + representable = false, + diagnostic = error.message ?: error::class.java.name, + ) + } +} + +private class ClassObservationValueResolver( + private val state: TsState, + private val model: UModelBase, +) { + private val hierarchy = EtsHierarchy(state.ctx.scene) + + fun resolve(domain: PropertyDomain, type: EtsType, expression: UExpr): JsConcreteValue = with( + state.ctx + ) { + val value = unbox(expression) + when (domain) { + BooleanDomain -> JsConcreteValue.Boolean(model.eval(value.asExpr(boolSort)).toConcreteBoolValue()) + is NumberDomain, is IntegerDomain -> JsConcreteValue.number( + model.eval(value.asExpr(fp64Sort)).extractDouble() + ) + is StringDomain -> JsConcreteValue.String( + state.resolveStringValue( + string = value.asExpr(addressSort), + model = model, + maxLength = domain.maxLength + ), + ) + is ConstantDomain -> resolveConstant(domain, value) + is OptionalDomain -> resolveOptional(domain, type, value) + is ObjectDomain -> resolveObject(domain, type, value) + is ArrayDomain -> resolveArray(domain, type, value) + is TupleDomain -> error("Tuple class observations are unsupported") + } + } + + private fun unbox(value: UExpr): UExpr = with(state.ctx) { + if (value.sort != addressSort) return@with value + + val reference = model.eval(value.asExpr(addressSort)) + if (!reference.isFakeObject()) return@with value + + val fakeType = reference.getFakeType(state.memory) + when { + model.eval(fakeType.boolTypeExpr).isTrue -> reference.extractBool(state.memory) + model.eval(fakeType.fpTypeExpr).isTrue -> reference.extractFp(state.memory) + model.eval(fakeType.refTypeExpr).isTrue -> reference.extractRef(state.memory) + else -> error("Observed fake value has no selected runtime type") + } + } + + private fun resolveOptional( + domain: OptionalDomain, + type: EtsType, + value: UExpr, + ): JsConcreteValue = with(state.ctx) { + val nil = if (domain.nil == JsConcreteValue.Null) mkTsNullValue() else mkUndefinedValue() + val isNil = value.sort == addressSort && model.eval(mkHeapRefEq(value.asExpr(addressSort), nil)).isTrue + if (isNil) return@with domain.nil + + val nestedType = (type as? EtsUnionType)?.types?.firstOrNull { nested -> + nested != org.jacodb.ets.model.EtsNullType && nested != org.jacodb.ets.model.EtsUndefinedType + } ?: type + resolve(domain.value, nestedType, value) + } + + private fun resolveObject( + domain: ObjectDomain, + type: EtsType, + value: UExpr, + ): JsConcreteValue = with(state.ctx) { + val reference = value.asExpr(addressSort) + val values = domain.fields.mapValues { (name, nested) -> + val fieldType = declaredInstanceProperty(type = type, name = name, hierarchy = hierarchy)?.type + ?: error("Observed field $name has no declared type in $type") + val sort = typeToSort(fieldType) + val storageSort = if (sort is TsUnresolvedSort) addressSort else sort + val fieldValue = state.memory.read(mkFieldLValue(storageSort, reference, name)) + resolve(nested, fieldType, fieldValue) + } + JsConcreteValue.Object(values) + } + + private fun resolveConstant(domain: ConstantDomain, value: UExpr): JsConcreteValue = with(state.ctx) { + val resolved = when (val expected = domain.value) { + JsConcreteValue.Undefined, JsConcreteValue.Null -> { + val nil = if (expected == JsConcreteValue.Null) mkTsNullValue() else mkUndefinedValue() + require(value.sort == addressSort && model.eval(mkHeapRefEq(value.asExpr(addressSort), nil)).isTrue) { + "Observed constant differs from its declared value" + } + expected + } + is JsConcreteValue.Boolean -> { + JsConcreteValue.Boolean(model.eval(value.asExpr(boolSort)).toConcreteBoolValue()) + } + is JsConcreteValue.Number -> { + JsConcreteValue.number(model.eval(value.asExpr(fp64Sort)).extractDouble()) + } + is JsConcreteValue.String -> { + JsConcreteValue.String( + state.resolveStringValue( + string = value.asExpr(addressSort), + model = model, + maxLength = expected.value.length, + ), + ) + } + else -> { error("Structured constant class observations are unsupported") } + } + require(resolved == domain.value) { "Observed constant differs from its declared value" } + resolved + } + + private fun resolveArray(domain: ArrayDomain, type: EtsType, value: UExpr): JsConcreteValue = with( + state.ctx + ) { + val arrayType = type as? EtsArrayType ?: error("Observed array has no declared array type: $type") + val reference = value.asExpr(addressSort) + val length = model.eval(state.memory.read(mkArrayLengthLValue(reference, arrayType))).extractInt() + require(length in 0..domain.maxLength) { "Observed array exceeds the frozen observation capacity" } + val elementSort = typeToSort(arrayType.componentType) + val storageSort = if (elementSort is TsUnresolvedSort) addressSort else elementSort + val elements = (0 until length).map { index -> + val lValue = mkArrayIndexLValue(storageSort, reference, mkBv(index), arrayType) + resolve(domain.element, arrayType.componentType, state.memory.read(lValue)) + } + JsConcreteValue.Array(elements) + } +} + +/** Replays the exact same scenario in a fresh process against original or counter-instrumented source. */ +internal class OriginalTypeScriptClassReplayer( + sourceRoot: Path, + private val function: CallsFunctionCase, + private val observationFields: Map, + sourceOverride: String? = null, + overrideSourceFile: String? = null, + private val counterGlobalName: String? = null, +) : AutoCloseable { + private val root = sourceRoot.toRealPath() + private val workspace = Files.createTempDirectory("usvm-calls-class-replay-") + private val overlay = workspace.resolve("overlay") + private val observationPath = workspace.resolve("observation.json") + private val wrapperName = ".usvm-class-replay-${UUID.randomUUID()}.mts" + private val backend: FastCheckBackend + + init { + val setup = runCatching { + require( + (sourceOverride == null) == (overrideSourceFile == null) + ) { "Source override needs a relative target file" } + createClassReplayOverlay(root, overlay, overrideSourceFile, sourceOverride) + val replayEntry = overlay.resolve(function.sourceFile) + Files.deleteIfExists(replayEntry) + val harnessSource = Files.readString(root.resolve(function.sourceFile)) + Files.writeString(replayEntry, nativeCheckedClassHarness(harnessSource)) + Files.writeString( + overlay.resolve(wrapperName), + classReplayWrapper( + sourceFile = function.sourceFile, + exportName = function.entryPoint.exportName, + observationPath = observationPath, + dropSentinel = function.inputs.isEmpty(), + counterGlobalName = counterGlobalName, + fields = observationFields.keys, + ) + ) + FastCheckBackend(sourceRoots = listOf(overlay, root)) + } + backend = setup.getOrElse { error -> + close() + throw error + } + } + + fun replay(inputs: List, timeoutMillis: Long): CallsClassReplayObservation { + require(inputs.size == function.inputs.size) + val emptyInputs = function.inputs.isEmpty() + val observation = execute( + seed = 0, + timeoutMillis = timeoutMillis, + examples = listOf(if (emptyInputs) listOf(JsConcreteValue.Boolean(true)) else inputs), + ) + require(observation.inputs == inputs) { "Class replay input transport changed" } + return observation + } + + /** Native-only sampling uses the declared domains; it does not consult any symbolic profile or candidate. */ + fun sample(seed: Long, timeoutMillis: Long): CallsClassReplayObservation = execute( + seed = seed, + timeoutMillis = timeoutMillis, + examples = emptyList(), + ) + + private fun execute( + seed: Long, + timeoutMillis: Long, + examples: List>, + ): CallsClassReplayObservation { + require(timeoutMillis > 0) + require(seed in Int.MIN_VALUE.toLong()..Int.MAX_VALUE.toLong()) { + "Native PBT seed must fit a signed 32-bit integer" + } + val nativeSeed = seed.toInt() + Files.deleteIfExists(observationPath) + val started = TimeSource.Monotonic.markNow() + val emptyInputs = function.inputs.isEmpty() + val propertyInputs = if (emptyInputs) { + listOf(PropertyInput(name = "sentinel", domain = BooleanDomain)) + } else { + function.inputs + } + val result = backend.run( + property = PropertyDefinition( + id = PropertyId("calls.class.exact-replay"), + inputs = propertyInputs, + predicate = TypeScriptEntryPoint(module = wrapperName, exportName = "replayClassScenario"), + ), + configuration = PropertyRunConfiguration( + seed = nativeSeed, + numRuns = 1, + timeoutMillis = timeoutMillis, + examples = examples, + ), + ) + require(result.status == PropertyRunStatus.SUCCESS && result.numRuns == 1) { + "Class replay did not complete exactly once: $result" + } + val observation = CallsExperimentJson.json.decodeFromString( + Files.readString(observationPath) + ) + val inputsInsideDomains = observation.inputs.size == function.inputs.size && + observation.inputs.zip(function.inputs).all { (value, input) -> value in input.domain } + require(inputsInsideDomains) { "Native sample is outside the frozen constructor input domains" } + val expectedFields = observationFields.keys + val hasExactFields = !observation.observation.representable || + observation.observation.values.keys == expectedFields + require(hasExactFields) { "Class replay omitted or added observation fields" } + return observation.copy(replayElapsedMillis = started.elapsedNow().inWholeMilliseconds) + } + + override fun close() { + Files.walk(workspace).use { paths -> paths.sorted(Comparator.reverseOrder()).forEach(Files::deleteIfExists) } + } +} + +private fun createClassReplayOverlay(root: Path, overlay: Path, relative: String?, source: String?) { + Files.createDirectories(overlay) + val target = relative?.let(Path::of) + val isSafeTarget = target == null || !target.isAbsolute && target.normalize() == target && !target.startsWith("..") + require(isSafeTarget) { "Replay source override must stay inside source root" } + fun mirror(directory: Path, destination: Path, remaining: Path?) { + Files.newDirectoryStream(directory).use { entries -> + for (entry in entries) { + val targetSuffix = remaining?.takeIf { entry.fileName == it.getName(0) } + val output = destination.resolve(entry.fileName) + if (targetSuffix == null) { + Files.createSymbolicLink(output, entry.toAbsolutePath()) + } else if (targetSuffix.nameCount == 1) { + Files.writeString(output, requireNotNull(source)) + } else { + Files.createDirectories(output) + mirror(entry, output, targetSuffix.subpath(1, targetSuffix.nameCount)) + } + } + } + } + mirror(root, overlay, target) +} + +private fun classReplayWrapper( + sourceFile: String, + exportName: String, + observationPath: Path, + dropSentinel: Boolean, + counterGlobalName: String?, + fields: Set, +): String { + val json = CallsExperimentJson.json + val protocolModule = json.encodeToUtf8SafeString(FastCheckRuntime.protocolJsonModule().toUri().toString()) + val sourceModule = json.encodeToUtf8SafeString("./$sourceFile") + val exportKey = json.encodeToUtf8SafeString(exportName) + val outputPath = json.encodeToUtf8SafeString(observationPath.toString()) + val expectedFields = json.encodeToUtf8SafeString(fields.sorted()) + val actualArgs = if (dropSentinel) "args.slice(1)" else "args" + val counterSetup = counterGlobalName?.let { name -> + "(globalThis as any)[${json.encodeToUtf8SafeString( + name + )}] = new Set();" + }.orEmpty() + val counterRead = counterGlobalName?.let { name -> + "(globalThis as any)[${json.encodeToUtf8SafeString(name)}]" + } ?: "[]" + return """ + import { writeFileSync } from 'node:fs'; + import { stringifyProtocolJson } from $protocolModule; + import * as scenarioModule from $sourceModule; + + function encode(value: unknown, seen = new Set()): unknown { + if (value === undefined) return { kind: 'undefined' }; + if (value === null) return { kind: 'null' }; + if (typeof value === 'boolean') return { kind: 'boolean', value }; + if (typeof value === 'string') return { kind: 'string', value }; + if (typeof value === 'number') { + if (Number.isNaN(value)) return { kind: 'number', value: 'nan' }; + if (value === Infinity) return { kind: 'number', value: 'positive-infinity' }; + if (value === -Infinity) return { kind: 'number', value: 'negative-infinity' }; + const view = new DataView(new ArrayBuffer(8)); + view.setFloat64(0, value, false); + const bits = view.getBigUint64(0, false).toString(16).padStart(16, '0'); + return { kind: 'number', value: 'finite', bits }; + } + if (typeof value !== 'object') throw new Error('Unrepresentable observed value'); + if (seen.has(value)) throw new Error('Cyclic observed value'); + seen.add(value); + let encoded: unknown; + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + if (!Object.hasOwn(value, i)) throw new Error('Sparse observed array'); + } + encoded = { kind: 'array', elements: value.map(element => encode(element, seen)) }; + } else { + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) throw new Error('Non-plain observed object'); + const fields: Record = {}; + for (const key of Reflect.ownKeys(value)) { + if (typeof key !== 'string' || ['__proto__', 'constructor', 'prototype'].includes(key)) { + throw new Error('Unsupported observed property'); + } + const descriptor = Object.getOwnPropertyDescriptor(value, key)!; + if (!('value' in descriptor)) throw new Error('Accessor observed property'); + fields[key] = encode(descriptor.value, seen); + } + encoded = { kind: 'object', fields }; + } + seen.delete(value); + return encoded; + } + +${classReplayThrownValueDiagnosticSource().prependIndent(indent = " ")} + + export function replayClassScenario(...args: unknown[]): boolean { + const actualArgs = $actualArgs; + const inputs = actualArgs.map(value => encode(value)); + $counterSetup + let observation: any; + let returned: any; + let completion = 'RETURNED'; + try { + returned = (scenarioModule as any)[$exportKey](...actualArgs); + } catch (error) { + completion = 'THREW'; + const isShapeFailure = error instanceof Error && error.message.startsWith('USVM class observation shape:'); + const diagnostic = isShapeFailure ? error.message : + 'Thrown class scenarios have no complete return/state snapshot; ' + describeThrownValue(error); + observation = { completion, representable: false, diagnostic }; + } + if (completion === 'RETURNED') { + try { + const encoded = encode(returned) as any; + const exactFields = encoded.kind === 'object' && + JSON.stringify(Object.keys(encoded.fields).sort()) === JSON.stringify($expectedFields); + if (!exactFields) { + throw new Error('Class scenario observation fields differ from frozen surface'); + } + observation = { completion, values: encoded.fields }; + } catch (error) { + const diagnostic = error instanceof Error ? error.message : String(error); + observation = { completion, representable: false, diagnostic }; + } + } + const coveredStatementKeys = Array.from($counterRead); + writeFileSync($outputPath, stringifyProtocolJson({ inputs, observation, coveredStatementKeys }), 'utf8'); + return true; + } + """.trimIndent() + "\n" +} + +private fun classReplayThrownValueDiagnosticSource(): String = """ + function describeThrownValue(error: unknown): string { + if (error instanceof Error) { + return 'exception name=' + stringifyProtocolJson(error.name) + + ', message=' + stringifyProtocolJson(error.message); + } + const valueType = typeof error; + if (valueType === 'bigint' || valueType === 'symbol') { + return 'thrown value type=' + valueType + ', value=' + stringifyProtocolJson(String(error)); + } + if ((valueType === 'object' && error !== null) || valueType === 'function') { + return 'thrown value type=' + valueType; + } + return 'thrown value=' + stringifyProtocolJson(encode(error)); + } +""".trimIndent() + + +/** Check raw values before projection, so snapshots cannot silently hide an unlisted concrete property. */ +private fun nativeCheckedClassHarness(source: String): String { + val marker = "// __usvm_class_observe " + val registryMarker = "// __usvm_class_registry " + val checked = source.lines().joinToString(separator = "\n") { line -> + val offset = line.indexOf(marker) + val registryOffset = line.indexOf(registryMarker) + if (registryOffset >= 0) { + val raw = line.substring(registryOffset + registryMarker.length) + val registry = CallsExperimentJson.json.decodeFromString>(raw) + val entries = registry.mapIndexed { index, dataClass -> + val fields = CallsExperimentJson.json.encodeToString(ObjectDomain(dataClass.fields)) + val name = CallsExperimentJson.json.encodeToUtf8SafeString(dataClass.exportName) + "{ constructor: CallsDataClass$index, prototype: CallsDataClass$index.prototype, " + + "name: $name, length: CallsDataClass$index.length, domain: $fields }" + } + "const __usvmClassDataRegistry = [${entries.joinToString()}];" + } else if (offset < 0) { + line + } else { + val raw = line.substring(offset + marker.length) + val check = CallsExperimentJson.json.decodeFromString(raw) + val domain = CallsExperimentJson.json.encodeToString(check.domain) + val allowCallableFields = if (check.skipCallableProperties) { + "(${check.expression}) === receiver" + } else { + "false" + } + val allowedClass = if (check.allowTargetClass) "CallsTargetClass" else "undefined" + val callableNames = CallsExperimentJson.json.encodeToUtf8SafeString(check.callablePropertyNames.sorted()) + val registry = if (check.dataClasses.isEmpty()) "[]" else "__usvmClassDataRegistry" + " __usvmAssertObservedShape(${check.expression}, $domain, " + + "$allowCallableFields, $allowedClass, $callableNames, $registry);" + } + } + return "$NATIVE_CLASS_SHAPE_VALIDATOR\n$checked" +} + +private val NATIVE_CLASS_SHAPE_VALIDATOR = """ + function __usvmCanonicalDomain(domain: any): any { + if (domain === null || typeof domain !== 'object') return domain; + if (Array.isArray(domain)) return domain.map(__usvmCanonicalDomain); + return Object.fromEntries(Object.keys(domain).sort().map(key => [key, __usvmCanonicalDomain(domain[key])])); + } + function __usvmAssertObservedShape( + value: any, domain: any, skipCallableProperties = false, allowedClass?: any, + callablePropertyNames: string[] = [], dataClasses: any[] = [], + ): void { + const fail = (reason: string): never => { throw new Error('USVM class observation shape: ' + reason); }; + const exactOwnKeys = (value: any, names: string[]): boolean => { + const keys = Reflect.ownKeys(value); + return keys.every(key => typeof key === 'string') && + JSON.stringify(keys.sort()) === JSON.stringify([...names].sort()); + }; + for (const entry of dataClasses) { + const ctor = entry.constructor; + const prototype = entry.prototype; + const validConstructor = Object.getPrototypeOf(ctor) === Function.prototype && + ctor.prototype === prototype && ctor.name === entry.name && ctor.length === entry.length && + exactOwnKeys(ctor, ['length', 'name', 'prototype']); + const descriptor = Object.getOwnPropertyDescriptor(prototype, 'constructor'); + const validPrototype = Object.getPrototypeOf(prototype) === Object.prototype && + exactOwnKeys(prototype, ['constructor']) && descriptor && 'value' in descriptor && + descriptor.value === ctor && descriptor.writable && descriptor.configurable && !descriptor.enumerable; + const validConstructorProperties = ['length', 'name', 'prototype'].every(name => { + const property = Object.getOwnPropertyDescriptor(ctor, name)!; + return property && 'value' in property && !property.enumerable && !property.writable && + property.configurable === (name !== 'prototype'); + }); + if (!validConstructor || !validPrototype || !validConstructorProperties) { + fail('Registered data-class constructor or prototype changed'); + } + } + if (domain.kind === 'optional') { + const nil = domain.nil?.kind === 'null' ? null : undefined; + if (value !== nil) { + __usvmAssertObservedShape( + value, domain.value, skipCallableProperties, allowedClass, callablePropertyNames, dataClasses, + ); + } + } else if (domain.kind === 'object') { + if (value === null || typeof value !== 'object' || Array.isArray(value)) fail('Expected object'); + const prototype = Object.getPrototypeOf(value); + const knownClass = allowedClass !== undefined && value instanceof allowedClass && + prototype === allowedClass.prototype; + const dataClass = dataClasses.find(entry => + prototype === entry.prototype && value instanceof entry.constructor); + if (prototype !== Object.prototype && prototype !== null && !knownClass && !dataClass) { + fail('Unlisted object prototype'); + } + if (dataClass && JSON.stringify(__usvmCanonicalDomain(domain)) !== + JSON.stringify(__usvmCanonicalDomain(dataClass.domain))) { + fail('Registered data-class observation domain changed'); + } + if (dataClass && !exactOwnKeys(value, Object.keys(dataClass.domain.fields))) { + fail('Registered data-class own fields changed'); + } + if (dataClass) { + for (const [key, nested] of Object.entries(dataClass.domain.fields)) { + const descriptor = Object.getOwnPropertyDescriptor(value, key)!; + const ordinaryField = 'value' in descriptor && descriptor.enumerable && + descriptor.writable && descriptor.configurable; + if (!ordinaryField) fail('Registered data-class field is not an ordinary data property'); + __usvmAssertObservedShape(value[key], nested, false, undefined, [], dataClasses); + } + } + const allowedCallables = new Set(skipCallableProperties ? callablePropertyNames : []); + for (const name of allowedCallables) { + const descriptor = Object.getOwnPropertyDescriptor(value, name); + if (!descriptor || !('value' in descriptor) || typeof descriptor.value !== 'function') { + fail('Declared callable property changed kind or disappeared'); + } + } + const keys = Reflect.ownKeys(value).filter(key => { + const descriptor = Object.getOwnPropertyDescriptor(value, key)!; + if (!('value' in descriptor)) fail('Accessor in observed data'); + return !(typeof key === 'string' && allowedCallables.has(key) && typeof descriptor.value === 'function'); + }); + const exactKeys = !keys.some(key => typeof key !== 'string') && + JSON.stringify(keys.sort()) === JSON.stringify(Object.keys(domain.fields).sort()); + if (!exactKeys) { + fail('Unlisted or missing object property'); + } + for (const [key, nested] of Object.entries(domain.fields)) { + __usvmAssertObservedShape(value[key], nested, false, undefined, [], dataClasses); + } + } else if (domain.kind === 'array') { + const validArray = Array.isArray(value) && value.length <= (domain.maxLength ?? 10); + if (!validArray) fail('Array exceeds observation capacity'); + for (let i = 0; i < value.length; i++) { + if (!Object.hasOwn(value, i)) fail('Sparse array'); + __usvmAssertObservedShape(value[i], domain.element, false, undefined, [], dataClasses); + } + const allowed = new Set(['length', ...value.map((_: any, index: number) => String(index))]); + const unlistedProperty = Reflect.ownKeys(value).some(key => typeof key !== 'string' || !allowed.has(key)); + if (unlistedProperty) fail('Unlisted array property'); + } else if (domain.kind === 'string') { + const validString = typeof value === 'string' && value.length <= (domain.maxLength ?? 10); + if (!validString) fail('String exceeds observation capacity'); + } else if (domain.kind === 'number' || domain.kind === 'integer') { + if (typeof value !== 'number') fail('Expected number'); + } else if (domain.kind === 'boolean') { + if (typeof value !== 'boolean') fail('Expected boolean'); + } else if (domain.kind === 'constant') { + const expected = domain.value; + if (expected.kind === 'undefined' && value !== undefined) fail('Expected undefined'); + if (expected.kind === 'null' && value !== null) fail('Expected null'); + if (expected.kind === 'boolean' || expected.kind === 'string') { + if (value !== expected.value) fail('Constant changed'); + } + if (expected.kind === 'number' && typeof value !== 'number') fail('Expected constant number'); + } else fail('Unsupported observation domain'); + } +""".trimIndent() diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassSourceInstrumentation.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassSourceInstrumentation.kt new file mode 100644 index 0000000000..62d9dc2a51 --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassSourceInstrumentation.kt @@ -0,0 +1,129 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import org.usvm.ts.pbt.fastcheck.TypeScriptClassStatementDescriptor +import org.usvm.ts.pbt.fastcheck.TypeScriptClassSurfaceInventory +import org.usvm.ts.pbt.fastcheck.TypeScriptSourceInspector +import java.nio.file.Files +import java.nio.file.Path + +/** Instrument only a source overlay; the original source and frozen AST offsets remain untouched. */ +@Serializable +internal data class CallsClassSourceInstrumentation( + val universe: CallsClassSourceUniverse, + val instrumentedSource: String, + val instrumentedSha256: String, + val hitGlobalName: String, + val inventory: TypeScriptClassSurfaceInventory, +) { + init { + require(callsClassSourceSha256(instrumentedSource.toByteArray(Charsets.UTF_8)) == instrumentedSha256) { + "Class instrumentation source hash mismatch" + } + } + + fun validateSurface(surface: CallsClassSurface) { + require(surface.sourceFile == universe.sourceFile && surface.exportName == universe.className) { + "Class harness surface selects a different source class" + } + require(surface.constructorInputs.map { input -> input.name } == inventory.constructorParameterNames) { + "Class harness must represent every constructor parameter in source order" + } + val expectedMembers = inventory.publicMembers.map { member -> + Triple(member.name, member.kind, member.isStatic) + }.toSet() + val actualMembers = surface.members.map { member -> + Triple(member.name, member.kind.name, member.isStatic) + }.toSet() + require(actualMembers == expectedMembers && actualMembers.size == surface.members.size) { + "Class harness must represent the complete original public callable surface" + } + for (member in surface.members) { + val original = inventory.publicMembers.single { original -> + original.name == member.name && original.kind == member.kind.name && + original.isStatic == member.isStatic + } + require(member.inputs.map { input -> input.name } == original.parameterNames) { + "Class harness must represent every parameter of ${member.name} in source order" + } + } + val hasUnobservablePrivateState = inventory.instanceDataFields.any { field -> field.isEcmaPrivate } + require(!hasUnobservablePrivateState) { "ECMAScript private receiver state cannot be observed by this harness" } + val expectedFields = inventory.instanceDataFields.map { field -> field.name }.toSet() + require(surface.stateFields.keys == expectedFields) { + "Class harness must observe every original receiver data field: $expectedFields" + } + validateDataClasses(surface) + } + + private fun validateDataClasses(surface: CallsClassSurface) { + val duplicateExports = surface.dataClasses.map { dataClass -> dataClass.exportName }.distinct().size != + surface.dataClasses.size + require(!duplicateExports) { "Companion data-class exports must be unique" } + for (dataClass in surface.dataClasses) { + require(dataClass.exportName != universe.className) { "The target class has its own observation identity" } + val original = inventory.dataClasses.singleOrNull { original -> + original.exportName == dataClass.exportName + } + requireNotNull(original) { + "Unverified companion data class ${dataClass.exportName} in ${universe.sourceFile}" + } + require(dataClass.fields.keys == original.fields.toSet()) { + "Companion data class ${dataClass.exportName} must preserve every original own data field" + } + } + } + + companion object { + fun create(sourceRoot: Path, sourceFile: String, className: String): CallsClassSourceInstrumentation { + val root = sourceRoot.toRealPath() + val source = root.resolve(sourceFile).normalize().toRealPath() + require(source.startsWith(root)) { "Class source is outside the frozen source root" } + val sourceBytes = Files.readAllBytes(source) + val sourceSha256 = callsClassSourceSha256(sourceBytes) + val inspected = TypeScriptSourceInspector.instrumentClassCoverage(source = source, className = className) + val instrumentedSource = requireNotNull(inspected.source) { + inspected.reason ?: "Unsupported class statement instrumentation" + } + require(inspected.sourceSha256 == sourceSha256) { "Class source changed during AST inspection" } + + val universeIdentity = CallsExperimentJson.json.encodeToString( + CallsClassUniverseHashInput( + sourceFile = sourceFile, + className = className, + sourceSha256 = sourceSha256, + instrumenterVersion = inspected.instrumenterVersion, + statements = inspected.statements, + inventory = requireNotNull(inspected.inventory), + ), + ) + val universe = CallsClassSourceUniverse( + sourceFile = sourceFile, + className = className, + sourceSha256 = sourceSha256, + universeSha256 = callsClassSourceSha256(universeIdentity.toByteArray(Charsets.UTF_8)), + instrumenterVersion = inspected.instrumenterVersion, + statements = inspected.statements, + ) + + return CallsClassSourceInstrumentation( + universe = universe, + instrumentedSource = instrumentedSource, + instrumentedSha256 = requireNotNull(inspected.instrumentedSha256), + hitGlobalName = inspected.counterGlobalName, + inventory = requireNotNull(inspected.inventory), + ) + } + } +} + +@Serializable +private data class CallsClassUniverseHashInput( + val sourceFile: String, + val className: String, + val sourceSha256: String, + val instrumenterVersion: String, + val statements: List, + val inventory: TypeScriptClassSurfaceInventory, +) diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassSourceUniverse.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassSourceUniverse.kt new file mode 100644 index 0000000000..f920110c4a --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassSourceUniverse.kt @@ -0,0 +1,57 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.Serializable +import org.usvm.ts.pbt.fastcheck.TypeScriptClassStatementDescriptor +import java.nio.file.Files +import java.nio.file.Path +import java.security.MessageDigest + +/** Frozen original-source executable statements, independent of policy, seed and replay hits. */ +@Serializable +internal data class CallsClassSourceUniverse( + val sourceFile: String, + val className: String, + val sourceSha256: String, + val universeSha256: String, + val instrumenterVersion: String, + val statements: List, +) { + init { + require(sourceFile.isNotBlank() && className.isNotBlank()) + require(statements.isNotEmpty()) { "Class coverage universe has no executable statements" } + require(statements.map { statement -> statement.id }.toSet().size == statements.size) { + "Class statement IDs must be unique" + } + val hasExactOriginalRanges = statements.all { statement -> + statement.startOffset >= 0 && statement.endOffset > statement.startOffset && + statement.id.startsWith("$sourceSha256:") + } + require(hasExactOriginalRanges) { "Class statements must identify exact original-source AST ranges" } + } + + val statementIds: Set + get() = statements.mapTo(linkedSetOf()) { statement -> statement.id } + + fun verifyUnchanged(sourceRoot: Path) { + val root = sourceRoot.toRealPath() + val source = root.resolve(sourceFile).normalize().toRealPath() + require(source.startsWith(root)) { "Class source is outside the frozen source root" } + require(callsClassSourceSha256(Files.readAllBytes(source)) == sourceSha256) { + "Class source changed after freezing its statement universe" + } + } + + /** Counter IDs are native hits of these exact AST statements, never inferred from c8 line records. */ + fun checkedCoveredIds(nativeHits: Collection): Set { + val covered = nativeHits.toSet() + val unknownIds = covered - statementIds + require(unknownIds.isEmpty()) { "Native class coverage reported unknown statement IDs: $unknownIds" } + + return covered + } +} + +internal fun callsClassSourceSha256(bytes: ByteArray): String = MessageDigest.getInstance("SHA-256") + .digest(bytes).joinToString(separator = "") { byte -> "%02x".format(byte.toInt() and UNSIGNED_BYTE_MASK) } + +private const val UNSIGNED_BYTE_MASK = 0xff diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt index ff96c00951..80e427551e 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt @@ -124,8 +124,8 @@ internal fun runCoverageUniverse( val universe = probeFunction(source) source.verifyUnchanged() val record = CallsCoverageUniverseRecord(identity = identity, universe = universe) - atomicWrite(output.resolve("universe.json"), encode(record)) - System.out.appendLine(encode(record)) + writeCallsArtifactAtomically(output.resolve("universe.json"), encodeCallsArtifact(record)) + System.out.appendLine(encodeCallsArtifact(record)) } internal fun runCoverageCell( @@ -147,12 +147,12 @@ internal fun runCoverageCell( Files.createDirectories(output) val identityPath = output.resolve("identity.json") - val expectedIdentity = encode(identity) + val expectedIdentity = encodeCallsArtifact(identity) if (Files.exists(identityPath)) { require(Files.readString(identityPath).trim() == expectedIdentity) { "Cell identity changed" } } else { require(Files.list(output).use { entries -> entries.findAny().isEmpty }) { "Output directory is not empty" } - atomicWrite(identityPath, expectedIdentity) + writeCallsArtifactAtomically(identityPath, expectedIdentity) } require(!Files.exists(output.resolve("result.json"))) { "Cell already completed" } @@ -182,8 +182,8 @@ internal fun runCoverageCell( budgetMillis = identity.budgetMillis, ), ) - atomicWrite(output.resolve("result.json"), encode(result)) - System.out.appendLine(encode(result)) + writeCallsArtifactAtomically(output.resolve("result.json"), encodeCallsArtifact(result)) + System.out.appendLine(encodeCallsArtifact(result)) } private data class CoverageCellRequest( @@ -254,7 +254,7 @@ private fun parseCoverageCellRequest( budgetMillis = budgetMillis, solverLimitMillis = solverLimitMillis, candidateCap = candidateCap, - coverageUniverseSha256 = coverageUniverseSha256(universeBytes), + coverageUniverseSha256 = callsArtifactSha256(universeBytes), ) return CoverageCellRequest(source = source, identity = identity, output = output, universe = universe) @@ -286,13 +286,13 @@ private fun loadOrRunCoverageSearch( solverQueryLimit = identity.solverLimitMillis.milliseconds, solverType = SolverType.valueOf(identity.solver), candidateCap = identity.candidateCap, - onCandidate = { candidate -> appendDurably(candidatePath, encode(candidate)) }, + onCandidate = { candidate -> appendCallsJournalDurably(candidatePath, encodeCallsArtifact(candidate)) }, ) val result = searchFunction(request) source.verifyUnchanged() - val persistedCandidates = readLines(candidatePath) + val persistedCandidates = readCallsJournal(candidatePath) require(result.candidates == persistedCandidates) { "Persisted candidate journal differs from search result" } - atomicWrite(searchPath, encode(result)) + writeCallsArtifactAtomically(searchPath, encodeCallsArtifact(result)) result } } @@ -303,7 +303,7 @@ private fun replayCoverageCandidates( output: Path, ): List { val replayPath = output.resolve("replays.jsonl") - val events = readLines(replayPath).toMutableList() + val events = readCallsJournal(replayPath).toMutableList() require(events.map(CallsCoverageReplayEvent::index) == events.indices.toList()) { "Replay journal has missing or duplicate indices" } @@ -325,7 +325,7 @@ private fun replayCoverageCandidates( replay = replay.getOrNull(), error = replay.exceptionOrNull()?.let { error -> error.message ?: error::class.java.name }, ) - appendDurably(replayPath, encode(event)) + appendCallsJournalDurably(replayPath, encodeCallsArtifact(event)) events += event } } @@ -379,7 +379,7 @@ private fun validateCoverageManifestBuild( } } -private inline fun readLines(path: Path): List = if (Files.exists(path)) { +internal inline fun readCallsJournal(path: Path): List = if (Files.exists(path)) { Files.readAllLines(path).filter(String::isNotBlank).map { line -> CallsExperimentJson.json.decodeFromString(line) } @@ -387,9 +387,10 @@ private inline fun readLines(path: Path): List = if (Files.exists emptyList() } -private inline fun encode(value: T): String = CallsExperimentJson.json.encodeToUtf8SafeString(value) +internal inline fun encodeCallsArtifact(value: T): String = + CallsExperimentJson.json.encodeToUtf8SafeString(value) -private fun appendDurably(path: Path, line: String) { +internal fun appendCallsJournalDurably(path: Path, line: String) { val bytes = ByteBuffer.wrap((line + "\n").toByteArray(Charsets.UTF_8)) FileChannel.open( path, @@ -402,7 +403,7 @@ private fun appendDurably(path: Path, line: String) { } } -private fun atomicWrite(path: Path, contents: String) { +internal fun writeCallsArtifactAtomically(path: Path, contents: String) { val temporary = path.resolveSibling("${path.fileName}.tmp") Files.writeString(temporary, contents + "\n") FileChannel.open(temporary, StandardOpenOption.READ).use { it.force(true) } @@ -420,7 +421,7 @@ private val COVERAGE_CHECKPOINT_MILLIS = listOf( THIRD_CHECKPOINT_MILLIS, ) -private fun coverageUniverseSha256(bytes: ByteArray): String = MessageDigest.getInstance("SHA-256").digest(bytes) +internal fun callsArtifactSha256(bytes: ByteArray): String = MessageDigest.getInstance("SHA-256").digest(bytes) .joinToString(separator = "") { byte -> "%02x".format(byte.toInt() and UNSIGNED_BYTE_MASK) } private const val COVERAGE_UNIVERSE_ARGUMENT_COUNT = 4 diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt index 969e3b2ade..c18b82d7f9 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt @@ -29,8 +29,27 @@ internal data class CallsCoverageCandidate( val emittedAtStep: Long, val inputs: List, val newSymbolicStatements: Int, - val completion: CallsCoverageCompletion, -) + val completion: CallsCoverageCompletion?, + val entryPointId: String? = null, + val observation: CallsClassObservation? = null, + val origin: CallsCoverageCandidateOrigin = CallsCoverageCandidateOrigin.TERMINATED, + val suspendedAtMachineStop: Boolean = false, + val suspensionReason: String? = null, +) { + init { + when (origin) { + CallsCoverageCandidateOrigin.TERMINATED -> require( + completion != null && !suspendedAtMachineStop && suspensionReason == null, + ) + CallsCoverageCandidateOrigin.SUSPENDED -> require(completion == null && observation == null) { + "Suspended candidates carry inputs only, without a symbolic completion or observation" + } + } + } +} + +@Serializable +internal enum class CallsCoverageCandidateOrigin { TERMINATED, SUSPENDED } @Serializable internal enum class CallsCoverageCompletion { @@ -68,6 +87,10 @@ internal data class CallsCoverageSearchResult( val runtimeLimited: Boolean = false, val runtimeLimitationReasons: List = emptyList(), val diagnostic: String? = null, + val suspendedStates: Int = 0, + val suspendedExtractionElapsedMillis: Long = 0, + val frontierFlushElapsedMillis: Long = 0, + val machineStopReason: String? = null, ) internal data class CallsCoverageSearchRequest( @@ -95,6 +118,7 @@ internal data class CallsCoverageSearchRequest( internal class CurrentTsCallsCoverageEngine( private val sourceEngine: CurrentTsCallsSymbolicEngine = CurrentTsCallsSymbolicEngine(), ) { + @Suppress("LongMethod") fun search(request: CallsCoverageSearchRequest): CallsCoverageSearchResult { val preparationStarted = TimeSource.Monotonic.markNow() val preparation = sourceEngine.prepareFunction(request.toFunctionPreflightRequest()) @@ -122,10 +146,14 @@ internal class CurrentTsCallsCoverageEngine( val extractionFailures = mutableListOf() val machineCallStarted = TimeSource.Monotonic.markNow() val observer = CompletedCoverageCandidateObserver( - method = preparation.method, + contexts = listOf( + CallsCoverageCandidateContext( + method = preparation.method, + inputDomains = request.function.inputs, + symbolicInputs = symbolicInputs, + ), + ), coverageStatements = scope.statements, - inputDomains = request.function.inputs, - symbolicInputs = symbolicInputs, candidateCap = request.candidateCap, budgetMillis = request.budget.inWholeMilliseconds, machineCallStarted = machineCallStarted, @@ -167,6 +195,7 @@ internal class CurrentTsCallsCoverageEngine( ) } } + observer.finishTiming() val machineCallElapsedMillis = machineCallStarted.elapsedNow().inWholeMilliseconds val machineTeardownElapsedMillis = ( machineCallElapsedMillis - observer.machineSetupElapsedMillis - observer.searchElapsedMillis @@ -256,20 +285,35 @@ private fun emptyCoverageResult( ) } -private class CompletedCoverageCandidateObserver( - private val method: EtsMethod, +internal data class CallsCoverageCandidateContext( + val method: EtsMethod, + val inputDomains: List, + val symbolicInputs: CallsSymbolicInputs, + val entryPointId: String? = null, + val observationFields: Map? = null, +) + +/** + * Shared snapshot cap and input deduplication per origin and entry point. + * Suspended snapshots require neither results nor novelty. + */ +internal class CompletedCoverageCandidateObserver( + contexts: List, private val coverageStatements: Set, - private val inputDomains: List, - private val symbolicInputs: CallsSymbolicInputs, private val candidateCap: Int, private val budgetMillis: Long, private val machineCallStarted: TimeSource.Monotonic.ValueTimeMark, private val onCandidate: (CallsCoverageCandidate) -> Unit, private val onExtractionFailure: (String) -> Unit, + private val requireEntryPointWitness: Boolean = false, + private val emitSuspendedCandidates: Boolean = false, ) : UMachineObserver { + private val contextsByMethod = contexts.associateBy(CallsCoverageCandidateContext::method) private val coveredStatements = hashSetOf() private val inputKeys = hashSetOf() + private val witnessedMethods = hashSetOf() private var machineStarted: TimeSource.Monotonic.ValueTimeMark? = null + private var machineStopped = false var selectedStates: Int = 0 private set @@ -289,6 +333,18 @@ private class CompletedCoverageCandidateObserver( var searchElapsedMillis: Long = 0 private set + var suspendedStates: Int = 0 + private set + + private var suspendedExtractionElapsedNanos: Long = 0 + private var frontierFlushElapsedNanos: Long = 0 + + val suspendedExtractionElapsedMillis: Long + get() = suspendedExtractionElapsedNanos / NANOS_PER_MILLISECOND + + val frontierFlushElapsedMillis: Long + get() = frontierFlushElapsedNanos / NANOS_PER_MILLISECOND + override fun onMachineStarted() { machineSetupElapsedMillis = machineCallStarted.elapsedNow().inWholeMilliseconds machineStarted = TimeSource.Monotonic.markNow() @@ -296,6 +352,16 @@ private class CompletedCoverageCandidateObserver( override fun onMachineStopped() { searchElapsedMillis = elapsedSinceMachineStart() + machineStopped = true + } + + fun finishTiming() { + if (machineStarted == null) { + machineSetupElapsedMillis = machineCallStarted.elapsedNow().inWholeMilliseconds + } else if (!machineStopped) { + searchElapsedMillis = elapsedSinceMachineStart() + machineStopped = true + } } override fun onState(parent: TsState, forks: Sequence) { @@ -308,6 +374,8 @@ private class CompletedCoverageCandidateObserver( override fun onStateTerminated(state: TsState, stateReachable: Boolean) { if (!stateReachable) return + val context = contextsByMethod[state.entrypoint] ?: return + val method = context.method val completion = when (val result = state.methodResult) { is TsMethodResult.Success.RegularCall -> { if (result.method != method) return @@ -324,20 +392,73 @@ private class CompletedCoverageCandidateObserver( statement in coverageStatements } val newStatements = pathStatements.count { statement -> statement !in coveredStatements } - if (newStatements == 0) return + val needsWitness = requireEntryPointWitness && method !in witnessedMethods + if (newStatements == 0 && !needsWitness) return + + emitCandidate( + state = state, + context = context, + pathStatements = pathStatements, + newStatements = newStatements, + completion = completion, + origin = CallsCoverageCandidateOrigin.TERMINATED, + ) + } + + override fun onStateSuspended(state: TsState) { + if (!emitSuspendedCandidates) return + val started = TimeSource.Monotonic.markNow() + suspendedStates++ + try { + val context = contextsByMethod[state.entrypoint] ?: return + emitCandidate( + state = state, + context = context, + pathStatements = emptySet(), + newStatements = 0, + completion = null, + origin = CallsCoverageCandidateOrigin.SUSPENDED, + ) + } finally { + val elapsedNanos = started.elapsedNow().inWholeNanoseconds + suspendedExtractionElapsedNanos += elapsedNanos + if (machineStopped) frontierFlushElapsedNanos += elapsedNanos + } + } + + private fun emitCandidate( + state: TsState, + context: CallsCoverageCandidateContext, + pathStatements: Set, + newStatements: Int, + completion: CallsCoverageCompletion?, + origin: CallsCoverageCandidateOrigin, + ) { if (selectedStates >= candidateCap) { capReached = true return } selectedStates++ val candidate = runCatching { - val inputs = symbolicInputs.resolve(state).firstOrNull { candidate -> + val inputDomains = context.inputDomains + val symbolicInputs = context.symbolicInputs + val extractedInputs = symbolicInputs.resolve(state) + val canResolveAnyModel = context.observationFields == null || + origin == CallsCoverageCandidateOrigin.SUSPENDED + val eligibleInputs = if (canResolveAnyModel) { + extractedInputs + } else { + // Output/state extraction uses the first model too; do not pair it with another model's inputs. + extractedInputs.take(1) + } + val inputs = eligibleInputs.firstOrNull { candidate -> candidate.size == inputDomains.size && candidate.zip(inputDomains).all { (value, input) -> value in input.domain } } requireNotNull(inputs) { "No extracted input is inside every frozen domain" } - val key = CallsExperimentJson.json.encodeToUtf8SafeString(inputs) + val key = origin.name + ":" + context.entryPointId.orEmpty() + + CallsExperimentJson.json.encodeToUtf8SafeString(inputs) if (inputKeys.add(key)) { CallsCoverageCandidate( emittedAtMillis = elapsedSinceMachineStart(), @@ -345,6 +466,17 @@ private class CompletedCoverageCandidateObserver( inputs = inputs, newSymbolicStatements = newStatements, completion = completion, + entryPointId = context.entryPointId, + observation = context.observationFields?.takeIf { + origin == CallsCoverageCandidateOrigin.TERMINATED + }?.let { fields -> + extractCallsClassObservation(state = state, method = context.method, fields = fields) + }, + origin = origin, + suspendedAtMachineStop = origin == CallsCoverageCandidateOrigin.SUSPENDED && machineStopped, + suspensionReason = state.unsupportedReason.takeIf { + origin == CallsCoverageCandidateOrigin.SUSPENDED + }, ) } else { null @@ -356,7 +488,10 @@ private class CompletedCoverageCandidateObserver( if (candidate != null) { onCandidate(candidate) - coveredStatements += pathStatements + if (origin == CallsCoverageCandidateOrigin.TERMINATED) { + coveredStatements += pathStatements + witnessedMethods += context.method + } } } @@ -371,3 +506,5 @@ private fun CallsSymbolicStatus.toCoverageStatus(): CallsCoverageSearchStatus = CallsSymbolicStatus.AMBIGUOUS -> CallsCoverageSearchStatus.AMBIGUOUS else -> CallsCoverageSearchStatus.TOOL_ERROR } + +private const val NANOS_PER_MILLISECOND = 1_000_000L diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt index 9588cf70ef..587aa2237f 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt @@ -9,6 +9,10 @@ fun main(args: Array) { require(args.isNotEmpty()) { usage() } when (args.first()) { + "class-catalog" -> runClassCatalog(args.drop(1)) + "class-prepare" -> runClassPrepare(args.drop(1)) + "class-preflight" -> runClassPreflight(args.drop(1)) + "class-cell" -> runClassCell(args.drop(1)) "run" -> runExperiment(args.drop(1)) "coverage-universe" -> runCoverageUniverse(args.drop(1)) "coverage-cell" -> runCoverageCell(args.drop(1)) @@ -89,6 +93,10 @@ private fun summarize(args: List) { private fun usage(): String = """ Usage: + calls class-catalog + calls class-prepare + calls class-preflight + calls class-cell calls run calls coverage-universe calls coverage-cell diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsSymbolicInputs.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsSymbolicInputs.kt index a4f314ff79..fd3c927b18 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsSymbolicInputs.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsSymbolicInputs.kt @@ -52,8 +52,11 @@ internal class CallsSymbolicInputs( private val lexicalEnvironment: EtsLexicalEnvironmentBinding?, private val captures: List, sourceScene: EtsScene, + private val objectShapes: PlainObjectInputShapes = PlainObjectInputShapes( + sourceScene = sourceScene, + domains = inputs.map(PropertyInput::domain), + ), ) { - private val objectShapes = PlainObjectInputShapes(sourceScene, inputs.map(PropertyInput::domain)) val scene: EtsScene = objectShapes.scene private val boundInputs: List> diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsUnknownCallTelemetry.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsUnknownCallTelemetry.kt index 9af0ae2733..c2015aaf42 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsUnknownCallTelemetry.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsUnknownCallTelemetry.kt @@ -138,7 +138,7 @@ internal fun CallsExperimentCellIdentity.unknownCallRecord( ) } -private fun EtsStmt.toCallsUnknownCallSite(): CallsUnknownCallSite { +internal fun EtsStmt.toCallsUnknownCallSite(): CallsUnknownCallSite { val stmtLocation = location val origin = stmtLocation.origin val containingMethod = stmtLocation.method.signature @@ -165,7 +165,7 @@ internal fun callsUnknownCallEventSink( } } -private fun EtsMethodSignature.toCallsUnknownCallCallee(): CallsUnknownCallCallee { +internal fun EtsMethodSignature.toCallsUnknownCallCallee(): CallsUnknownCallCallee { val parameterTypes = parameters.map { parameter -> parameter.type.toString() } val namespace = enclosingClass.namespace?.qualifiedName() val className = listOfNotNull(namespace, enclosingClass.name).joinToString(separator = "::") diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt index 3e4ce47438..1e1375a164 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt @@ -48,6 +48,7 @@ internal data class CallsFunctionPreflightRequest( val project: CallsProjectCase, val function: CallsFunctionCase, val expectedNativeFrontendRevision: String, + val verifySourceCheckout: Boolean = true, ) internal sealed interface CallsFunctionPreparation { @@ -109,6 +110,7 @@ internal class CurrentTsCallsSymbolicEngine( private val bundledNativeFrontendRevision: String = CallsBuildIdentity.nativeFrontendRevision, ) : CallsSymbolicEngine { private val verifiedProjects = mutableMapOf() + private val preparedSources = mutableMapOf, CallsSourceProject>() private var preparedTarget: Pair? = null private var verifiedNativeFrontendIdentity: String? = null @@ -311,17 +313,25 @@ internal class CurrentTsCallsSymbolicEngine( ) } - verifyGitCheckoutOnce( - checkout = request.sourceRoot, - expectedRevision = request.project.revision, - cache = verifiedProjects, - ) + if (request.verifySourceCheckout) { + verifyGitCheckoutOnce( + checkout = request.sourceRoot, + expectedRevision = request.project.revision, + cache = verifiedProjects, + ) + } verifyNativeFrontendOnce(expectedRevision = request.expectedNativeFrontendRevision) val source = request.sourceRoot.resolve(request.function.sourceFile).normalize() require(source.startsWith(request.sourceRoot)) { "Function source escapes its frozen source root" } - val project = loadCallsSourceProject(sourceRoot = request.sourceRoot, source = source) + val project = if (request.verifySourceCheckout) { + loadCallsSourceProject(sourceRoot = request.sourceRoot, source = source) + } else { + preparedSources.getOrPut(request.sourceRoot to source) { + loadCallsSourceProject(sourceRoot = request.sourceRoot, source = source) + } + } val loaded = when (project) { is CallsSourceProject.Loaded -> project is CallsSourceProject.Unsupported -> return CallsFunctionPreparation.Rejected( diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassAdmissionTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassAdmissionTest.kt new file mode 100644 index 0000000000..e71f012d1e --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassAdmissionTest.kt @@ -0,0 +1,403 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject +import org.junit.jupiter.api.io.TempDir +import org.usvm.ts.pbt.model.BooleanDomain +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.NumberDomain +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.model.StringDomain +import org.usvm.ts.pbt.model.contains +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +class CallsClassAdmissionTest { + @TempDir + lateinit var directory: Path + + @Test + fun `class admission does not require a symbolic pilot to complete any hashed method scenario`() { + val harness = fixture() + val instrumentation = instrumentation() + val sourceFailures = callsClassSourceAdmissionFailures(harness, CallsBuildIdentity.nativeFrontendRevision) + + val smoke = callsClassNativeConstructorSmoke(harness, instrumentation, seed = 17, timeoutMillis = 20_000) + val repeated = callsClassNativeConstructorSmoke(harness, instrumentation, seed = 17, timeoutMillis = 20_000) + val result = admission(harness, sourceFailures, smoke) + + assertEquals(emptyList(), sourceFailures) + assertEquals(CallsClassNativeSmokeStatus.VERIFIED, smoke.status) + val original = assertNotNull(smoke.original) + assertEquals(original.inputs, assertNotNull(repeated.original).inputs) + assertEquals(original.observation, repeated.original.observation) + assertTrue(original.inputs.single() in harness.surface.constructorInputs.single().domain) + assertTrue(result.eligible) + assertEquals(true, result.primaryInferenceEligible) + assertEquals(false, result.diagnosticReady) + assertEquals(CallsCoverageSearchStatus.TIMEOUT, result.search.search.status) + assertEquals(harness.scenarios.map { it.id }, result.missingRequiredScenarios) + assertEquals(harness.scenarios.size, result.missingRequiredScenarios.size) + } + + @Test + fun `native sampler rejects seed overflow instead of truncating the frozen seed`() { + val harness = fixture() + + OriginalTypeScriptClassReplayer( + directory, + harness.functions.first(), + harness.observations.first(), + ).use { replayer -> + assertFailsWith { + replayer.sample(seed = Int.MAX_VALUE.toLong() + 1, timeoutMillis = 20_000) + } + assertFailsWith { + replayer.sample(seed = Int.MIN_VALUE.toLong() - 1, timeoutMillis = 20_000) + } + } + } + + @Test + fun `pilot native corruption remains an acquisition failure without selecting the class population`() { + val harness = fixture() + val originalInstrumentation = instrumentation() + val modified = originalInstrumentation.instrumentedSource.replace( + "return key.length === this.size;", + "return false;", + ) + val changed = originalInstrumentation.copy( + instrumentedSource = modified, + instrumentedSha256 = callsClassSourceSha256(modified.toByteArray(Charsets.UTF_8)), + ) + val smoke = callsClassNativeConstructorSmoke(harness, changed, seed = 17, timeoutMillis = 20_000) + val inputs = listOf(JsConcreteValue.String("aa"), JsConcreteValue.String("aa")) + + val original = OriginalTypeScriptClassReplayer(directory, harness.functions[1], harness.observations[1]).use { + it.replay(inputs = inputs, timeoutMillis = 20_000) + } + val instrumented = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[1], + observationFields = harness.observations[1], + sourceOverride = changed.instrumentedSource, + overrideSourceFile = "SlowMap.ts", + counterGlobalName = changed.hitGlobalName, + ).use { it.replay(inputs = inputs, timeoutMillis = 20_000) } + val replay = CallsClassReplayEvent( + index = 0, + scenarioId = harness.scenarios[1].id, + original = original, + instrumented = instrumented, + oracle = compareCallsClassObservations( + original.observation, + original.observation, + instrumented.observation, + ), + nativeCoverage = callsClassNativeCoverageAcceptance(original, instrumented, changed.universe), + ) + val failures = callsClassPilotNativeFailures(listOf(replay), changed.universe) + val baseline = admission(harness, sourceFailures = emptyList(), smoke = smoke) + val result = callsClassPreflightResult( + classCase = classCase(harness), + preparedSha256 = baseline.preparedSha256, + toolRevision = baseline.toolRevision, + search = baseline.search, + replays = listOf(replay), + missingScenarios = baseline.missingRequiredScenarios, + pilotDiagnostics = emptyList(), + sourceAdmissionFailures = emptyList(), + nativeSmoke = smoke, + nativeInfrastructureFailures = failures, + ) + + assertEquals(CallsClassNativeSmokeStatus.VERIFIED, smoke.status) + assertTrue(failures.isNotEmpty(), replay.toString()) + assertFalse(replay.oracle.accepted) + assertTrue(result.eligible) + assertEquals(true, result.primaryInferenceEligible) + assertEquals(false, result.diagnosticReady) + assertEquals(failures, result.nativeInfrastructureFailures) + } + + @Test + fun `changed native constructor state rejects the observation mechanism independently of pilot coverage`() { + val harness = fixture() + val originalInstrumentation = instrumentation() + val modified = originalInstrumentation.instrumentedSource.replace( + "this.size = initial.length;", + "this.size = initial.length + 1;", + ) + val changed = originalInstrumentation.copy( + instrumentedSource = modified, + instrumentedSha256 = callsClassSourceSha256(modified.toByteArray(Charsets.UTF_8)), + ) + + val smoke = callsClassNativeConstructorSmoke(harness, changed, seed = 17, timeoutMillis = 20_000) + val result = admission(harness, sourceFailures = emptyList(), smoke = smoke) + + assertEquals(CallsClassNativeSmokeStatus.INVALID, smoke.status) + assertFalse(result.eligible) + assertEquals(false, result.primaryInferenceEligible) + val changedState = smoke.diagnostic?.contains( + "Instrumentation changed native returned values or receiver state", + ) == true + assertTrue(changedState) + } + + @Test + fun `instrumentation changing native observability is invalid even with unchanged frozen inputs`() { + val harness = fixture() + val originalInstrumentation = instrumentation() + val modified = originalInstrumentation.instrumentedSource.replace( + "this.size = initial.length;", + "this.size = 'outside-the-number-schema' as any;", + ) + val changed = originalInstrumentation.copy( + instrumentedSource = modified, + instrumentedSha256 = callsClassSourceSha256(modified.toByteArray(Charsets.UTF_8)), + ) + + val smoke = callsClassNativeConstructorSmoke(harness, changed, seed = 17, timeoutMillis = 20_000) + val original = assertNotNull(smoke.original) + val instrumented = assertNotNull(smoke.instrumented) + val sameCompletionIncomplete = instrumented.copy( + observation = instrumented.observation.copy(completion = original.observation.completion), + ) + + assertEquals(original.inputs, instrumented.inputs) + assertTrue(original.observation.representable) + assertFalse(instrumented.observation.representable) + assertEquals(CallsClassNativeSmokeStatus.INVALID, smoke.status) + assertEquals( + "Instrumentation changed native observability or observation schema", + callsClassNativeInfrastructureFailure(original, sameCompletionIncomplete), + ) + } + + @Test + fun `lawful incomplete native constructor observation remains inconclusive without silently excluding raw cells`() { + val harness = fixture(source = source.replace("this.size = initial.length;", "throw 7;")) + + val smoke = callsClassNativeConstructorSmoke(harness, instrumentation(), seed = 17, timeoutMillis = 20_000) + val result = admission(harness, sourceFailures = emptyList(), smoke = smoke) + + assertEquals(CallsClassNativeSmokeStatus.INCONCLUSIVE, smoke.status) + assertTrue(result.eligible) + assertEquals(false, result.primaryInferenceEligible) + assertNotNull(smoke.diagnostic) + } + + @Test + fun `source closure preparation failure is a hard admission gate independent of pilot outcomes`() { + val harness = fixture(source = "const cache: number[][] = [];\n" + source) + val failures = callsClassSourceAdmissionFailures(harness, CallsBuildIdentity.nativeFrontendRevision) + + val result = admission(harness, sourceFailures = failures, smoke = null) + + assertTrue(failures.any { it.contains("TOP_LEVEL_INITIALIZATION_UNSUPPORTED") }, failures.toString()) + assertFalse(result.eligible) + assertEquals(false, result.primaryInferenceEligible) + assertEquals(failures, result.sourceAdmissionFailures) + } + + @Test + fun `symbolic fresh-result disagreement remains a pilot outcome with strict candidate rejection`() { + val harness = fixture() + val smoke = callsClassNativeConstructorSmoke(harness, instrumentation(), seed = 17, timeoutMillis = 20_000) + val original = assertNotNull(smoke.original) + val instrumented = assertNotNull(smoke.instrumented) + val symbolic = original.observation.copy(values = mapOf("state_0_size" to JsConcreteValue.number(-1.0))) + val oracle = compareCallsClassObservations(symbolic, original.observation, instrumented.observation) + val baseline = admission(harness, sourceFailures = emptyList(), smoke = smoke) + val replay = CallsClassReplayEvent( + index = 0, + scenarioId = smoke.scenarioId, + original = original, + instrumented = instrumented, + oracle = oracle, + nativeCoverage = callsClassNativeCoverageAcceptance(original, instrumented, instrumentation().universe), + ) + val candidate = CallsCoverageCandidate( + emittedAtMillis = 1, + emittedAtStep = 1, + inputs = original.inputs, + newSymbolicStatements = 1, + completion = CallsCoverageCompletion.RETURNED, + entryPointId = smoke.scenarioId, + observation = symbolic, + ) + val pilot = baseline.search.copy( + search = baseline.search.search.copy( + candidates = listOf(candidate), + selectedStates = 1, + ), + ) + + val result = callsClassPreflightResult( + classCase = classCase(harness), + preparedSha256 = baseline.preparedSha256, + toolRevision = baseline.toolRevision, + search = pilot, + replays = listOf(replay), + missingScenarios = baseline.missingRequiredScenarios, + pilotDiagnostics = listOf("Symbolic and original returned values or receiver state disagree"), + sourceAdmissionFailures = emptyList(), + nativeSmoke = smoke, + nativeInfrastructureFailures = listOfNotNull(callsClassNativeInfrastructureFailure(original, instrumented)), + ) + + assertFalse(oracle.accepted) + assertTrue(replay.nativeCoverage.accepted) + assertTrue(result.eligible) + assertEquals(true, result.primaryInferenceEligible) + assertEquals(false, result.diagnosticReady) + assertEquals(listOf(replay), result.replays) + } + + @Test + fun `documented frozen correctness frontier excludes primary inference while preserving raw diagnostics`() { + val harness = fixture() + val smoke = callsClassNativeConstructorSmoke(harness, instrumentation(), seed = 17, timeoutMillis = 20_000) + val failure = CallsClassCorrectnessFrontierFailure( + reason = "Known constructor lowering defect", + provenance = "Pinned frontend regression and independently replayed counterexample", + ) + val classCase = classCase(harness).copy(correctnessFrontierFailures = listOf(failure)) + + val result = admission(harness, sourceFailures = emptyList(), smoke = smoke, selectedCase = classCase) + + assertTrue(result.eligible) + assertEquals(false, result.primaryInferenceEligible) + assertEquals(listOf(failure), result.correctnessFrontierFailures) + } + + @Test + fun `new admission serializes explicitly and legacy strict preflight remains version zero`() { + val harness = fixture() + val smoke = callsClassNativeConstructorSmoke(harness, instrumentation(), seed = 17, timeoutMillis = 20_000) + val result = admission(harness, sourceFailures = emptyList(), smoke = smoke) + + val encoded = CallsExperimentJson.json.encodeToString(result) + val decoded = CallsExperimentJson.json.decodeFromString(encoded) + val addedFields = setOf( + "admissionVersion", + "sourceAdmissionFailures", + "nativeInfrastructureFailures", + "nativeSmoke", + "diagnosticReady", + "primaryInferenceEligible", + "correctnessFrontierFailures", + ) + val legacy = JsonObject( + CallsExperimentJson.json.parseToJsonElement(encoded).jsonObject.filterKeys { + it !in addedFields + }, + ) + val decodedLegacy = CallsExperimentJson.json.decodeFromString(legacy.toString()) + + assertEquals(result, decoded) + assertEquals(1, decoded.admissionVersion) + assertEquals(0, decodedLegacy.admissionVersion) + assertEquals(null, decodedLegacy.primaryInferenceEligible) + assertEquals(null, decodedLegacy.nativeSmoke) + } + + private fun fixture(source: String = this.source): CallsClassPreparedHarness { + Files.writeString(directory.resolve("SlowMap.ts"), source) + val surface = CallsClassSurface( + sourceFile = "SlowMap.ts", + exportName = "SlowMap", + constructorInputs = listOf( + PropertyInput(name = "initial", domain = StringDomain(minLength = 2, maxLength = 6)), + ), + members = listOf( + CallsClassMember( + id = "has", + name = "has", + kind = CallsClassMemberKind.METHOD, + inputs = listOf(PropertyInput(name = "key", domain = StringDomain(minLength = 2, maxLength = 6))), + resultDomain = BooleanDomain, + ), + ), + stateFields = mapOf("size" to NumberDomain()), + ) + val project = CallsProjectCase( + projectId = "fixture", + revision = "0".repeat(40), + sourceRoot = ".", + development = true, + functions = emptyList(), + ) + + return prepareCallsClassHarness(directory, project, surface, maxSequenceLength = 2) + } + + private fun instrumentation() = CallsClassSourceInstrumentation.create( + sourceRoot = directory, + sourceFile = "SlowMap.ts", + className = "SlowMap", + ) + + private fun classCase(harness: CallsClassPreparedHarness) = CallsClassCase( + classId = "slow-map", + projectId = "fixture", + revision = "0".repeat(40), + sourceRoot = directory.toString(), + sourceSha256 = mapOf( + "SlowMap.ts" to callsClassSourceSha256(Files.readAllBytes(directory.resolve("SlowMap.ts"))), + ), + surface = harness.surface, + ) + + private fun admission( + harness: CallsClassPreparedHarness, + sourceFailures: List, + smoke: CallsClassNativeSmokeResult?, + selectedCase: CallsClassCase = classCase(harness), + ) = callsClassPreflightResult( + classCase = selectedCase, + preparedSha256 = "a".repeat(64), + toolRevision = "b".repeat(40), + search = CallsClassSearchResult( + search = CallsCoverageSearchResult( + status = CallsCoverageSearchStatus.TIMEOUT, + candidates = emptyList(), + selectedStates = 0, + executedSteps = 10, + stepsWithinBudget = 10, + extractionFailures = emptyList(), + candidateCapReached = false, + preparationElapsedMillis = 0, + machineSetupElapsedMillis = 0, + searchElapsedMillis = 120_000, + machineTeardownElapsedMillis = 0, + ), + scenarios = emptyList(), + visits = emptyList(), + unknownCallEvents = 0, + ), + replays = emptyList(), + missingScenarios = harness.scenarios.map { it.id }, + pilotDiagnostics = listOf("Symbolic pilot reached the time budget", "No scenario witnesses"), + sourceAdmissionFailures = sourceFailures, + nativeSmoke = smoke, + nativeInfrastructureFailures = emptyList(), + ) + + private val source = """ + export class SlowMap { + size: number; + constructor(initial: string) { this.size = initial.length; } + has(key: string): boolean { return key.length === this.size; } + } + """.trimIndent() +} diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassCoverageSourceTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassCoverageSourceTest.kt new file mode 100644 index 0000000000..5b234a7af2 --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassCoverageSourceTest.kt @@ -0,0 +1,171 @@ +package org.usvm.ts.calls + +import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.model.BooleanDomain +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.PropertyInput +import java.nio.file.Files +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNotEquals +import kotlin.test.assertTrue + +internal class CallsClassCoverageSourceTest { + @Test + fun `class universe freezes exact statements and checks native IDs`() { + val sourceRoot = Files.createTempDirectory("calls-class-universe-") + try { + Files.writeString( + sourceRoot.resolve("Target.ts"), + "export class Target { run(x: boolean) { if (x) return 1; return 2; } }", + ) + + val instrumented = CallsClassSourceInstrumentation.create( + sourceRoot = sourceRoot, + sourceFile = "Target.ts", + className = "Target", + ) + val universe = instrumented.universe + val surface = CallsClassSurface( + sourceFile = "Target.ts", + exportName = "Target", + constructorInputs = emptyList(), + members = listOf( + CallsClassMember( + id = "run", + name = "run", + kind = CallsClassMemberKind.METHOD, + inputs = listOf(PropertyInput(name = "x", domain = BooleanDomain)), + resultDomain = IntegerDomain(min = 1, max = 2), + ), + ), + stateFields = emptyMap(), + ) + instrumented.validateSurface(surface = surface) + assertFailsWith { + instrumented.validateSurface(surface = surface.copy(members = emptyList())) + } + assertFailsWith { + instrumented.validateSurface(surface = surface.copy(stateFields = mapOf("foreign" to BooleanDomain))) + } + universe.verifyUnchanged(sourceRoot = sourceRoot) + + assertEquals(expected = 3, actual = universe.statements.size) + assertEquals(expected = 3, actual = universe.statementIds.size) + assertTrue(universe.statements.all { statement -> statement.startLine == 1 }) + assertNotEquals(illegal = universe.sourceSha256, actual = instrumented.instrumentedSha256) + val firstId = universe.statements.first().id + assertEquals(expected = setOf(firstId), actual = universe.checkedCoveredIds(listOf(firstId, firstId))) + assertFailsWith { universe.checkedCoveredIds(listOf("foreign-id")) } + + val repeated = CallsClassSourceInstrumentation.create( + sourceRoot = sourceRoot, + sourceFile = "Target.ts", + className = "Target", + ) + assertEquals(expected = instrumented, actual = repeated) + } finally { + Files.walk(sourceRoot).use { files -> + files.sorted(Comparator.reverseOrder()).forEach(Files::deleteIfExists) + } + } + } + + @Test + fun `companion registry preserves exact same-source data class fields`() { + val sourceRoot = Files.createTempDirectory("calls-class-data-registry-") + try { + Files.writeString( + sourceRoot.resolve("Target.ts"), + """ + export class Target { run() { return 1; } } + export class MapEntry { + key: K + value: V + + constructor(key: K, value: V) { + this.key = key + this.value = value + } + } + export class CallableCompanion { value = 1; run() { return 1; } } + """.trimIndent(), + ) + val companion = CallsClassDataClass( + id = "entry", + exportName = "MapEntry", + fields = mapOf("key" to IntegerDomain(min = 0, max = 1), "value" to BooleanDomain), + ) + val surface = CallsClassSurface( + sourceFile = "Target.ts", + exportName = "Target", + constructorInputs = emptyList(), + members = listOf( + CallsClassMember( + id = "run", + name = "run", + kind = CallsClassMemberKind.METHOD, + inputs = emptyList(), + resultDomain = IntegerDomain(min = 1, max = 1), + ), + ), + stateFields = emptyMap(), + dataClasses = listOf(companion), + ) + val instrumented = CallsClassSourceInstrumentation.create( + sourceRoot = sourceRoot, + sourceFile = "Target.ts", + className = "Target", + ) + + instrumented.validateSurface(surface = surface) + val missingField = companion.copy(fields = companion.fields - "value") + val extraField = companion.copy(fields = companion.fields + ("foreign" to BooleanDomain)) + val unknownClass = companion.copy(exportName = "Unknown") + val callableClass = companion.copy( + exportName = "CallableCompanion", + fields = mapOf("value" to BooleanDomain), + ) + for (invalid in listOf(missingField, extraField, unknownClass, callableClass)) { + assertFailsWith { + instrumented.validateSurface(surface = surface.copy(dataClasses = listOf(invalid))) + } + } + } finally { + Files.walk(sourceRoot).use { files -> + files.sorted(Comparator.reverseOrder()).forEach(Files::deleteIfExists) + } + } + } + + @Test + fun `class source drift and unsupported reachability remain explicit failures`() { + val sourceRoot = Files.createTempDirectory("calls-class-source-drift-") + try { + val source = sourceRoot.resolve("Target.ts") + Files.writeString(source, "export class Target { run() { return 1; } }") + val universe = CallsClassSourceInstrumentation.create( + sourceRoot = sourceRoot, + sourceFile = "Target.ts", + className = "Target", + ).universe + + Files.writeString(source, "export class Target { run() { return 2; } }") + assertFailsWith { universe.verifyUnchanged(sourceRoot = sourceRoot) } + + Files.writeString(source, "export class Target { run(key: string) { return this[key]; } }") + val failure = assertFailsWith { + CallsClassSourceInstrumentation.create( + sourceRoot = sourceRoot, + sourceFile = "Target.ts", + className = "Target", + ) + } + assertTrue(failure.message.orEmpty().contains("Computed receiver")) + } finally { + Files.walk(sourceRoot).use { files -> + files.sorted(Comparator.reverseOrder()).forEach(Files::deleteIfExists) + } + } + } +} diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassDataRegistryOracleTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassDataRegistryOracleTest.kt new file mode 100644 index 0000000000..f575a1071f --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassDataRegistryOracleTest.kt @@ -0,0 +1,388 @@ +package org.usvm.ts.calls + +import org.junit.jupiter.api.io.TempDir +import org.usvm.SolverType +import org.usvm.ts.pbt.model.ArrayDomain +import org.usvm.ts.pbt.model.BooleanDomain +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.NumberDomain +import org.usvm.ts.pbt.model.ObjectDomain +import org.usvm.ts.pbt.model.PropertyDomain +import org.usvm.ts.pbt.model.PropertyInput +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +class CallsClassDataRegistryOracleTest { + @TempDir + lateinit var directory: Path + + @Test + fun `registered nested data class preserves nominal identity against a plain object`() { + val harness = dataClassFixture() + val function = harness.functions[1] + val observations = harness.observations[1] + val original = OriginalTypeScriptClassReplayer(directory, function, observations).use { + it.replay(inputs = emptyList(), timeoutMillis = 20_000) + } + val plainSource = dataClassSource.replace("new Entry(1)", "{ value: 1 }") + val plain = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = function, + observationFields = observations, + sourceOverride = plainSource, + overrideSourceFile = "Entries.ts", + ).use { it.replay(inputs = emptyList(), timeoutMillis = 20_000) } + + val entry = JsConcreteValue.Object( + fields = mapOf("value" to number(1), CLASS_IDENTITY_FIELD to JsConcreteValue.String("entry")), + ) + val plainEntry = entry.copy(fields = entry.fields + (CLASS_IDENTITY_FIELD to JsConcreteValue.String("plain"))) + assertTrue(original.observation.representable, original.toString()) + assertTrue(plain.observation.representable, plain.toString()) + assertEquals(JsConcreteValue.Array(listOf(entry)), original.observation.values["result_0"]) + assertEquals(JsConcreteValue.Array(listOf(plainEntry)), plain.observation.values["result_0"]) + assertEquals(JsConcreteValue.String("target"), original.observation.values["state_1_$CLASS_IDENTITY_FIELD"]) + assertFalse( + compareCallsClassObservations(original.observation, original.observation, plain.observation).accepted + ) + + val self = OriginalTypeScriptClassReplayer(directory, harness.functions[2], harness.observations[2]).use { + it.replay(inputs = emptyList(), timeoutMillis = 20_000) + } + val selfResult = self.observation.values["result_0"] as JsConcreteValue.Object + assertTrue(self.observation.representable, self.toString()) + assertEquals(JsConcreteValue.String("target"), selfResult.fields[CLASS_IDENTITY_FIELD]) + assertEquals(JsConcreteValue.Array(listOf(entry)), selfResult.fields["items"]) + } + + @Test + fun `registered nested data class rejects unlisted prototypes and changed own fields`() { + val harness = dataClassFixture() + val replacements = listOf( + "new OtherEntry(1)" to "Unlisted object prototype", + "new ChildEntry(1)" to "Unlisted object prototype", + "Object.assign(new Entry(1), { extra: 2 })" to "Registered data-class own fields changed", + "Object.defineProperty(new Entry(1), 'value', { get: () => 1 })" to + "Registered data-class field is not an ordinary data property", + "new Date(1)" to "Unlisted object prototype", + ) + for ((replacement, expectedDiagnostic) in replacements) { + val changed = dataClassSource.replace("new Entry(1)", replacement) + """ + + class OtherEntry { value: number; constructor(value: number) { this.value = value; } } + class ChildEntry extends Entry {} + """.trimIndent() + val replay = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[1], + observationFields = harness.observations[1], + sourceOverride = changed, + overrideSourceFile = "Entries.ts", + ).use { it.replay(inputs = emptyList(), timeoutMillis = 20_000) } + + assertFalse(replay.observation.representable, replay.toString()) + assertTrue(replay.observation.diagnostic?.contains(expectedDiagnostic) == true, replay.toString()) + } + } + + @Test + fun `registered data class rejects constructor and prototype mutation`() { + val harness = dataClassFixture() + val mutations = listOf( + "(Entry as any).hidden = 2;", + "(Entry.prototype as any).hidden = 2;", + "Object.defineProperty(Entry.prototype, 'value', { get: () => 1 });", + "Object.setPrototypeOf(Entry.prototype, {});", + "Object.defineProperty(Entry, 'name', { get: () => 'Entry' });", + "Object.defineProperty(Entry, 'name', { value: 'Changed' });", + "Object.defineProperty(Entry, 'length', { value: 99 });", + ) + for (mutation in mutations) { + val changed = dataClassSource.replace("read(): Entry[] {", "read(): Entry[] { $mutation") + val replay = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[1], + observationFields = harness.observations[1], + sourceOverride = changed, + overrideSourceFile = "Entries.ts", + ).use { it.replay(inputs = emptyList(), timeoutMillis = 20_000) } + + assertFalse(replay.observation.representable, replay.toString()) + val changedContract = replay.observation.diagnostic?.contains( + "Registered data-class constructor or prototype changed" + ) == true + assertTrue(changedContract, replay.toString()) + } + } + + @Test + fun `registered class fields require the exact frozen observation domain`() { + val harness = dataClassFixture(observationFieldDomain = IntegerDomain(min = 0, max = 2)) + + val replay = OriginalTypeScriptClassReplayer(directory, harness.functions[1], harness.observations[1]).use { + it.replay(inputs = emptyList(), timeoutMillis = 20_000) + } + + assertFalse(replay.observation.representable, replay.toString()) + assertTrue( + replay.observation.diagnostic?.contains("Registered data-class observation domain changed") == true, + replay.toString(), + ) + } + + @Test + fun `registry identity field collisions reject before execution`() { + val entryFields = mapOf("value" to NumberDomain()) + val registry = listOf(CallsClassDataClass(id = "entry", exportName = "Entry", fields = entryFields)) + val collision = ObjectDomain(fields = mapOf(CLASS_IDENTITY_FIELD to NumberDomain())) + + assertFailsWith { + surface().copy(stateFields = mapOf("nested" to ArrayDomain(element = collision)), dataClasses = registry) + } + assertFailsWith { + surface().copy(dataClasses = registry + registry) + } + assertFailsWith { + surface().copy(dataClasses = listOf(registry.single().copy(fields = collision.fields))) + } + assertFailsWith { + surface().copy(dataClasses = listOf(registry.single().copy(id = "plain"))) + } + } + + @Test + fun `symbolic nested registered instances and target identity agree with native observations`() { + val harness = dataClassFixture() + val result = CurrentTsCallsClassCoverageEngine().search( + CallsClassSearchRequest( + harness = harness, + profile = CallsExperimentProfile.EMPTY_STOP, + frozenModelIds = emptySet(), + expectedNativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision, + seed = 1, + budget = 3.seconds, + timeQuantum = 100.milliseconds, + solverQueryLimit = 1.seconds, + solverType = SolverType.YICES, + candidateCap = 10, + requireEntryPointWitness = true, + ), + ) + + assertEquals(harness.scenarios.map { it.id }.toSet(), result.search.candidates.map { it.entryPointId }.toSet()) + for (candidate in result.search.candidates) { + val index = harness.scenarios.indexOfFirst { it.id == candidate.entryPointId } + val original = OriginalTypeScriptClassReplayer( + directory, + harness.functions[index], + harness.observations[index], + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = 20_000) } + val instrumented = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[index], + observationFields = harness.observations[index], + sourceOverride = dataClassSource.replace( + "return this.items;", + "; (globalThis as any).entryHits?.add('read'); return this.items;", + ), + overrideSourceFile = "Entries.ts", + counterGlobalName = "entryHits", + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = 20_000) } + + val checked = compareCallsClassObservations( + candidate.observation, + original.observation, + instrumented.observation + ) + assertTrue(checked.accepted, "$candidate / $original / $instrumented / $checked") + } + } + + @Test + fun `boxed conditional array elements retain registered or plain identity on both branches`() { + val conditionalSource = dataClassSource + .replace("items: Entry[];", "items: any[];") + .replace( + "constructor() { this.items = [new Entry(1)]; }", + "constructor(flag: boolean) { this.items = [flag ? new Entry(1) : { value: 1 }]; }", + ) + val harness = dataClassFixture( + fixtureSource = conditionalSource, + constructorInputs = listOf(PropertyInput(name = "flag", domain = BooleanDomain)), + ) + + val result = CurrentTsCallsClassCoverageEngine().search( + CallsClassSearchRequest( + harness = harness, + profile = CallsExperimentProfile.EMPTY_STOP, + frozenModelIds = emptySet(), + expectedNativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision, + seed = 1, + budget = 3.seconds, + timeQuantum = 100.milliseconds, + solverQueryLimit = 1.seconds, + solverType = SolverType.YICES, + candidateCap = 20, + requireEntryPointWitness = true, + ), + ) + + val constructorScenarioId = harness.scenarios.first().id + val constructorBranches = result.search.candidates.filter { it.entryPointId == constructorScenarioId } + .map { it.inputs.single() }.toSet() + assertEquals(setOf(JsConcreteValue.Boolean(true), JsConcreteValue.Boolean(false)), constructorBranches) + assertEquals(harness.scenarios.map { it.id }.toSet(), result.search.candidates.map { it.entryPointId }.toSet()) + for (candidate in result.search.candidates) { + val index = harness.scenarios.indexOfFirst { it.id == candidate.entryPointId } + val original = OriginalTypeScriptClassReplayer( + directory, + harness.functions[index], + harness.observations[index], + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = 20_000) } + val instrumented = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[index], + observationFields = harness.observations[index], + sourceOverride = conditionalSource.replace( + "return this.items;", + "; (globalThis as any).entryHits?.add('read'); return this.items;", + ), + overrideSourceFile = "Entries.ts", + counterGlobalName = "entryHits", + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = 20_000) } + + val checked = compareCallsClassObservations( + candidate.observation, + original.observation, + instrumented.observation + ) + assertTrue(checked.accepted, "$candidate / $original / $instrumented / $checked") + val createsRegisteredEntry = (candidate.inputs.single() as JsConcreteValue.Boolean).value + val expectedIdentity = if (createsRegisteredEntry) "entry" else "plain" + val symbolic = requireNotNull(candidate.observation) + val items = symbolic.values.getValue("state_0_items") as JsConcreteValue.Array + val observedEntry = items.elements.single() as JsConcreteValue.Object + assertEquals(JsConcreteValue.String(expectedIdentity), observedEntry.fields[CLASS_IDENTITY_FIELD]) + } + } + + @Test + fun `frozen registry domains compare fields without relying on insertion order`() { + Files.writeString( + directory.resolve("Entries.ts"), + """ + export class PairEntry { + key: number; + value: number; + constructor(key: number, value: number) { this.key = key; this.value = value; } + } + export class Entries { + item: PairEntry = new PairEntry(1, 2); + } + """.trimIndent() + ) + val registryFields = mapOf("key" to NumberDomain(), "value" to NumberDomain()) + val observationFields = registryFields.entries.reversed().associate { it.toPair() } + val surface = CallsClassSurface( + sourceFile = "Entries.ts", + exportName = "Entries", + constructorInputs = emptyList(), + members = emptyList(), + stateFields = mapOf("item" to ObjectDomain(fields = observationFields)), + dataClasses = listOf(CallsClassDataClass(id = "pair", exportName = "PairEntry", fields = registryFields)), + ) + val harness = prepareCallsClassHarness(directory, project(), surface, maxSequenceLength = 1) + + val replay = OriginalTypeScriptClassReplayer( + directory, + harness.functions.single(), + harness.observations.single() + ) + .use { it.replay(inputs = emptyList(), timeoutMillis = 20_000) } + + assertTrue(replay.observation.representable, replay.toString()) + val item = replay.observation.values.getValue("state_0_item") as JsConcreteValue.Object + assertEquals(number(1), item.fields["key"]) + assertEquals(number(2), item.fields["value"]) + assertEquals(JsConcreteValue.String("pair"), item.fields[CLASS_IDENTITY_FIELD]) + } + + private fun dataClassFixture( + observationFieldDomain: PropertyDomain = NumberDomain(), + fixtureSource: String = dataClassSource, + constructorInputs: List = emptyList(), + ): CallsClassPreparedHarness { + Files.writeString(directory.resolve("Entries.ts"), fixtureSource) + val entryFields = mapOf("value" to NumberDomain()) + val array = ArrayDomain( + element = ObjectDomain(fields = mapOf("value" to observationFieldDomain)), + maxLength = 4, + ) + val stateFields = mapOf("items" to array) + val surface = CallsClassSurface( + sourceFile = "Entries.ts", + exportName = "Entries", + constructorInputs = constructorInputs, + members = listOf( + CallsClassMember( + id = "read", + name = "read", + kind = CallsClassMemberKind.METHOD, + inputs = emptyList(), + resultDomain = array, + ), + CallsClassMember( + id = "self", + name = "self", + kind = CallsClassMemberKind.METHOD, + inputs = emptyList(), + resultDomain = ObjectDomain(fields = stateFields), + resultClassName = "Entries", + ), + ), + stateFields = stateFields, + dataClasses = listOf(CallsClassDataClass(id = "entry", exportName = "Entry", fields = entryFields)), + ) + + return prepareCallsClassHarness(directory, project(), surface, maxSequenceLength = 1) + } + + private val dataClassSource = """ + export class Entry { + value: number; + constructor(value: number) { this.value = value; } + } + export class Entries { + items: Entry[]; + constructor() { this.items = [new Entry(1)]; } + read(): Entry[] { return this.items; } + self(): Entries { return this; } + } + """.trimIndent() + + private fun surface() = CallsClassSurface( + sourceFile = "Entries.ts", + exportName = "Entries", + constructorInputs = emptyList(), + members = emptyList(), + stateFields = emptyMap(), + ) + + private fun project() = CallsProjectCase( + projectId = "fixture", + revision = "0".repeat(40), + sourceRoot = ".", + development = false, + functions = emptyList(), + ) + + private fun number(value: Int) = JsConcreteValue.number(value.toDouble()) +} diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassExperimentTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassExperimentTest.kt new file mode 100644 index 0000000000..aabe0bbc0a --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassExperimentTest.kt @@ -0,0 +1,289 @@ +package org.usvm.ts.calls + +import org.usvm.ts.pbt.fastcheck.TypeScriptClassStatementDescriptor +import org.usvm.ts.pbt.model.JsConcreteValue +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class CallsClassExperimentTest { + @Test + fun `class treatments reject sixty seconds and duplicate paired seeds`() { + val manifest = manifest() + + assertEquals(listOf(30_000L, 120_000L), manifest.budgetsMillis) + assertFailsWith { manifest.copy(budgetsMillis = listOf(30_000L, 60_000L, 120_000L)) } + assertFailsWith { manifest.copy(seeds = listOf(17, 17)) } + } + + @Test + fun `native primary coverage survives strict symbolic rejection and excludes late terminal candidates`() { + val hash = "a".repeat(64) + val ids = (0..2).map { "$hash:$it" } + val universe = CallsClassSourceUniverse( + sourceFile = "Counter.ts", + className = "Counter", + sourceSha256 = hash, + universeSha256 = "b".repeat(64), + instrumenterVersion = "test", + statements = ids.mapIndexed { index, id -> statement(id, index) }, + ) + val candidates = listOf( + candidate(emittedAtMillis = 20, scenarioId = "first"), + candidate(emittedAtMillis = 30, scenarioId = "second"), + candidate(emittedAtMillis = 40, scenarioId = "second"), + candidate(emittedAtMillis = 30_001, scenarioId = "late"), + ) + val events = listOf( + replay(index = 0, scenarioId = "first", covered = setOf(ids[0]), accepted = true), + replay(index = 1, scenarioId = "second", covered = setOf(ids[0], ids[1]), accepted = true), + replay(index = 2, scenarioId = "second", covered = setOf(ids[2]), accepted = false), + replay(index = 3, scenarioId = "late", covered = setOf(ids[2]), accepted = true), + ) + + val checkpoint = callsClassCoverageCheckpoints( + search = search(candidates), + events = events, + universe = universe, + budgetMillis = 30_000, + ).last() + + assertEquals(ids.toSet(), checkpoint.coveredStatementIds) + assertEquals(emptySet(), checkpoint.uncoveredStatementIds) + assertEquals(3, checkpoint.totalStatements) + assertEquals(3, checkpoint.acceptedCandidates) + assertEquals(0, checkpoint.rejectedCandidates) + assertEquals(setOf(ids[0], ids[1]), checkpoint.strictCoveredStatementIds) + assertEquals(2, checkpoint.strictAcceptedCandidates) + assertEquals(1, checkpoint.strictRejectedCandidates) + assertTrue(checkpoint.strictCoveredStatementIds.size < checkpoint.totalStatements) + } + + @Test + fun `reached prefix uses real search time and post stop extraction appears only in final checkpoint`() { + val universe = universe() + val id = universe.statementIds.single() + val prefix = candidate(emittedAtMillis = 20, scenarioId = "prefix").copy( + completion = null, + origin = CallsCoverageCandidateOrigin.SUSPENDED, + suspensionReason = "Unsupported native call", + ) + val frontier = prefix.copy(entryPointId = "frontier", emittedAtMillis = 30_050, suspendedAtMachineStop = true) + val lateTerminal = candidate(emittedAtMillis = 30_051, scenarioId = "late") + val latePrefix = prefix.copy(entryPointId = "late-prefix", emittedAtMillis = 30_052) + val candidates = listOf(prefix, frontier, lateTerminal, latePrefix) + val events = candidates.mapIndexed { index, candidate -> + replay(index, requireNotNull(candidate.entryPointId), setOf(id), accepted = false).copy( + origin = candidate.origin, + strictOracleApplicable = candidate.origin == CallsCoverageCandidateOrigin.TERMINATED, + ) + } + + val checkpoints = callsClassCoverageCheckpoints(search(candidates), events, universe, budgetMillis = 30_000) + + assertEquals(30_050, candidates[1].emittedAtMillis) + assertEquals(1, checkpoints.first().emittedCandidates) + assertEquals(0, checkpoints.first().postStopSuspendedCandidates) + assertFalse(checkpoints.first().includesSuspendedExtraction) + assertEquals(2, checkpoints.last().emittedCandidates) + assertEquals(2, checkpoints.last().acceptedCandidates) + assertEquals(2, checkpoints.last().strictOracleNotApplicable) + assertEquals(1, checkpoints.last().postStopSuspendedCandidates) + assertTrue(checkpoints.last().includesSuspendedExtraction) + } + + @Test + fun `primary coverage rejects transport mismatch foreign hits and incomplete native observations`() { + val universe = universe() + val observation = CallsClassReplayObservation( + inputs = emptyList(), + observation = CallsClassObservation(completion = CallsCoverageCompletion.RETURNED), + coveredStatementKeys = universe.statementIds, + ) + val incomplete = observation.copy(observation = observation.observation.copy(representable = false)) + val mismatch = observation.copy( + observation = observation.observation.copy(completion = CallsCoverageCompletion.THREW), + ) + val foreign = observation.copy(coveredStatementKeys = setOf("foreign")) + val changedInputs = observation.copy(inputs = listOf(JsConcreteValue.Boolean(value = true))) + + assertTrue(callsClassNativeCoverageAcceptance(observation, observation, universe).accepted) + assertFalse(callsClassNativeCoverageAcceptance(observation, mismatch, universe).accepted) + assertFalse(callsClassNativeCoverageAcceptance(observation, foreign, universe).accepted) + assertFalse(callsClassNativeCoverageAcceptance(observation, changedInputs, universe).accepted) + assertFalse(callsClassNativeCoverageAcceptance(incomplete, incomplete, universe).accepted) + assertFalse(callsClassNativeCoverageAcceptance(observation, observation, universe, error = "timeout").accepted) + assertFalse(callsClassNativeCoverageAcceptance(null, observation, universe).accepted) + } + + @Test + fun `suspended candidates reject invented terminal observations and old measurement schema`() { + val terminal = candidate(emittedAtMillis = 1, scenarioId = "constructor") + + assertFailsWith { terminal.copy(origin = CallsCoverageCandidateOrigin.SUSPENDED) } + assertFailsWith { + terminal.copy( + completion = null, + origin = CallsCoverageCandidateOrigin.SUSPENDED, + observation = CallsClassObservation(completion = CallsCoverageCompletion.RETURNED), + ) + } + assertFailsWith { manifest().copy(schemaVersion = 1) } + } + + @Test + fun `resume recomputes native and strict decisions and refuses changed candidate origin`() { + val universe = universe() + val observation = CallsClassObservation(completion = CallsCoverageCompletion.RETURNED) + val candidate = candidate(emittedAtMillis = 1, scenarioId = "constructor").copy(observation = observation) + val event = replay(index = 0, scenarioId = "constructor", covered = universe.statementIds, accepted = true) + + validateCallsClassReplayEvent(candidate, index = 0, event = event, universe = universe) + + assertFailsWith { + validateCallsClassReplayEvent( + candidate, + index = 0, + event = event.copy(nativeCoverage = CallsClassOracleResult(accepted = false)), + universe = universe, + ) + } + assertFailsWith { + validateCallsClassReplayEvent( + candidate, + index = 0, + event = event.copy(oracle = CallsClassOracleResult(accepted = false)), + universe = universe, + ) + } + assertFailsWith { + validateCallsClassReplayEvent( + candidate, + index = 0, + event = event.copy(origin = CallsCoverageCandidateOrigin.SUSPENDED), + universe = universe, + ) + } + } + + @Test + fun `class cell refuses a truncated replay journal`() { + val hash = "a".repeat(64) + val universe = CallsClassSourceUniverse( + sourceFile = "Counter.ts", + className = "Counter", + sourceSha256 = hash, + universeSha256 = hash, + instrumenterVersion = "test", + statements = listOf(statement("$hash:0", index = 0)), + ) + + assertFailsWith { + callsClassCoverageCheckpoints( + search = search(listOf(candidate(emittedAtMillis = 1, scenarioId = "constructor"))), + events = emptyList(), + universe = universe, + budgetMillis = 30_000, + ) + } + } + + private fun manifest(): CallsClassExperimentManifest = CallsClassExperimentManifest( + experimentId = "class-pilot", + toolRevision = "a".repeat(40), + nativeFrontendRevision = "bundled:test", + modelSet = CallsModelSetIdentity(ids = emptySet(), toolRevision = "a".repeat(40)), + seeds = listOf(17), + classes = listOf( + CallsClassCase( + classId = "counter", + projectId = "fixture", + revision = "c".repeat(40), + sourceRoot = "fixture", + sourceSha256 = mapOf("Counter.ts" to "a".repeat(64)), + surface = CallsClassSurface( + sourceFile = "Counter.ts", + exportName = "Counter", + constructorInputs = emptyList(), + members = emptyList(), + stateFields = emptyMap(), + ), + ), + ), + ) + + private fun candidate(emittedAtMillis: Long, scenarioId: String): CallsCoverageCandidate = CallsCoverageCandidate( + emittedAtMillis = emittedAtMillis, + emittedAtStep = 1, + inputs = emptyList(), + newSymbolicStatements = 1, + completion = CallsCoverageCompletion.RETURNED, + entryPointId = scenarioId, + ) + + private fun replay( + index: Int, + scenarioId: String, + covered: Set, + accepted: Boolean, + ): CallsClassReplayEvent = CallsClassReplayEvent( + index = index, + scenarioId = scenarioId, + original = CallsClassReplayObservation( + inputs = emptyList(), + observation = CallsClassObservation(completion = CallsCoverageCompletion.RETURNED), + ), + instrumented = CallsClassReplayObservation( + inputs = emptyList(), + observation = CallsClassObservation(completion = CallsCoverageCompletion.RETURNED), + coveredStatementKeys = covered, + ), + oracle = CallsClassOracleResult(accepted = accepted), + nativeCoverage = CallsClassOracleResult(accepted = true), + ) + + private fun universe(): CallsClassSourceUniverse { + val hash = "a".repeat(64) + return CallsClassSourceUniverse( + sourceFile = "Counter.ts", + className = "Counter", + sourceSha256 = hash, + universeSha256 = hash, + instrumenterVersion = "test", + statements = listOf(statement("$hash:0", index = 0)), + ) + } + + private fun search(candidates: List): CallsClassSearchResult = CallsClassSearchResult( + search = CallsCoverageSearchResult( + status = CallsCoverageSearchStatus.EXHAUSTED, + candidates = candidates, + selectedStates = candidates.size, + executedSteps = candidates.size.toLong(), + stepsWithinBudget = candidates.size.toLong(), + extractionFailures = emptyList(), + candidateCapReached = false, + preparationElapsedMillis = 0, + machineSetupElapsedMillis = 0, + searchElapsedMillis = 0, + machineTeardownElapsedMillis = 0, + ), + scenarios = emptyList(), + visits = emptyList(), + unknownCallEvents = 0, + ) + + private fun statement(id: String, index: Int) = TypeScriptClassStatementDescriptor( + id = id, + memberName = "increment", + kind = "ExpressionStatement", + startOffset = index, + endOffset = index + 1, + startLine = 1, + startColumn = index, + endLine = 1, + endColumn = index + 1, + ) +} diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassHarnessOracleTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassHarnessOracleTest.kt new file mode 100644 index 0000000000..11abffc97c --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassHarnessOracleTest.kt @@ -0,0 +1,473 @@ +package org.usvm.ts.calls + +import org.junit.jupiter.api.io.TempDir +import org.usvm.SolverType +import org.usvm.ts.pbt.model.ArrayDomain +import org.usvm.ts.pbt.model.ConstantDomain +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.NumberDomain +import org.usvm.ts.pbt.model.ObjectDomain +import org.usvm.ts.pbt.model.OptionalDomain +import org.usvm.ts.pbt.model.PropertyInput +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +class CallsClassHarnessOracleTest { + @TempDir + lateinit var directory: Path + + @Test + fun `scenario schedule freezes constructor every member and every ordered pair`() { + val harness = fixture() + + assertEquals(expected = 21, actual = harness.scenarios.size) + assertEquals(emptyList(), harness.scenarios.first().memberIds) + assertEquals( + listOf("add", "read", "write", "arrow"), + harness.scenarios.drop(1).take(4).map { it.memberIds.single() } + ) + assertTrue(harness.scenarios.any { it.memberIds == listOf("write", "read") }) + assertTrue(harness.scenarios.any { it.memberIds == listOf("add", "add") }) + assertTrue(harness.functions.all { it.symbolicCoverageClass?.className == "Counter" }) + assertTrue(harness.functions[1].inputs[1].domain is ObjectDomain) + } + + @Test + fun `same receiver observes setter mutation before getter and resets between replays`() { + val harness = fixture() + val index = harness.scenarios.indexOfFirst { it.memberIds == listOf("write", "read") } + OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[index], + observationFields = harness.observations[index], + ).use { replayer -> + val first = replayer.replay(inputs = listOf(number(2), number(9)), timeoutMillis = 20_000) + val second = replayer.replay(inputs = listOf(number(3), number(4)), timeoutMillis = 20_000) + + assertTrue(first.observation.representable, first.observation.diagnostic) + assertEquals(number(2), first.observation.values["state_0_count"]) + assertEquals(number(9), first.observation.values["state_1_count"]) + assertEquals(number(9), first.observation.values["result_1"]) + assertEquals(JsConcreteValue.Undefined, first.observation.values["result_0"]) + assertEquals(number(3), second.observation.values["state_0_count"]) + assertEquals(number(4), second.observation.values["result_1"]) + } + } + + @Test + fun `object parameter transport stays frozen when callable mutates it`() { + val harness = fixture() + val index = harness.scenarios.indexOfFirst { it.memberIds == listOf("add", "read") } + val options = JsConcreteValue.Object(mapOf("amount" to number(3))) + val inputs = listOf(number(2), options) + + val replay = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[index], + observationFields = harness.observations[index], + ).use { it.replay(inputs = inputs, timeoutMillis = 20_000) } + + assertEquals(inputs, replay.inputs) + assertTrue(replay.observation.representable, replay.observation.diagnostic) + assertEquals(number(5), replay.observation.values["result_0"]) + assertEquals(number(5), replay.observation.values["result_1"]) + } + + @Test + fun `exact counters come only from equivalent instrumented scenario execution`() { + val harness = fixture() + val index = harness.scenarios.indexOfFirst { it.memberIds == listOf("arrow") } + val inputs = listOf(number(2), number(5)) + val source = Files.readString(directory.resolve("Counter.ts")) + val instrumented = source.replace( + "return this.count * x;", + "(globalThis as any).classHits?.add('arrow-body'); return this.count * x;" + ) + val original = OriginalTypeScriptClassReplayer( + directory, + harness.functions[index], + harness.observations[index] + ).use { + it.replay(inputs = inputs, timeoutMillis = 20_000) + } + val counted = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[index], + observationFields = harness.observations[index], + sourceOverride = instrumented, + overrideSourceFile = "Counter.ts", + counterGlobalName = "classHits", + ).use { it.replay(inputs = inputs, timeoutMillis = 20_000) } + + assertEquals(setOf("arrow-body"), counted.coveredStatementKeys) + assertTrue( + compareCallsClassObservations(original.observation, original.observation, counted.observation).accepted + ) + } + + @Test + fun `returned values and receiver state are independently checked`() { + val original = CallsClassObservation( + completion = CallsCoverageCompletion.RETURNED, + values = mapOf("result_0" to number(1), "state_1_count" to number(2)), + ) + val wrongResult = original.copy(values = original.values + ("result_0" to number(0))) + val wrongState = original.copy(values = original.values + ("state_1_count" to number(0))) + + assertFalse(compareCallsClassObservations(wrongResult, original, original).accepted) + assertFalse(compareCallsClassObservations(wrongState, original, original).accepted) + assertFalse(compareCallsClassObservations(original, original, wrongState).accepted) + assertTrue(compareCallsClassObservations(original, original, original).accepted) + } + + @Test + fun `matching unavailable outcomes are never equivalent`() { + val unavailable = CallsClassObservation( + completion = CallsCoverageCompletion.THREW, + representable = false, + diagnostic = "Incomplete thrown receiver state", + ) + + assertFalse(compareCallsClassObservations(unavailable, unavailable, unavailable).accepted) + assertFalse(compareCallsClassObservations(null, unavailable, unavailable).accepted) + } + + @Test + fun `invalid getter and sequence bound reject before execution`() { + assertFailsWith { + surface().copy( + members = listOf( + CallsClassMember( + id = "bad", + name = "bad", + kind = CallsClassMemberKind.GETTER, + inputs = listOf(PropertyInput("x", IntegerDomain())), + resultDomain = NumberDomain(), + ) + ) + ) + } + Files.writeString(directory.resolve("Counter.ts"), source) + + assertFailsWith { + prepareCallsClassHarness(directory, project(), surface(), maxSequenceLength = 3) + } + } + + @Test + fun `array result is snapshotted before subsequent mutation`() { + Files.writeString( + directory.resolve("Bag.ts"), + """ + export class Bag { + items: number[] = [1]; + take(): number[] { return this.items; } + replace(x: number): number { this.items[0] = x; return x; } + } + """.trimIndent() + ) + val arrayDomain = ArrayDomain(element = NumberDomain(), maxLength = 4) + val surface = CallsClassSurface( + sourceFile = "Bag.ts", + exportName = "Bag", + constructorInputs = emptyList(), + members = listOf( + CallsClassMember("take", "take", CallsClassMemberKind.METHOD, emptyList(), arrayDomain), + CallsClassMember( + "replace", + "replace", + CallsClassMemberKind.METHOD, + listOf(PropertyInput("x", NumberDomain())), + NumberDomain() + ), + ), + stateFields = mapOf("items" to arrayDomain), + ) + val harness = prepareCallsClassHarness(directory, project(), surface) + val index = harness.scenarios.indexOfFirst { it.memberIds == listOf("take", "replace") } + + val replay = OriginalTypeScriptClassReplayer( + directory, + harness.functions[index], + harness.observations[index] + ).use { + it.replay(inputs = listOf(number(9)), timeoutMillis = 20_000) + } + + assertTrue(replay.observation.representable, replay.observation.diagnostic) + assertEquals(JsConcreteValue.Array(listOf(number(1))), replay.observation.values["result_0"]) + assertEquals(JsConcreteValue.Array(listOf(number(1))), replay.observation.values["state_1_items"]) + assertEquals(JsConcreteValue.Array(listOf(number(9))), replay.observation.values["state_2_items"]) + } + + @Test + fun `symbolic constructed state and return snapshots agree with original replay`() { + Files.writeString( + directory.resolve("Scalar.ts"), + """ + export class Scalar { + count: number; + constructor(start: number) { this.count = start; } + add(amount: number): number { this.count += amount; return this.count; } + read(): number { return this.count; } + } + """.trimIndent() + ) + val surface = CallsClassSurface( + sourceFile = "Scalar.ts", + exportName = "Scalar", + constructorInputs = listOf(PropertyInput("start", IntegerDomain(min = 0, max = 4))), + members = listOf( + CallsClassMember( + "add", + "add", + CallsClassMemberKind.METHOD, + listOf(PropertyInput("amount", IntegerDomain(min = 0, max = 4))), + NumberDomain() + ), + CallsClassMember("read", "read", CallsClassMemberKind.METHOD, emptyList(), NumberDomain()), + ), + stateFields = mapOf("count" to NumberDomain()), + ) + val harness = prepareCallsClassHarness(directory, project(), surface, maxSequenceLength = 2) + + val result = CurrentTsCallsClassCoverageEngine().search( + CallsClassSearchRequest( + harness = harness, + profile = CallsExperimentProfile.EMPTY_STOP, + frozenModelIds = emptySet(), + expectedNativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision, + seed = 1, + budget = 2.seconds, + timeQuantum = 100.milliseconds, + solverQueryLimit = 1.seconds, + solverType = SolverType.YICES, + candidateCap = 20, + requireEntryPointWitness = true, + ) + ) + + assertTrue(result.search.candidates.isNotEmpty(), result.toString()) + assertTrue((result.solverCalls ?: 0) > 0, result.toString()) + assertTrue(result.solverQueries.isNotEmpty(), result.toString()) + assertTrue(result.timeQuanta.isNotEmpty(), result.toString()) + assertTrue( + result.timeQuanta.all { it.requestedQuantumMillis <= 100 && it.elapsedNanos >= 0 }, + result.toString() + ) + assertTrue(result.search.executedSteps > 0, result.toString()) + assertTrue(result.search.searchElapsedMillis <= 3_000, result.toString()) + assertEquals( + harness.scenarios.map { it.id }.toSet(), + result.search.candidates.map { it.entryPointId }.toSet(), + result.toString() + ) + for (candidate in result.search.candidates) { + val index = harness.scenarios.indexOfFirst { it.id == candidate.entryPointId } + val original = OriginalTypeScriptClassReplayer( + directory, + harness.functions[index], + harness.observations[index] + ).use { + it.replay(candidate.inputs, timeoutMillis = 20_000) + } + val checked = + compareCallsClassObservations(candidate.observation, original.observation, original.observation) + assertTrue(checked.accepted, "$candidate / $original / $checked") + } + } + + @Test + fun `optional array and nested object snapshots guard nullish values before copying`() { + Files.writeString( + directory.resolve("OptionalBag.ts"), + """ + export class OptionalBag { + items: number[] | undefined = undefined; + object: { items: number[] } | null = null; + read(): number[] | undefined { return this.items; } + } + """.trimIndent() + ) + val array = ArrayDomain(element = NumberDomain(), maxLength = 4) + val optionalArray = OptionalDomain(value = array, nil = JsConcreteValue.Undefined) + val surface = CallsClassSurface( + sourceFile = "OptionalBag.ts", + exportName = "OptionalBag", + constructorInputs = emptyList(), + members = listOf(CallsClassMember("read", "read", CallsClassMemberKind.METHOD, emptyList(), optionalArray)), + stateFields = mapOf( + "items" to optionalArray, + "object" to OptionalDomain(value = ObjectDomain(mapOf("items" to array)), nil = JsConcreteValue.Null), + ), + ) + val harness = prepareCallsClassHarness(directory, project(), surface, maxSequenceLength = 1) + + val replay = OriginalTypeScriptClassReplayer(directory, harness.functions[1], harness.observations[1]).use { + it.replay(inputs = emptyList(), timeoutMillis = 20_000) + } + + assertTrue(replay.observation.representable, replay.observation.diagnostic) + assertEquals(JsConcreteValue.Undefined, replay.observation.values["result_0"]) + assertEquals(JsConcreteValue.Null, replay.observation.values["state_1_object"]) + } + + @Test + fun `raw output and receiver shape checks reject fields hidden by projection`() { + Files.writeString( + directory.resolve("Hidden.ts"), + """ + export class Hidden { + count: number = 1; + result(): { kept: number; omitted: number } { return { kept: 1, omitted: 2 }; } + mutate(): number { (this as any).extra = 3; return this.count; } + mutateCallback(): number { (this as any).extraCallback = () => 3; return this.count; } + } + """.trimIndent() + ) + val surface = CallsClassSurface( + sourceFile = "Hidden.ts", + exportName = "Hidden", + constructorInputs = emptyList(), + members = listOf( + CallsClassMember( + "result", + "result", + CallsClassMemberKind.METHOD, + emptyList(), + ObjectDomain(mapOf("kept" to NumberDomain())) + ), + CallsClassMember("mutate", "mutate", CallsClassMemberKind.METHOD, emptyList(), NumberDomain()), + CallsClassMember( + id = "mutateCallback", + name = "mutateCallback", + kind = CallsClassMemberKind.METHOD, + inputs = emptyList(), + resultDomain = NumberDomain(), + ), + ), + stateFields = mapOf("count" to NumberDomain()), + ) + val harness = prepareCallsClassHarness(directory, project(), surface, maxSequenceLength = 1) + + for (index in 1..3) { + val replay = OriginalTypeScriptClassReplayer( + directory, + harness.functions[index], + harness.observations[index] + ).use { + it.replay(inputs = emptyList(), timeoutMillis = 20_000) + } + + assertFalse(replay.observation.representable) + assertTrue( + replay.observation.diagnostic?.contains("Unlisted or missing object property") == true, + replay.toString() + ) + } + } + + @Test + fun `native empty object shape rejects builtin internal slot values`() { + Files.writeString( + directory.resolve("Builtin.ts"), + """ + export class Builtin { + result(): object { return new Date(1234); } + } + """.trimIndent(), + ) + val surface = CallsClassSurface( + sourceFile = "Builtin.ts", + exportName = "Builtin", + constructorInputs = emptyList(), + members = listOf( + CallsClassMember( + id = "result", + name = "result", + kind = CallsClassMemberKind.METHOD, + inputs = emptyList(), + resultDomain = ObjectDomain(fields = emptyMap()), + ), + ), + stateFields = emptyMap(), + ) + val harness = prepareCallsClassHarness(directory, project(), surface, maxSequenceLength = 1) + + val replay = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[1], + observationFields = harness.observations[1], + ).use { it.replay(inputs = emptyList(), timeoutMillis = 20_000) } + + assertFalse(replay.observation.representable) + assertTrue(replay.observation.diagnostic?.contains("Unlisted object prototype") == true) + } + + private fun fixture(): CallsClassPreparedHarness { + Files.writeString(directory.resolve("Counter.ts"), source) + return prepareCallsClassHarness(directory, project(), surface()) + } + + private fun surface(): CallsClassSurface = CallsClassSurface( + sourceFile = "Counter.ts", + exportName = "Counter", + constructorInputs = listOf(PropertyInput("start", IntegerDomain(min = 0, max = 10))), + members = listOf( + CallsClassMember( + "add", + "add", + CallsClassMemberKind.METHOD, + listOf(PropertyInput("options", ObjectDomain(mapOf("amount" to IntegerDomain(min = 0, max = 10))))), + NumberDomain() + ), + CallsClassMember("read", "value", CallsClassMemberKind.GETTER, emptyList(), NumberDomain()), + CallsClassMember( + "write", + "value", + CallsClassMemberKind.SETTER, + listOf(PropertyInput("value", IntegerDomain(min = 0, max = 10))), + ConstantDomain(JsConcreteValue.Undefined) + ), + CallsClassMember( + "arrow", + "multiply", + CallsClassMemberKind.FUNCTION_FIELD, + listOf(PropertyInput("x", IntegerDomain(min = 0, max = 10))), + NumberDomain() + ), + ), + stateFields = mapOf("count" to NumberDomain()), + ) + + private fun project() = CallsProjectCase( + "fixture", + "0".repeat(40), + ".", + development = false, + functions = emptyList() + ) + + private fun number(value: Int) = JsConcreteValue.number(value.toDouble()) + + private val source = """ + export class Counter { + count: number; + constructor(start: number) { this.count = start; } + add(options: { amount: number }): number { + this.count += options.amount; options.amount = 99; return this.count; + } + get value(): number { return this.count; } + set value(value: number) { this.count = value; } + multiply = (x: number): number => { return this.count * x; }; + } + """.trimIndent() +} diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassNativeDiagnosticTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassNativeDiagnosticTest.kt new file mode 100644 index 0000000000..fdaffa9a0d --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassNativeDiagnosticTest.kt @@ -0,0 +1,120 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.encodeToString +import org.junit.jupiter.api.io.TempDir +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.NumberDomain +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class CallsClassNativeDiagnosticTest { + @TempDir + lateinit var directory: Path + + @Test + fun `native numeric throws retain the actual primitive value`() { + val harness = fixture(constructorBody = "throw 7;") + + val replay = replay(harness) + + assertEquals(CallsCoverageCompletion.THREW, replay.observation.completion) + assertFalse(replay.observation.representable) + val expectedValue = CallsExperimentJson.json.encodeToString(JsConcreteValue.number(7.0)) + assertEquals("$thrownPrefix; thrown value=$expectedValue", replay.observation.diagnostic) + } + + @Test + fun `native string throws retain the actual string value`() { + val harness = fixture(constructorBody = "throw 'actual thrown string';") + + val replay = replay(harness) + + assertEquals(CallsCoverageCompletion.THREW, replay.observation.completion) + assertFalse(replay.observation.representable) + val expectedValue = CallsExperimentJson.json.encodeToString( + JsConcreteValue.String("actual thrown string"), + ) + assertEquals("$thrownPrefix; thrown value=$expectedValue", replay.observation.diagnostic) + } + + @Test + fun `native errors retain the actual name and message`() { + val harness = fixture(constructorBody = "throw new TypeError('actual native message');") + + val replay = replay(harness) + + assertEquals(CallsCoverageCompletion.THREW, replay.observation.completion) + assertFalse(replay.observation.representable) + assertEquals( + "$thrownPrefix; exception name=\"TypeError\", message=\"actual native message\"", + replay.observation.diagnostic, + ) + } + + @Test + fun `repeated loop coverage preserves native snapshots and contains only unique statement keys`() { + val harness = fixture(constructorBody = "for (let i = 0; i < 20000; i++) { this.count++; }") + val instrumentation = CallsClassSourceInstrumentation.create( + sourceRoot = directory, + sourceFile = "NativeOutcome.ts", + className = "NativeOutcome", + ) + + val original = replay(harness) + val instrumented = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions.single(), + observationFields = harness.observations.single(), + sourceOverride = instrumentation.instrumentedSource, + overrideSourceFile = "NativeOutcome.ts", + counterGlobalName = instrumentation.hitGlobalName, + ).use { it.replay(inputs = emptyList(), timeoutMillis = 20_000) } + + assertTrue(original.observation.representable, original.observation.diagnostic) + assertEquals(JsConcreteValue.number(20_000.0), original.observation.values["state_0_count"]) + assertEquals(original.observation, instrumented.observation) + val coveredIds = instrumentation.universe.checkedCoveredIds(instrumented.coveredStatementKeys) + assertEquals(instrumentation.universe.statements.size, coveredIds.size) + } + + private fun fixture(constructorBody: String): CallsClassPreparedHarness { + Files.writeString( + directory.resolve("NativeOutcome.ts"), + """ + export class NativeOutcome { + count: number = 0; + constructor() { $constructorBody } + } + """.trimIndent(), + ) + val surface = CallsClassSurface( + sourceFile = "NativeOutcome.ts", + exportName = "NativeOutcome", + constructorInputs = emptyList(), + members = emptyList(), + stateFields = mapOf("count" to NumberDomain()), + ) + val project = CallsProjectCase( + projectId = "native-diagnostic-fixture", + revision = "0".repeat(40), + sourceRoot = ".", + development = true, + functions = emptyList(), + ) + + return prepareCallsClassHarness(directory, project, surface, maxSequenceLength = 2) + } + + private fun replay(harness: CallsClassPreparedHarness): CallsClassReplayObservation = + OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions.single(), + observationFields = harness.observations.single(), + ).use { it.replay(inputs = emptyList(), timeoutMillis = 20_000) } + + private val thrownPrefix = "Thrown class scenarios have no complete return/state snapshot" +} diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassUnknownCallJournalTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassUnknownCallJournalTest.kt new file mode 100644 index 0000000000..30e56c2ea1 --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassUnknownCallJournalTest.kt @@ -0,0 +1,323 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject +import org.junit.jupiter.api.io.TempDir +import org.usvm.CoverageZone +import org.usvm.SolverType +import org.usvm.UMachineOptions +import org.usvm.machine.TsMachine +import org.usvm.machine.TsOptions +import org.usvm.machine.state.TsMethodResult +import org.usvm.machine.state.TsState +import org.usvm.statistics.UMachineObserver +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.NumberDomain +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.model.contains +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeSource + +class CallsClassUnknownCallJournalTest { + @TempDir + lateinit var directory: Path + + @Test + fun `actual repeated source calls retain ordered model and residual decisions in serialized search`() { + val harness = prepareFixture( + source = """ + export class Telemetry { + value: number = 0; + constructor(input: number) { + for (let index = 0; index < 2; index++) { + this.value = Math.floor(input); + } + } + } + """.trimIndent(), + inputs = listOf(PropertyInput(name = "input", domain = IntegerDomain(min = 0, max = 3))), + stateFields = mapOf("value" to NumberDomain()), + ) + + val modeled = search(harness, profile = CallsExperimentProfile.FROZEN_FRESH, models = setOf("ts.math.floor")) + val residual = search(harness, profile = CallsExperimentProfile.EMPTY_FRESH, models = emptySet()) + + assertTrue(modeled.search.candidates.isNotEmpty(), modeled.toString()) + assertTrue(residual.search.candidates.isNotEmpty(), residual.toString()) + for (result in listOf(modeled, residual)) { + assertTrue(result.unknownCalls.size >= 2, result.toString()) + assertEquals(result.unknownCallEvents, result.unknownCalls.size) + assertEquals((1..result.unknownCalls.size).toList(), result.unknownCalls.map { it.eventIndex }) + val sites = result.unknownCalls.groupingBy { it.callSite }.eachCount() + assertTrue(sites.values.any { it >= 2 }, result.toString()) + assertTrue(result.unknownCalls.all { it.callSite.sourcePath.endsWith("Telemetry.ts") }, result.toString()) + assertTrue(result.unknownCalls.all { it.callerMethodSignature.contains("Telemetry::constructor") }) + assertTrue(result.unknownCalls.all { it.callee.name == "floor" }) + assertTrue(result.unknownCalls.all { it.failureReason == "PARTIAL_APPROXIMATION" }) + assertNotNull(result.unknownCalls.first().callSite.startOffset) + + val encoded = CallsExperimentJson.json.encodeToString(result) + assertEquals(result, CallsExperimentJson.json.decodeFromString(encoded)) + val legacy = JsonObject( + CallsExperimentJson.json.parseToJsonElement(encoded).jsonObject.filterKeys { it != "unknownCalls" }, + ) + val decodedLegacy = CallsExperimentJson.json.decodeFromString(legacy.toString()) + assertEquals(emptyList(), decodedLegacy.unknownCalls) + assertEquals(result.unknownCallEvents, decodedLegacy.unknownCallEvents) + } + for (event in modeled.unknownCalls) { + assertEquals(CallsUnknownCallDecisionKind.MODEL_APPLIED, event.decision) + assertEquals("MODEL_APPLIED", event.outcome) + assertEquals("ts.math.floor", event.modelId) + assertNull(event.residualPolicy) + } + for (event in residual.unknownCalls) { + assertEquals(CallsUnknownCallDecisionKind.RESIDUAL_FALLBACK, event.decision) + assertEquals("FRESH_SYMBOLIC_RETURN", event.outcome) + assertNull(event.modelId) + assertEquals("FRESH_SYMBOLIC_RETURN", event.residualPolicy) + } + } + + @Test + fun `stop path harvests reached prefix inputs without pretending that constructor returned`() { + val domain = IntegerDomain(min = 0, max = 3) + val harness = prepareFixture( + source = """ + export class Telemetry { + value: number = 0; + constructor(input: number) { + this.value = Math.floor(input); + } + } + """.trimIndent(), + inputs = listOf(PropertyInput(name = "input", domain = domain)), + stateFields = mapOf("value" to NumberDomain()), + ) + + val result = search(harness, profile = CallsExperimentProfile.EMPTY_STOP, models = emptySet()) + + val candidate = result.search.candidates.single() + assertEquals(CallsCoverageCandidateOrigin.SUSPENDED, candidate.origin) + assertNull(candidate.completion) + assertNull(candidate.observation) + assertFalse(candidate.suspendedAtMachineStop) + assertNotNull(candidate.suspensionReason) + assertTrue(candidate.inputs.single() in domain) + assertTrue(result.search.extractionFailures.isEmpty(), result.toString()) + assertEquals(expected = "PATH_STOPPED", actual = result.unknownCalls.single().outcome) + + val instrumentation = CallsClassSourceInstrumentation.create( + sourceRoot = directory, + sourceFile = "Telemetry.ts", + className = "Telemetry", + ) + val original = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions.single(), + observationFields = harness.observations.single(), + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = 20_000) } + val instrumented = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions.single(), + observationFields = harness.observations.single(), + sourceOverride = instrumentation.instrumentedSource, + overrideSourceFile = "Telemetry.ts", + counterGlobalName = instrumentation.hitGlobalName, + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = 20_000) } + + assertTrue(callsClassNativeCoverageAcceptance(original, instrumented, instrumentation.universe).accepted) + assertTrue(instrumented.coveredStatementKeys.isNotEmpty()) + val strictOracle = compareCallsClassObservations( + symbolic = candidate.observation, + original = original.observation, + instrumented = instrumented.observation, + ) + assertFalse(strictOracle.accepted) + } + + @Test + fun `prefix snapshot does not hide completed novelty or strict observation for the same inputs`() { + val harness = prepareFixture( + source = """ + export class Telemetry { + value: number = 0; + constructor() { this.value = 1; } + } + """.trimIndent(), + inputs = emptyList(), + stateFields = mapOf("value" to NumberDomain()), + ) + + val candidates = terminalAfterPrefix(harness) + + assertEquals( + listOf(CallsCoverageCandidateOrigin.SUSPENDED, CallsCoverageCandidateOrigin.TERMINATED), + candidates.map { it.origin }, + ) + assertEquals(candidates[0].inputs, candidates[1].inputs) + assertEquals(expected = 0, actual = candidates[0].newSymbolicStatements) + assertNull(candidates[0].completion) + assertNull(candidates[0].observation) + assertTrue(candidates[1].newSymbolicStatements > 0) + assertEquals(CallsCoverageCompletion.RETURNED, candidates[1].completion) + assertTrue(assertNotNull(candidates[1].observation).representable) + } + + @Test + fun `symbolic thrown diagnostic retains actual exception type and expression`() { + val harness = prepareFixture( + source = """ + export class Telemetry { + constructor() { throw 7; } + } + """.trimIndent(), + inputs = emptyList(), + stateFields = emptyMap(), + ) + + val result = search(harness, profile = CallsExperimentProfile.EMPTY_STOP, models = emptySet()) + + val observation = assertNotNull(result.search.candidates.single().observation) + assertEquals(CallsCoverageCompletion.THREW, observation.completion) + assertFalse(observation.representable) + assertEquals(emptyMap(), observation.values) + val diagnostic = assertNotNull(observation.diagnostic) + assertTrue(diagnostic.contains("exception type=number"), diagnostic) + assertTrue(diagnostic.contains("value=(fp "), diagnostic) + assertFalse(compareCallsClassObservations(observation, observation, observation).accepted) + } + + private fun prepareFixture( + source: String, + inputs: List, + stateFields: Map, + ): CallsClassPreparedHarness { + Files.writeString(directory.resolve("Telemetry.ts"), source) + val surface = CallsClassSurface( + sourceFile = "Telemetry.ts", + exportName = "Telemetry", + constructorInputs = inputs, + members = emptyList(), + stateFields = stateFields, + ) + val project = CallsProjectCase( + projectId = "fixture", + revision = "0".repeat(40), + sourceRoot = ".", + development = false, + functions = emptyList(), + ) + + return prepareCallsClassHarness(directory, project, surface, maxSequenceLength = 1) + } + + private fun terminalAfterPrefix(harness: CallsClassPreparedHarness): List { + val function = harness.functions.single() + val preparation = CurrentTsCallsSymbolicEngine().prepareFunction( + CallsFunctionPreflightRequest( + sourceRoot = directory, + project = harness.project, + function = function, + expectedNativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision, + verifySourceCheckout = false, + ), + ) as CallsFunctionPreparation.Eligible + val scope = resolveCallsSymbolicCoverageScope( + preparation, + target = CallsSymbolicCoverageClass(sourceFile = "Telemetry.ts", className = "Telemetry"), + ) as CallsSymbolicCoverageScope.Selected + val inputs = CallsSymbolicInputs( + inputs = function.inputs, + bindings = preparation.inputBindings, + lexicalEnvironment = preparation.lexicalEnvironment, + captures = preparation.captures, + sourceScene = preparation.scene, + ) + val candidates = mutableListOf() + val observer = CompletedCoverageCandidateObserver( + contexts = listOf( + CallsCoverageCandidateContext( + method = preparation.method, + inputDomains = function.inputs, + symbolicInputs = inputs, + entryPointId = harness.scenarios.single().id, + observationFields = harness.observations.single(), + ), + ), + coverageStatements = scope.statements, + candidateCap = 2, + budgetMillis = 2_000, + machineCallStarted = TimeSource.Monotonic.markNow(), + onCandidate = { candidates += it }, + onExtractionFailure = { error(it) }, + emitSuspendedCandidates = true, + ) + val prefixThenTerminal = object : UMachineObserver by observer { + override fun onStateTerminated(state: TsState, stateReachable: Boolean) { + if (stateReachable) { + val prefix = state.clone() + prefix.methodResult = TsMethodResult.NoCall + prefix.unsupportedReason = "Synthetic reached-prefix observer contract" + observer.onStateSuspended(prefix) + } + observer.onStateTerminated(state = state, stateReachable = stateReachable) + } + } + + TsMachine( + scene = inputs.scene, + options = UMachineOptions( + timeout = 2.seconds, + solverTimeout = 1.seconds, + coverageZone = CoverageZone.CLASS, + stopOnCoverage = 0, + removeCoveredEntryPoints = false, + throwExceptionOnStepFailure = true, + ), + tsOptions = TsOptions( + classCoverageTargets = setOf(requireNotNull(scope.classSignature)), + eraseTypeAssertions = true, + ), + machineObserver = prefixThenTerminal, + ).use { machine -> + machine.analyzeWithMetadata( + methods = listOf(preparation.method), + configureInitialState = { _, state -> inputs.initialize(state) }, + ) + } + + return candidates + } + + private fun search( + harness: CallsClassPreparedHarness, + profile: CallsExperimentProfile, + models: Set, + ): CallsClassSearchResult = CurrentTsCallsClassCoverageEngine().search( + CallsClassSearchRequest( + harness = harness, + profile = profile, + frozenModelIds = models, + expectedNativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision, + seed = 1, + budget = 2.seconds, + timeQuantum = 100.milliseconds, + solverQueryLimit = 1.seconds, + solverType = SolverType.YICES, + candidateCap = 10, + requireEntryPointWitness = true, + ), + ) +} diff --git a/usvm-ts-fast-check/fast-check-adapter/package.json b/usvm-ts-fast-check/fast-check-adapter/package.json index 6764b2e13b..3fcbf35f61 100644 --- a/usvm-ts-fast-check/fast-check-adapter/package.json +++ b/usvm-ts-fast-check/fast-check-adapter/package.json @@ -9,7 +9,7 @@ "build": "tsc --project tsconfig.json", "pretest": "npm run build", "test": "npm run test:compiled", - "test:compiled": "node --test dist/test/coverage-branches.test.js dist/test/entry-point.test.js dist/test/execute-property.test.js dist/test/execution-cli.test.js dist/test/js-value.test.js dist/test/observe-object.test.js dist/test/process-group-shutdown.test.js dist/test/process-supervisor.test.js dist/test/project-domain.test.js dist/test/projection-cli.test.js dist/test/source-replay.test.js dist/test/source-captures.test.js dist/test/local-source-closure.test.js" + "test:compiled": "node --test dist/test/coverage-branches.test.js dist/test/entry-point.test.js dist/test/execute-property.test.js dist/test/execution-cli.test.js dist/test/js-value.test.js dist/test/observe-object.test.js dist/test/process-group-shutdown.test.js dist/test/process-supervisor.test.js dist/test/project-domain.test.js dist/test/projection-cli.test.js dist/test/source-replay.test.js dist/test/source-captures.test.js dist/test/local-source-closure.test.js dist/test/class-coverage-source.test.js" }, "dependencies": { "@jridgewell/trace-mapping": "0.3.31", diff --git a/usvm-ts-fast-check/fast-check-adapter/src/class-coverage-source.ts b/usvm-ts-fast-check/fast-check-adapter/src/class-coverage-source.ts new file mode 100644 index 0000000000..7ea82753fb --- /dev/null +++ b/usvm-ts-fast-check/fast-check-adapter/src/class-coverage-source.ts @@ -0,0 +1,671 @@ +import { createHash } from 'node:crypto' +import ts from 'typescript' + +export interface ClassStatementDescriptor { + id: string + memberName: string + kind: string + startOffset: number + endOffset: number + startLine: number + startColumn: number + endLine: number + endColumn: number +} + +export interface ClassCallableDescriptor { + name: string + kind: 'METHOD' | 'GETTER' | 'SETTER' | 'FUNCTION_FIELD' + isStatic: boolean + parameterNames: string[] + parameterTypeTexts: (string | null)[] + resultTypeText: string | null +} + +export interface ClassStateFieldDescriptor { name: string; isEcmaPrivate: boolean; typeText: string | null } +export interface ClassDataClassDescriptor { exportName: string; fields: string[] } +export interface ClassSurfaceInventory { + constructorParameterNames: string[] + publicMembers: ClassCallableDescriptor[] + instanceDataFields: ClassStateFieldDescriptor[] + dataClasses: ClassDataClassDescriptor[] +} + +export interface ClassCoverageInstrumentation { + source: string | null + reason: string | null + sourceSha256: string + instrumentedSha256: string | null + counterGlobalName: string + instrumenterVersion: string + statements: ClassStatementDescriptor[] + inventory: ClassSurfaceInventory | null +} + +interface MemberBody { name: string; node: ts.ClassElement; body: ts.Node; publicSurface: boolean; isStatic: boolean } +interface Insertion { offset: number; text: string; order: number } +const VERSION = 'class-statements-v2' + +/** Exact original AST IDs. Native counters avoid c8's ambiguous same-line statement records. */ +export function instrumentClassCoverage(file: ts.SourceFile, className: string): ClassCoverageInstrumentation { + const sourceSha256 = hash(file.text) + const counterGlobalName = `__usvmCallsClassStatementHits_${hash(`${sourceSha256}:${className}`).slice(0, 24)}` + const base = { sourceSha256, counterGlobalName, instrumenterVersion: VERSION } + const unsupported = (reason: string): ClassCoverageInstrumentation => + ({ ...base, source: null, reason, instrumentedSha256: null, statements: [], inventory: null }) + const matches = file.statements.filter(ts.isClassDeclaration).filter((node) => node.name?.text === className) + if (matches.length !== 1) return unsupported(`Expected one top-level named class ${className}`) + const selected = matches[0]! + if (selected.heritageClauses?.some((clause) => clause.token === ts.SyntaxKind.ExtendsKeyword)) { + return unsupported('Inherited class lifecycle is outside the checked class statement subset') + } + const decorated = ts.canHaveDecorators(selected) && (ts.getDecorators(selected)?.length ?? 0) > 0 + if (decorated) return unsupported('Decorated classes are outside the checked class statement subset') + + const members: MemberBody[] = [] + const initializers: MemberBody[] = [] + for (const member of selected.members) { + const modifiers = ts.canHaveModifiers(member) ? ts.getModifiers(member) ?? [] : [] + const isStatic = modifiers.some((modifier) => modifier.kind === ts.SyntaxKind.StaticKeyword) + const memberDecorated = ts.canHaveDecorators(member) && (ts.getDecorators(member)?.length ?? 0) > 0 + if (memberDecorated) return unsupported('Decorated members are outside the checked class statement subset') + if (ts.isClassStaticBlockDeclaration(member)) return unsupported('Static initialization is not constructor coverage') + if (ts.isIndexSignatureDeclaration(member) || ts.isSemicolonClassElement(member)) continue + const name = ts.isConstructorDeclaration(member) ? 'constructor' : memberName(member) + if (name === null) return unsupported('Computed or unnamed class members are outside the checked subset') + const publicSurface = !modifiers.some((modifier) => modifier.kind === ts.SyntaxKind.PrivateKeyword + || modifier.kind === ts.SyntaxKind.ProtectedKeyword) && !name.startsWith('#') + if (ts.isMethodDeclaration(member) || ts.isGetAccessorDeclaration(member) + || ts.isSetAccessorDeclaration(member) || ts.isConstructorDeclaration(member)) { + if (member.body === undefined) continue + const asynchronous = modifiers.some((modifier) => modifier.kind === ts.SyntaxKind.AsyncKeyword) + || (ts.isMethodDeclaration(member) && member.asteriskToken !== undefined) + if (asynchronous) return unsupported('Async or generator members are outside the synchronous class subset') + members.push({ name, node: member, body: member.body, publicSurface, isStatic }) + } else if (ts.isPropertyDeclaration(member) && member.initializer !== undefined) { + const initializer = member.initializer + if (ts.isArrowFunction(initializer) || ts.isFunctionExpression(initializer)) { + const asynchronous = initializer.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.AsyncKeyword) + || (ts.isFunctionExpression(initializer) && initializer.asteriskToken !== undefined) + if (asynchronous) return unsupported('Async or generator callable properties are unsupported') + members.push({ name, node: member, body: initializer.body, publicSurface, isStatic }) + } else { + if (isStatic) return unsupported('Static property initialization is outside constructor coverage') + initializers.push({ name, node: member, body: initializer, publicSurface: true, isStatic: false }) + } + } + } + + const memberKey = (name: string, isStatic: boolean): string => `${isStatic ? 'static' : 'instance'}:${name}` + const callableMemberKeys = new Set(selected.members.flatMap((member) => { + const name = memberName(member) + const functionProperty = ts.isPropertyDeclaration(member) + && ((member.initializer !== undefined && (ts.isArrowFunction(member.initializer) || ts.isFunctionExpression(member.initializer))) + || (member.type !== undefined && ts.isFunctionTypeNode(member.type))) + if (name === null || (!ts.isMethodDeclaration(member) && !functionProperty)) return [] + const isStatic = ts.canHaveModifiers(member) && (ts.getModifiers(member)?.some((modifier) => modifier.kind === ts.SyntaxKind.StaticKeyword) ?? false) + return [memberKey(name, isStatic)] + })) + const reachable = new Set(members.filter((member) => member.publicSurface).map((member) => memberKey(member.name, member.isStatic))) + const hasPrivateCallableBodies = members.some((member) => !member.publicSurface) + const nativeGlobals = new Set(['String', 'Number', 'Boolean', 'parseFloat', 'parseInt', 'isNaN', 'isFinite', + 'encodeURI', 'encodeURIComponent', 'decodeURI', 'decodeURIComponent', 'escape', 'unescape', 'atob', 'btoa']) + const shadowedNativeGlobals = declaredValueBindings(file) + let invalidReason: string | null = null + const localFunctionCache = new Map() + const nestedFunctions = (body: ts.Node): LocalFunctionReachability => { + const cached = localFunctionCache.get(body) + if (cached !== undefined) return cached + const result = inspectLocalFunctions(body) + localFunctionCache.set(body, result) + return result + } + const receiverReferences = (body: ts.Node, isStatic: boolean): Set => { + const names = new Set() + const locals = nestedFunctions(body) + if (locals.reason !== null) invalidReason = locals.reason + const visit = (node: ts.Node): void => { + if (ts.isClassExpression(node) || ts.isClassDeclaration(node)) { + invalidReason = 'Nested classes cannot be attributed to the selected class' + return + } + if (ts.isFunctionDeclaration(node)) { + if (locals.reachable.has(node) && node.body !== undefined) visit(node.body) + return + } + if (ts.isFunctionExpression(node) || ts.isArrowFunction(node)) { + invalidReason = 'Nested closures require a checked callable reachability graph' + return + } + if (ts.isElementAccessExpression(node) && node.expression.kind === ts.SyntaxKind.ThisKeyword) { + invalidReason = 'Computed receiver access cannot establish a frozen private-member reachability graph' + } + if (node.kind === ts.SyntaxKind.ThisKeyword) { + const directMemberReceiver = (ts.isPropertyAccessExpression(node.parent) + || ts.isElementAccessExpression(node.parent)) && node.parent.expression === node + const directFluentReturn = ts.isReturnStatement(node.parent) && node.parent.expression === node + if (!directMemberReceiver && !directFluentReturn) invalidReason = 'Aliased or escaping receivers require a checked private-member reachability graph' + } + const written = assignmentTarget(node) + if (written !== null && ts.isPropertyAccessExpression(written)) { + const receiver = written.expression + const receiverKey = receiver.kind === ts.SyntaxKind.ThisKeyword ? memberKey(written.name.text, isStatic) + : ts.isIdentifier(receiver) && receiver.text === className ? memberKey(written.name.text, true) : null + if (receiverKey !== null && callableMemberKeys.has(receiverKey)) { + invalidReason = 'Mutable callable member identity or captures cannot be represented by a complete state snapshot' + } + if (hasPrototypeAccess(written) || written.name.text === 'constructor') { + invalidReason = 'Prototype and constructor mutations are outside the checked class lifecycle subset' + } + } + if (ts.isDeleteExpression(node)) invalidReason = 'Deleted properties are outside the complete fixed-state snapshot subset' + if (ts.isCallExpression(node) && ts.isPropertyAccessExpression(node.expression) + && ts.isIdentifier(node.expression.expression) && ['Object', 'Reflect'].includes(node.expression.expression.text) + && ['setPrototypeOf', 'defineProperty', 'defineProperties', 'set', 'deleteProperty'].includes(node.expression.name.text)) { + invalidReason = 'Dynamic property or prototype mutation is outside the checked class lifecycle subset' + } + if (ts.isCallExpression(node) && ts.isPropertyAccessExpression(node.expression) + && ts.isIdentifier(node.expression.expression) && node.expression.expression.text === 'Object' + && node.expression.name.text === 'assign' && node.arguments[0] !== undefined + && (hasPrototypeAccess(node.arguments[0]) + || (ts.isIdentifier(node.arguments[0]) && node.arguments[0].text === className))) { + invalidReason = 'Dynamic property or prototype mutation is outside the checked class lifecycle subset' + } + if (hasPrivateCallableBodies && ts.isCallExpression(node) && ts.isIdentifier(node.expression)) { + const name = node.expression.text + const checkedLocal = [...locals.reachable].some((declaration) => declaration.name?.text === name) + const nativeGlobal = nativeGlobals.has(name) && !shadowedNativeGlobals.has(name) + if (!checkedLocal && !nativeGlobal) { + invalidReason = 'Unchecked free helper calls cannot establish the selected private callable statement universe' + } + } + if (ts.isCallExpression(node) && ts.isIdentifier(node.expression) && node.expression.text === 'eval') { + invalidReason = 'Dynamic evaluation cannot establish an original AST statement universe' + } + if (ts.isPropertyAccessExpression(node) && node.expression.kind === ts.SyntaxKind.ThisKeyword) { + names.add(memberKey(node.name.text, isStatic)) + } + if ((ts.isParameter(node) || ts.isVariableDeclaration(node) || ts.isBindingElement(node)) + && ts.isIdentifier(node.name) && node.name.text === className) { + invalidReason = 'A shadowed class binding would make static private-member reachability ambiguous' + } + if (ts.isIdentifier(node) && node.text === className) { + const parent = node.parent + const typeReference = ts.isTypeReferenceNode(parent) && parent.typeName === node + const constructorCall = ts.isNewExpression(parent) && parent.expression === node + const directClassMember = ts.isPropertyAccessExpression(parent) && parent.expression === node + const memberNameOnly = (ts.isPropertyAccessExpression(parent) && parent.name === node) + || (ts.isMethodDeclaration(parent) && parent.name === node) + || (ts.isPropertyAssignment(parent) && parent.name === node) + if (!typeReference && !constructorCall && !directClassMember && !memberNameOnly) { + invalidReason = 'Aliased, escaping or computed class receivers require a checked static private-member reachability graph' + } + } + if (ts.isPropertyAccessExpression(node) && ts.isIdentifier(node.expression) && node.expression.text === className) { + names.add(memberKey(node.name.text, true)) + } + if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.name.text === 'globalThis') { + invalidReason = 'A local globalThis binding would shadow native statement counter storage' + } + ts.forEachChild(node, visit) + } + visit(body) + return names + } + for (const initializer of initializers) receiverReferences(initializer.body, initializer.isStatic).forEach((name) => reachable.add(name)) + let changed = true + while (changed) { + changed = false + for (const member of members) { + if (!reachable.has(memberKey(member.name, member.isStatic))) continue + for (const name of receiverReferences(member.body, member.isStatic)) { + if (!reachable.has(name)) { reachable.add(name); changed = true } + } + } + } + if (invalidReason !== null) return unsupported(invalidReason) + if (hasGlobalThisBinding(file)) return unsupported('A globalThis binding would shadow native statement counter storage') + + const inventoried = inspectClassSurface(selected, file) + if (typeof inventoried === 'string') return unsupported(inventoried) + + const statements: ClassStatementDescriptor[] = [] + const insertions: Insertion[] = [] + const marker = (id: string): string => `(globalThis as unknown as Record>)[${JSON.stringify(counterGlobalName)}]?.add(${JSON.stringify(id)});` + const describe = (node: ts.Node, name: string, kind: string): ClassStatementDescriptor => { + const startOffset = node.getStart(file) + const endOffset = node.getEnd() + const start = file.getLineAndCharacterOfPosition(startOffset) + const end = file.getLineAndCharacterOfPosition(endOffset) + const descriptor = { id: `${sourceSha256}:${startOffset}:${endOffset}:${kind}`, memberName: name, kind, + startOffset, endOffset, startLine: start.line + 1, startColumn: start.character, + endLine: end.line + 1, endColumn: end.character } + statements.push(descriptor) + return descriptor + } + const instrumentExpression = (node: ts.Node, name: string, kind: string): void => { + const statement = describe(node, name, kind) + const hit = marker(statement.id).slice(0, -1) + insertions.push({ offset: node.getStart(file), text: `(${hit}, (`, order: 0 }) + insertions.push({ offset: node.getEnd(), text: '))', order: 1 }) + } + const instrumentBody = (body: ts.Node, name: string): void => { + const locals = nestedFunctions(body) + const visit = (node: ts.Node): void => { + if (ts.isFunctionDeclaration(node)) { + if (locals.reachable.has(node) && node.body !== undefined) visit(node.body) + return + } + if (ts.isStatement(node) && executableStatement(node)) { + const statement = describe(node, name, ts.SyntaxKind[node.kind]) + let anchor = node + while (ts.isLabeledStatement(anchor.parent)) anchor = anchor.parent + const parent = anchor.parent + const inList = ts.isBlock(parent) || ts.isCaseClause(parent) || ts.isDefaultClause(parent) + const prefix = `${marker(statement.id)} ` + if (inList) { + // A parenthesized counter must not continue a preceding semicolonless expression. + insertions.push({ offset: anchor.getStart(file), text: `; ${prefix}`, order: 0 }) + } else { + insertions.push({ offset: anchor.getStart(file), text: `{ ${prefix}`, order: 0 }) + insertions.push({ offset: anchor.getEnd(), text: ' }', order: 1 }) + } + } + ts.forEachChild(node, visit) + } + visit(body) + } + for (const initializer of initializers) instrumentExpression(initializer.body, initializer.name, 'PropertyInitializer') + for (const member of members) { + if (!reachable.has(memberKey(member.name, member.isStatic))) continue + if (ts.isBlock(member.body)) instrumentBody(member.body, member.name) + else instrumentExpression(member.body, member.name, 'CallableExpressionBody') + } + statements.sort((left, right) => left.startOffset - right.startOffset || left.endOffset - right.endOffset) + if (statements.length === 0) return unsupported('The selected class has no executable statements') + // For coincident close/open positions, close the earlier statement before opening the next one. + insertions.sort((left, right) => right.offset - left.offset || left.order - right.order) + let source = file.text + for (const insertion of insertions) source = source.slice(0, insertion.offset) + insertion.text + source.slice(insertion.offset) + return { ...base, source, reason: null, instrumentedSha256: hash(source), statements, inventory: inventoried } +} + +function memberName(member: ts.ClassElement): string | null { + const name = member.name + if (name !== undefined && (ts.isIdentifier(name) || ts.isPrivateIdentifier(name) || ts.isStringLiteral(name))) return name.text + return null +} + +function executableStatement(node: ts.Statement): boolean { + if (ts.isBlock(node) || ts.isEmptyStatement(node) || ts.isLabeledStatement(node) + || ts.isFunctionDeclaration(node) || ts.isClassDeclaration(node) || ts.isInterfaceDeclaration(node) + || ts.isTypeAliasDeclaration(node)) return false + if (ts.isVariableStatement(node) && node.declarationList.declarations.every((declaration) => declaration.initializer === undefined)) return false + const directive = ts.isExpressionStatement(node) && ts.isStringLiteral(node.expression) + return !directive +} + +function hasGlobalThisBinding(file: ts.SourceFile): boolean { + let found = false + const visit = (node: ts.Node): void => { + if (ts.isIdentifier(node) && node.text === 'globalThis') { + const parent = node.parent + const declaration = (ts.isParameter(parent) || ts.isVariableDeclaration(parent) || ts.isBindingElement(parent) + || ts.isImportSpecifier(parent) || ts.isImportClause(parent) || ts.isNamespaceImport(parent) + || ts.isFunctionDeclaration(parent) || ts.isClassDeclaration(parent)) && parent.name === node + if (declaration) found = true + } + ts.forEachChild(node, visit) + } + visit(file) + return found +} + +function hash(value: string): string { return createHash('sha256').update(value, 'utf8').digest('hex') } + + +interface LocalFunctionReachability { reachable: Set; reason: string | null } + +/** Checked local subset: uniquely named declarations in the member body, used only by direct calls. */ +function inspectLocalFunctions(body: ts.Node): LocalFunctionReachability { + const reachable = new Set() + let reason: string | null = null + const declarations = ts.isBlock(body) ? body.statements.filter(ts.isFunctionDeclaration) : [] + const byName = new Map() + for (const declaration of declarations) { + if (declaration.name === undefined || declaration.body === undefined) { + reason = 'Local callable declarations must have an exact name and body' + continue + } + const asynchronous = declaration.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.AsyncKeyword) + || declaration.asteriskToken !== undefined + if (asynchronous || byName.has(declaration.name.text)) { + reason = 'Async, generator or overloaded local callables are outside the checked reachability subset' + continue + } + byName.set(declaration.name.text, declaration) + } + const scan = (node: ts.Node, localOwner: ts.FunctionDeclaration | null): void => { + if (ts.isFunctionDeclaration(node)) { + if (!declarations.includes(node)) reason = 'Nested local function scopes require a checked callable reachability graph' + return + } + if (ts.isFunctionExpression(node) || ts.isArrowFunction(node)) { + reason = 'Nested closures require a checked callable reachability graph' + return + } + if (localOwner !== null && node.kind === ts.SyntaxKind.ThisKeyword) { + reason = 'Direct-called local functions cannot use a class receiver implicitly' + } + if (ts.isIdentifier(node) && byName.has(node.text)) { + const parent = node.parent + const directCall = ts.isCallExpression(parent) && parent.expression === node + const declarationName = ts.isFunctionDeclaration(parent) && parent.name === node + if (directCall) reachable.add(byName.get(node.text)!) + else if (!declarationName) reason = 'Aliased, escaped or shadowed local callables require a checked reachability graph' + } + ts.forEachChild(node, (child) => scan(child, localOwner)) + } + scan(body, null) + const scanned = new Set() + while (scanned.size < reachable.size) { + for (const declaration of reachable) { + if (scanned.has(declaration)) continue + scanned.add(declaration) + if (declaration.body !== undefined) scan(declaration.body, declaration) + } + } + return { reachable, reason } +} + + +function inspectClassSurface(selected: ts.ClassDeclaration, file: ts.SourceFile): ClassSurfaceInventory | string { + const constructorParameterNames: string[] = [] + const publicMembers: ClassCallableDescriptor[] = [] + const stateFields = new Map() + const callableOrAccessorNames = new Set() + let reason: string | null = null + const parameterNames = (parameters: ts.NodeArray): string[] => parameters.map((parameter) => { + if (!ts.isIdentifier(parameter.name) || parameter.dotDotDotToken !== undefined) { + reason = 'Destructured and rest class parameters are outside the frozen harness subset' + return '' + } + return parameter.name.text + }) + for (const member of selected.members) { + const modifiers = ts.canHaveModifiers(member) ? ts.getModifiers(member) ?? [] : [] + const isStatic = modifiers.some((modifier) => modifier.kind === ts.SyntaxKind.StaticKeyword) + if (ts.isConstructorDeclaration(member)) { + if (member.body === undefined) continue + constructorParameterNames.push(...parameterNames(member.parameters)) + for (const parameter of member.parameters) { + const parameterProperty = parameter.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.PublicKeyword + || modifier.kind === ts.SyntaxKind.PrivateKeyword || modifier.kind === ts.SyntaxKind.ProtectedKeyword + || modifier.kind === ts.SyntaxKind.ReadonlyKeyword) + if (parameterProperty && ts.isIdentifier(parameter.name)) { + stateFields.set(parameter.name.text, { name: parameter.name.text, isEcmaPrivate: false, + typeText: parameter.type?.getText(file) ?? null }) + } + } + continue + } + const name = memberName(member) + if (name === null) continue + const isPrivate = name.startsWith('#') || modifiers.some((modifier) => + modifier.kind === ts.SyntaxKind.PrivateKeyword || modifier.kind === ts.SyntaxKind.ProtectedKeyword) + const callable = ts.isPropertyDeclaration(member) + ? (member.initializer !== undefined && (ts.isArrowFunction(member.initializer) || ts.isFunctionExpression(member.initializer)) + ? member.initializer : member.type !== undefined && ts.isFunctionTypeNode(member.type) ? member.type : null) + : ts.isMethodDeclaration(member) || ts.isGetAccessorDeclaration(member) || ts.isSetAccessorDeclaration(member) ? member : null + if (callable !== null) { + if (!isStatic) callableOrAccessorNames.add(name) + const bodylessMethod = ts.isMethodDeclaration(callable) && callable.body === undefined + if (isPrivate || bodylessMethod) continue + const kind = ts.isGetAccessorDeclaration(member) ? 'GETTER' : ts.isSetAccessorDeclaration(member) ? 'SETTER' + : ts.isPropertyDeclaration(member) ? 'FUNCTION_FIELD' : 'METHOD' + publicMembers.push({ name, kind, isStatic, parameterNames: parameterNames(callable.parameters), + parameterTypeTexts: callable.parameters.map((parameter) => parameter.type?.getText(file) ?? null), + resultTypeText: callable.type?.getText(file) ?? null }) + } else if (ts.isPropertyDeclaration(member) && !isStatic) { + stateFields.set(name, { name, isEcmaPrivate: name.startsWith('#'), typeText: member.type?.getText(file) ?? null }) + } + } + const visitWrites = (node: ts.Node): void => { + if (node !== selected && (ts.isClassDeclaration(node) || ts.isClassExpression(node))) return + let written: ts.Node | undefined + if (ts.isBinaryExpression(node) && node.operatorToken.kind >= ts.SyntaxKind.FirstAssignment + && node.operatorToken.kind <= ts.SyntaxKind.LastAssignment) written = node.left + if (ts.isPrefixUnaryExpression(node) || ts.isPostfixUnaryExpression(node)) { + if (node.operator === ts.SyntaxKind.PlusPlusToken || node.operator === ts.SyntaxKind.MinusMinusToken) written = node.operand + } + if (written !== undefined && ts.isPropertyAccessExpression(written) && written.expression.kind === ts.SyntaxKind.ThisKeyword) { + const name = written.name.text + if (!callableOrAccessorNames.has(name) && !stateFields.has(name)) { + stateFields.set(name, { name, isEcmaPrivate: name.startsWith('#'), typeText: null }) + } + } + ts.forEachChild(node, visitWrites) + } + for (const member of selected.members) { + const isStatic = ts.canHaveModifiers(member) && ts.getModifiers(member)?.some((modifier) => modifier.kind === ts.SyntaxKind.StaticKeyword) + if (!isStatic) visitWrites(member) + } + if (reason !== null) return reason + return { constructorParameterNames, publicMembers, + instanceDataFields: [...stateFields.values()].sort((left, right) => left.name.localeCompare(right.name)), + dataClasses: inspectDataClasses(file) } +} + + +function assignmentTarget(node: ts.Node): ts.Node | null { + if (ts.isBinaryExpression(node) && node.operatorToken.kind >= ts.SyntaxKind.FirstAssignment + && node.operatorToken.kind <= ts.SyntaxKind.LastAssignment) return node.left + if ((ts.isPrefixUnaryExpression(node) || ts.isPostfixUnaryExpression(node)) + && (node.operator === ts.SyntaxKind.PlusPlusToken || node.operator === ts.SyntaxKind.MinusMinusToken)) return node.operand + return null +} + +function hasPrototypeAccess(node: ts.Node): boolean { + let found = false + const visit = (part: ts.Node): void => { + if (ts.isPropertyAccessExpression(part) && ['prototype', '__proto__'].includes(part.name.text)) found = true + if (ts.isElementAccessExpression(part) && ts.isStringLiteral(part.argumentExpression) + && ['prototype', '__proto__'].includes(part.argumentExpression.text)) found = true + ts.forEachChild(part, visit) + } + visit(node) + return found +} + + +function declaredValueBindings(file: ts.SourceFile): Set { + const names = new Set() + const visit = (node: ts.Node): void => { + if (ts.isIdentifier(node)) { + const parent = node.parent + const valueBinding = (ts.isParameter(parent) || ts.isVariableDeclaration(parent) || ts.isBindingElement(parent) + || ts.isImportSpecifier(parent) || ts.isImportClause(parent) || ts.isNamespaceImport(parent) + || ts.isFunctionDeclaration(parent) || ts.isClassDeclaration(parent)) && parent.name === node + if (valueBinding) names.add(node.text) + } + ts.forEachChild(node, visit) + } + visit(file) + return names +} + + +/** Source identity is the same captured module; registries cannot admit arbitrary instance prototypes. */ +function inspectDataClasses(file: ts.SourceFile): ClassDataClassDescriptor[] { + const candidates = file.statements.filter(ts.isClassDeclaration) + const typeAliases = new Map(file.statements.filter(ts.isTypeAliasDeclaration).map((node) => [node.name.text, node.type])) + const valueNames = declaredValueBindings(file) + const safeName = (name: string): boolean => /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(name) + && !['__proto__', 'prototype', 'constructor', '__usvmCallsClassIdentity'].includes(name) + const callableType = (type: ts.TypeNode | undefined, parameters: Map, seen = new Set()): boolean => { + if (type === undefined) return false + if (ts.isFunctionTypeNode(type) || ts.isConstructorTypeNode(type) || ts.isTypeQueryNode(type)) return true + if (ts.isTypeReferenceNode(type) && ts.isIdentifier(type.typeName)) { + const name = type.typeName.text + if (name === 'Function' || seen.has(name)) return true + const referenced = typeAliases.get(name) ?? parameters.get(name) + if (referenced !== undefined) { + const nestedSeen = new Set(seen) + nestedSeen.add(name) + if (callableType(referenced, parameters, nestedSeen)) return true + } + } + let found = false + ts.forEachChild(type, (node) => { + if (ts.isTypeNode(node) && callableType(node, parameters, seen)) found = true + if (ts.isCallSignatureDeclaration(node) || ts.isConstructSignatureDeclaration(node)) found = true + }) + return found + } + const pureValue = (node: ts.Expression, inputs: Set): boolean => { + if (ts.isParenthesizedExpression(node) || ts.isAsExpression(node) || ts.isTypeAssertionExpression(node) + || ts.isSatisfiesExpression(node) || ts.isNonNullExpression(node)) return pureValue(node.expression, inputs) + if (ts.isStringLiteral(node) || ts.isNumericLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node) + || node.kind === ts.SyntaxKind.TrueKeyword || node.kind === ts.SyntaxKind.FalseKeyword + || node.kind === ts.SyntaxKind.NullKeyword) return true + if (ts.isIdentifier(node)) return inputs.has(node.text) + || (['undefined', 'NaN', 'Infinity'].includes(node.text) && !valueNames.has(node.text)) + if (ts.isPrefixUnaryExpression(node) + && [ts.SyntaxKind.PlusToken, ts.SyntaxKind.MinusToken, ts.SyntaxKind.ExclamationToken, ts.SyntaxKind.TildeToken].includes(node.operator)) { + // Coercing an unknown constructor input can invoke valueOf or Symbol.toPrimitive. + return pureValue(node.operand, new Set()) + } + if (ts.isArrayLiteralExpression(node)) return node.elements.every((element) => + !ts.isSpreadElement(element) && !ts.isOmittedExpression(element) && pureValue(element, inputs)) + if (ts.isObjectLiteralExpression(node)) return node.properties.every((property) => { + if (!ts.isPropertyAssignment(property)) return false + const name = property.name + const literalName = ts.isIdentifier(name) || ts.isStringLiteral(name) + return literalName && safeName(name.text) && pureValue(property.initializer, inputs) + }) + return false + } + const result: ClassDataClassDescriptor[] = [] + for (const candidate of candidates) { + const modifiers = ts.getModifiers(candidate) ?? [] + const exported = modifiers.some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword) + const disallowedClassModifier = modifiers.some((modifier) => [ts.SyntaxKind.DefaultKeyword, + ts.SyntaxKind.AbstractKeyword, ts.SyntaxKind.DeclareKeyword].includes(modifier.kind)) + const exportName = candidate.name?.text + if (!exported || disallowedClassModifier || exportName === undefined || !safeName(exportName)) continue + const decorated = (ts.getDecorators(candidate)?.length ?? 0) > 0 + if (decorated || (candidate.heritageClauses?.length ?? 0) > 0 || !uniqueDirectClassExport(file, candidate, exportName)) continue + const parameters = new Map((candidate.typeParameters ?? []).map((parameter) => [parameter.name.text, parameter.constraint])) + const fields = new Set() + let eligible = true + let constructor: ts.ConstructorDeclaration | undefined + for (const member of candidate.members) { + if (ts.isSemicolonClassElement(member)) continue + const memberModifiers = ts.canHaveModifiers(member) ? ts.getModifiers(member) ?? [] : [] + const memberDecorated = ts.canHaveDecorators(member) && (ts.getDecorators(member)?.length ?? 0) > 0 + const isStatic = memberModifiers.some((modifier) => modifier.kind === ts.SyntaxKind.StaticKeyword) + if (memberDecorated || isStatic) { eligible = false; break } + if (ts.isConstructorDeclaration(member)) { + if (member.body === undefined) continue + if (constructor !== undefined) { eligible = false; break } + constructor = member + continue + } + if (!ts.isPropertyDeclaration(member) || member.name === undefined || ts.isPrivateIdentifier(member.name) + || (!ts.isIdentifier(member.name) && !ts.isStringLiteral(member.name))) { eligible = false; break } + const name = member.name.text + const valueInitializer = member.initializer + const safeInitializer = valueInitializer === undefined || pureValue(valueInitializer, new Set()) + if (!safeName(name) || fields.has(name) || callableType(member.type, parameters) || !safeInitializer) { + eligible = false + break + } + fields.add(name) + } + if (!eligible) continue + if (constructor !== undefined) { + const inputs = new Set() + for (const parameter of constructor.parameters) { + if (!ts.isIdentifier(parameter.name) || parameter.dotDotDotToken !== undefined + || inputs.has(parameter.name.text) || callableType(parameter.type, parameters)) { eligible = false; break } + inputs.add(parameter.name.text) + if (parameter.initializer !== undefined && !pureValue(parameter.initializer, inputs)) { eligible = false; break } + const parameterProperty = parameter.modifiers?.some((modifier) => [ts.SyntaxKind.PublicKeyword, + ts.SyntaxKind.PrivateKeyword, ts.SyntaxKind.ProtectedKeyword, ts.SyntaxKind.ReadonlyKeyword].includes(modifier.kind)) + if (parameterProperty) { + const name = parameter.name.text + if (!safeName(name) || fields.has(name)) { eligible = false; break } + fields.add(name) + } + } + if (!eligible) continue + for (const statement of constructor.body!.statements) { + if (ts.isEmptyStatement(statement)) continue + if (!ts.isExpressionStatement(statement) || !ts.isBinaryExpression(statement.expression) + || statement.expression.operatorToken.kind !== ts.SyntaxKind.EqualsToken) { eligible = false; break } + const assignment = statement.expression + const target = assignment.left + const ownField = ts.isPropertyAccessExpression(target) && target.expression.kind === ts.SyntaxKind.ThisKeyword + && fields.has(target.name.text) + if (!ownField || !pureValue(assignment.right, inputs)) { eligible = false; break } + } + } + if (!eligible) continue + // An exported class binding must retain the source-declared constructor identity. + const modified = classBindingModified(file, exportName) + if (modified) continue + result.push({ exportName, fields: [...fields].sort() }) + } + return result.sort((left, right) => left.exportName.localeCompare(right.exportName)) +} + +function uniqueDirectClassExport(file: ts.SourceFile, candidate: ts.ClassDeclaration, name: string): boolean { + const bindsName = (binding: ts.BindingName): boolean => ts.isIdentifier(binding) ? binding.text === name + : binding.elements.some((element) => !ts.isOmittedExpression(element) && bindsName(element.name)) + let declarations = 0 + for (const statement of file.statements) { + if ((ts.isClassDeclaration(statement) || ts.isFunctionDeclaration(statement) || ts.isEnumDeclaration(statement) + || ts.isModuleDeclaration(statement)) && statement.name?.getText(file) === name) declarations++ + if (ts.isVariableStatement(statement) && statement.declarationList.declarations.some((declaration) => + bindsName(declaration.name))) declarations++ + if (ts.isImportEqualsDeclaration(statement) && !statement.isTypeOnly && statement.name.text === name) declarations++ + if (ts.isImportDeclaration(statement) && statement.importClause !== undefined && !statement.importClause.isTypeOnly) { + const clause = statement.importClause + const bindings = clause.namedBindings + const defaultBinding = clause.name?.text === name + const namespaceBinding = bindings !== undefined && ts.isNamespaceImport(bindings) && bindings.name.text === name + const namedBinding = bindings !== undefined && ts.isNamedImports(bindings) + && bindings.elements.some((element) => !element.isTypeOnly && element.name.text === name) + if (defaultBinding || namespaceBinding || namedBinding) declarations++ + } + const conflictingExport = ts.isExportDeclaration(statement) && !statement.isTypeOnly + && statement.exportClause !== undefined && ts.isNamedExports(statement.exportClause) + && statement.exportClause.elements.some((element) => !element.isTypeOnly && element.name.text === name) + if (conflictingExport) return false + } + return declarations === 1 && candidate.name?.text === name +} + +function classBindingModified(file: ts.SourceFile, name: string): boolean { + const writesName = (target: ts.Node): boolean => { + if (ts.isIdentifier(target)) return target.text === name + if (ts.isParenthesizedExpression(target)) return writesName(target.expression) + if (ts.isArrayLiteralExpression(target)) return target.elements.some(writesName) + if (ts.isObjectLiteralExpression(target)) return target.properties.some((property) => { + if (ts.isShorthandPropertyAssignment(property)) return property.name.text === name + if (ts.isPropertyAssignment(property)) return writesName(property.initializer) + return ts.isSpreadAssignment(property) && writesName(property.expression) + }) + if (ts.isSpreadElement(target)) return writesName(target.expression) + if (ts.isBinaryExpression(target) && target.operatorToken.kind === ts.SyntaxKind.EqualsToken) return writesName(target.left) + return false + } + let modified = false + const visit = (node: ts.Node): void => { + const target = assignmentTarget(node) + if (target !== null && writesName(target)) modified = true + ts.forEachChild(node, visit) + } + visit(file) + return modified +} diff --git a/usvm-ts-fast-check/fast-check-adapter/src/local-source-closure.ts b/usvm-ts-fast-check/fast-check-adapter/src/local-source-closure.ts index 60d867772f..27513b6712 100644 --- a/usvm-ts-fast-check/fast-check-adapter/src/local-source-closure.ts +++ b/usvm-ts-fast-check/fast-check-adapter/src/local-source-closure.ts @@ -253,7 +253,9 @@ function safeModuleStatement(statement: ts.Statement, source: ts.SourceFile): bo } if (ts.isExportAssignment(statement)) return !statement.isExportEquals && safeInitializer(statement.expression, source) if (ts.isClassDeclaration(statement)) { - if (statement.heritageClauses?.length || ts.getDecorators(statement)?.length) return false + const extendsRuntimeBase = statement.heritageClauses?.some(clause => clause.token === ts.SyntaxKind.ExtendsKeyword) ?? false + const decorated = (ts.getDecorators(statement)?.length ?? 0) > 0 + if (extendsRuntimeBase || decorated) return false return statement.members.every(member => !ts.isClassStaticBlockDeclaration(member) && !(member.name && ts.isComputedPropertyName(member.name)) && !(ts.canHaveDecorators(member) && ts.getDecorators(member)?.length) diff --git a/usvm-ts-fast-check/fast-check-adapter/src/source-inspector-cli.ts b/usvm-ts-fast-check/fast-check-adapter/src/source-inspector-cli.ts index 5869fe5313..54e6e76795 100644 --- a/usvm-ts-fast-check/fast-check-adapter/src/source-inspector-cli.ts +++ b/usvm-ts-fast-check/fast-check-adapter/src/source-inspector-cli.ts @@ -3,6 +3,7 @@ import ts from 'typescript' import { instrumentSourceReplay } from './source-replay.js' import { inspectSourceCaptures } from './source-captures.js' import { inspectLocalSourceClosure } from './local-source-closure.js' +import { instrumentClassCoverage } from './class-coverage-source.js' const [operation, sourcePath, ...arguments_] = process.argv.slice(2) if (sourcePath === undefined) throw new Error('Expected a TypeScript source path') @@ -20,7 +21,11 @@ if (operation === 'inspect-local-closure') { }).diagnostics?.filter((diagnostic) => diagnostic.category === ts.DiagnosticCategory.Error) ?? [] if (diagnostics.length > 0) throw new Error(ts.flattenDiagnosticMessageText(diagnostics[0]?.messageText ?? '', '\n')) - if (operation === 'inspect-captures') { + if (operation === 'instrument-class-coverage') { + const className = arguments_[0] + if (className === undefined) throw new Error('Expected a target class name') + process.stdout.write(JSON.stringify(instrumentClassCoverage(sourceFile, className))) + } else if (operation === 'inspect-captures') { const [exportName, namesText] = arguments_ if (exportName === undefined || namesText === undefined) throw new Error('Expected export and capture names') const names: unknown = JSON.parse(namesText) diff --git a/usvm-ts-fast-check/fast-check-adapter/test/class-coverage-source.test.ts b/usvm-ts-fast-check/fast-check-adapter/test/class-coverage-source.test.ts new file mode 100644 index 0000000000..d2a12330dc --- /dev/null +++ b/usvm-ts-fast-check/fast-check-adapter/test/class-coverage-source.test.ts @@ -0,0 +1,396 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { runInNewContext } from 'node:vm' +import ts from 'typescript' +import { instrumentClassCoverage } from '../src/class-coverage-source.js' + +function instrument(source: string, className = 'Target') { + const file = ts.createSourceFile('target.ts', source, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS) + return instrumentClassCoverage(file, className) +} + +function load(source: string, counter: string | null) { + const exports: Record = {} + const sandbox: Record = { exports, module: { exports } } + if (counter !== null) sandbox[counter] = new Set() + const js = ts.transpileModule(source, { compilerOptions: { + target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.CommonJS, + } }).outputText + runInNewContext(js, sandbox) + return { sandbox, Target: exports['Target'] as new (...args: unknown[]) => Record unknown> } +} + +test('freezes distinct original AST statements on one line and only native hits', () => { + const source = 'export class Target { run(x: boolean) { if (x) return 1; return 2; } }' + const result = instrument(source) + assert.equal(result.reason, null) + assert.equal(result.statements.length, 3) + assert.equal(new Set(result.statements.map((statement) => statement.id)).size, 3) + assert.deepEqual(result.statements.map((statement) => statement.kind), ['IfStatement', 'ReturnStatement', 'ReturnStatement']) + + const runtime = load(result.source!, result.counterGlobalName) + const receiver = new runtime.Target() + assert.equal(receiver['run']!(true), 1) + const covered = Array.from(runtime.sandbox[result.counterGlobalName] as Set) + assert.deepEqual(covered, result.statements.slice(0, 2).map((statement) => statement.id)) + assert.equal(covered.includes(result.statements[2]!.id), false) + assert.deepEqual(instrument(source), result) +}) + +test('counts initialization and reachable private bodies while excluding foreign and dead code', () => { + const source = ` + interface TypeOnly { value: number } + export class Foreign { run() { return 9; } } + export class Target { + value = 0; + constructor(x: number) { this.value = this.helper(x); } + run() { return this.value; } + private helper(x: number) { if (x > 0) return x; return -x; } + private dead() { return 99; } + } + export function wrapper() { return 100; } + ` + const result = instrument(source) + assert.equal(result.reason, null) + assert.deepEqual(new Set(result.statements.map((statement) => statement.memberName)), + new Set(['value', 'constructor', 'run', 'helper'])) + assert.equal(result.statements[0]!.kind, 'PropertyInitializer') + + const original = load(source, null) + const instrumented = load(result.source!, result.counterGlobalName) + assert.equal(new original.Target(-2)['run']!(), new instrumented.Target(-2)['run']!()) + const covered = Array.from(instrumented.sandbox[result.counterGlobalName] as Set) + assert.equal(covered.length, 5) + assert.equal(covered.every((id) => result.statements.some((statement) => statement.id === id)), true) +}) + +test('preserves labelled loops, unbraced nested controls and else binding', () => { + const source = `export class Target { run(x: number) { + let value = 0; + outer: for (let i = 0; i < 4; i++) { + if (i === x) continue outer; + if (i < x) if (x > 1) value += i; else value -= i; + } + return value; + } }` + const result = instrument(source) + assert.equal(result.reason, null) + const original = load(source, null) + const instrumented = load(result.source!, result.counterGlobalName) + + for (const value of [-1, 0, 1, 2, 3, 8]) { + assert.equal(new instrumented.Target()['run']!(value), new original.Target()['run']!(value)) + } + assert.equal(result.statements.some((statement) => statement.kind === 'ContinueStatement'), true) + assert.equal(result.statements.some((statement) => statement.kind === 'LabeledStatement'), false) +}) + +test('supports private accessors and expression callable properties', () => { + const result = instrument(`export class Target { + private value = 2; + private get doubled() { return this.value * 2; } + run = () => this.doubled; + }`) + assert.equal(result.reason, null) + assert.deepEqual(result.statements.map((statement) => statement.kind), + ['PropertyInitializer', 'ReturnStatement', 'CallableExpressionBody']) + const runtime = load(result.source!, result.counterGlobalName) + assert.equal(new runtime.Target()['run']!(), 4) + assert.equal((Array.from(runtime.sandbox[result.counterGlobalName] as Set)).length, 3) +}) + +test('retains explicit unsupported dynamic reachability and closure outcomes', () => { + assert.match(instrument('export class Target { run(name: string) { return this[name]; } }').reason!, /Computed receiver/) + assert.match(instrument('export class Target { run() { return [1].map(x => x + 1); } }').reason!, /Nested closures/) + assert.match(instrument('export class Target { run() { const alias = this; return alias.hidden(); } private hidden() { return 1; } }').reason!, /Aliased or escaping/) + assert.match(instrument('export class Target { run() { return eval("this.hidden()"); } private hidden() { return 1; } }').reason!, /Dynamic evaluation/) + assert.match(instrument('export class Target extends Base { run() { return 1; } }').reason!, /Inherited/) + assert.match(instrument('export class Target { static value = 1; static run() { return 1; } }').reason!, /Static/) + assert.match(instrument('export class Target { run(globalThis: unknown) { return 1; } }').reason!, /globalThis binding/) +}) + + +test('includes static private helpers reachable through the class or static receiver', () => { + const source = `export class Target { + static run(x: number) { return this.helper(x) + Target.helper(x); } + private static helper(x: number) { if (x > 0) return x; return -x; } + private helper() { return 99; } + private static dead() { return 100; } + }` + const result = instrument(source) + assert.equal(result.reason, null) + assert.equal(result.statements.length, 4) + assert.equal(result.statements.some((statement) => source.slice(statement.startOffset, statement.endOffset) === 'return 99;'), false) + + const runtime = load(result.source!, result.counterGlobalName) + const target = runtime.Target as unknown as Record unknown> + assert.equal(target['run']!(-2), 4) + const covered = Array.from(runtime.sandbox[result.counterGlobalName] as Set) + assert.equal(new Set(covered).size, 3) +}) + + +test('includes exact bodies of direct-called local functions without adding unused local bodies', () => { + const source = `export class Target { + static run(value: number) { + function pad(input: number) { + let result = String(input); + while (result.length < 2) result = "0" + result; + return result; + } + function dead() { return 99; } + let declarationOnly: string; + return pad(value); + } + }` + const result = instrument(source) + assert.equal(result.reason, null) + assert.equal(result.statements.length, 5) + assert.equal(result.statements.some((statement) => source.slice(statement.startOffset, statement.endOffset) === 'return 99;'), false) + + const original = load(source, null) + const instrumented = load(result.source!, result.counterGlobalName) + const originalClass = original.Target as unknown as Record unknown> + const instrumentedClass = instrumented.Target as unknown as Record unknown> + assert.equal(instrumentedClass['run']!(1), originalClass['run']!(1)) + assert.equal(new Set(Array.from(instrumented.sandbox[result.counterGlobalName] as Set)).size, 5) + assert.match(instrument('export class Target { run() { function local() { return 1; } return [1].map(local); } }').reason!, /escaped/) +}) + + +test('freezes the complete constructor callable and receiver data inventory', () => { + const source = `export class Target { + private flag = false; + constructor(public value: number) { this.extra = 1; } + run(x: number) { this.value += x; return this; } + get current() { return this.value; } + set current(value: number) { this.value = value; } + callback = (input: number) => input + 1; + static utility(value: number) { return value + 1; } + }` + const result = instrument(source) + assert.equal(result.reason, null) + assert.deepEqual(result.inventory!.constructorParameterNames, ['value']) + assert.deepEqual(result.inventory!.instanceDataFields.map((field) => field.name), ['extra', 'flag', 'value']) + assert.deepEqual(result.inventory!.publicMembers.map((member) => [member.name, member.kind, member.isStatic, member.parameterNames]), + [['run', 'METHOD', false, ['x']], ['current', 'GETTER', false, []], ['current', 'SETTER', false, ['value']], + ['callback', 'FUNCTION_FIELD', false, ['input']], ['utility', 'METHOD', true, ['value']]]) +}) + + +test('rejects mutable callable identity and dynamic lifecycle state without rejecting stable initializers', () => { + assert.equal(instrument('export class Target { run = () => 1; other() { return this.run(); } }').reason, null) + assert.match(instrument('export class Target { run = () => 1; other() { this.run = this.other; return 1; } }').reason!, /Mutable callable/) + assert.match(instrument('export class Target { run() { this.run = this.other; return 1; } other() { return 2; } }').reason!, /Mutable callable/) + assert.match(instrument('export class Target { static run() { Target.run = Target.other; return 1; } static other() { return 2; } }').reason!, /Mutable callable/) + assert.match(instrument('export class Target { value = 1; run() { delete this.value; return 1; } }').reason!, /Deleted properties/) + assert.match(instrument('export class Target { run() { Target.prototype.run = Target.prototype.other; return 1; } other() { return 2; } }').reason!, /Prototype/) + assert.match(instrument('export class Target { run() { Object.defineProperty(Target.prototype, "extra", {value: 1}); return 1; } }').reason!, /Dynamic property/) + assert.match(instrument('export class Target { static run() { const Alias = Target; return Alias.hidden(); } private static hidden() { return 1; } }').reason!, /Aliased, escaping or computed class/) + assert.match(instrument('export class Target { static run(key: string) { return Target[key](); } private static hidden() { return 1; } }').reason!, /Aliased, escaping or computed class/) + assert.match(instrument('export class Target { static run() { Object.assign(Target.prototype, {extra: 1}); return 1; } }').reason!, /Dynamic property/) + assert.match(instrument('export class Target { static run() { Reflect.set(Target.prototype, "extra", 1); return 1; } }').reason!, /Dynamic property/) +}) + + +test('rejects unchecked free helpers that could enter private class bodies', () => { + const unsafe = instrument('function bridge(){return Target.hidden()} export class Target{static run(){return bridge()} private static hidden(){return 1}}') + assert.equal(unsafe.source, null) + assert.match(unsafe.reason!, /Unchecked free helper/) + assert.equal(instrument('export class Target { static run(value: number) { return this.helper(String(value)); } private static helper(value: string) { return value.length; } }').reason, null) + assert.equal(instrument('export class Target { static run(value: number) { function local(input: number) { return String(input); } return this.helper(local(value)); } private static helper(value: string) { return value.length; } }').reason, null) + assert.match(instrument('function String(value: unknown) { return Target.hidden(); } export class Target { static run() { return String(1); } private static hidden() { return 2; } }').reason!, /Unchecked free helper/) +}) + +test('preserves automatic semicolon insertion between assignments and statement counters', () => { + const source = `export class Target { + private size!: number + private buckets!: number[][] + private readonly loadFactor = 0.75 + + constructor() { + this.clear() + } + + clear() { + this.size = 0 + this.initializeBuckets(3) + } + + private initializeBuckets(amount: number) { + this.buckets = [] + for (let i = 0; i < amount; i++) { + this.buckets.push([]) + } + } + + run() { + this.size = 0 + if (this.size === 0) + this.initializeBuckets(3) + else + this.buckets = [] + return this.buckets.length + } + }` + const result = instrument(source) + assert.equal(result.reason, null) + + const original = load(source, null) + const instrumented = load(result.source!, result.counterGlobalName) + const originalReceiver = new original.Target() + const instrumentedReceiver = new instrumented.Target() + + assert.equal(instrumentedReceiver['run']!(), originalReceiver['run']!()) + assert.equal(instrumentedReceiver['run']!(), 3) + const nativeHits = Array.from(instrumented.sandbox[result.counterGlobalName] as Set) + const coveredMemberNames = result.statements.filter((statement) => nativeHits.includes(statement.id)) + .map((statement) => statement.memberName) + assert.equal(coveredMemberNames.includes('loadFactor'), true) + assert.equal(coveredMemberNames.includes('constructor'), true) + assert.equal(coveredMemberNames.includes('initializeBuckets'), true) +}) + + +test('inventories same-source exported generic data classes with complete own fields', () => { + const source = ` + export class Target { run() { return 1; } } + export class MapEntry { + key: K + value: V + + constructor(key: K, value: V) { + this.key = key + this.value = value + } + } + export class Coordinates { + readonly label = 'point'; + readonly negative = -1; + constructor(public x: number, private y: number = 0) {} + } + ` + const result = instrument(source) + + assert.equal(result.reason, null) + assert.deepEqual(result.inventory!.dataClasses, [ + { exportName: 'Coordinates', fields: ['label', 'negative', 'x', 'y'] }, + { exportName: 'MapEntry', fields: ['key', 'value'] }, + ]) + assert.deepEqual(result.statements.map((statement) => statement.memberName), ['run']) +}) + + +test('excludes non-data companion classes without rejecting unrelated target coverage', () => { + const companions = [ + 'export class Companion { value = 1; method() {} }', + 'export class Companion { get value() { return 1; } }', + 'export class Companion extends Base { value = 1; }', + 'export class Companion implements Shape { value = 1; }', + 'export class Companion { static value = 1; }', + 'export class Companion { static {} }', + 'export class Companion { ["value"] = 1; }', + 'export class Companion { #value = 1; }', + 'export class Companion { constructor: number = 1; }', + 'export class Companion { __usvmCallsClassIdentity = "fake"; }', + 'export class Companion { callback: () => number; }', + 'type Callback = () => number; export class Companion { callback: Callback; }', + 'export class Companion number> { callback: T; }', + 'export class Companion { callback = () => 1; }', + 'export class Companion { value = effect(); }', + 'export class Companion { value: number; constructor(value: number) { this.value = effect(value); } }', + 'export class Companion { amount: number; constructor(value: unknown) { this.amount = +value; } }', + 'export class Companion { amount: number; constructor(value: unknown) { this.amount = -value; } }', + 'export class Companion { amount: number; constructor(value: unknown) { this.amount = ~value; } }', + 'export class Companion { value: number; constructor(value: number) { this.extra = value; } }', + 'export class Companion { constructor(callback: () => number) {} }', + 'export abstract class Companion { value = 1; }', + 'export default class Companion { value = 1; }', + 'export declare class Companion { value: number; }', + '@decorate export class Companion { value = 1; }', + ] + + for (const companion of companions) { + const result = instrument(`export class Target { run() { return 1; } } ${companion}`) + + assert.equal(result.reason, null, companion) + assert.deepEqual(result.inventory!.dataClasses, [], companion) + } +}) + + +test('excludes companion exports with shadowed or re-exported constructor identity', () => { + const conflicts = [ + 'export { Foreign as Companion } from "./foreign";', + 'const Companion = replacement;', + 'const { original: Companion } = replacement;', + 'const [Companion] = replacement;', + 'import Companion from "./foreign";', + 'import { Foreign as Companion } from "./foreign";', + 'import * as Companion from "./foreign";', + 'import Companion = require("./foreign");', + ] + + for (const conflict of conflicts) { + const result = instrument(`export class Target { run() { return 1; } } + export class Companion { value = 1; } ${conflict}`) + + assert.equal(result.reason, null, conflict) + assert.deepEqual(result.inventory!.dataClasses, [], conflict) + } +}) + + +test('excludes reassigned companion constructors including destructuring writes', () => { + const mutations = [ + 'Companion = replacement;', + 'Companion++;', + '[Companion] = replacement;', + '({ Companion } = replacement);', + '({ original: Companion } = replacement);', + '[Companion = replacement] = replacements;', + '[...Companion] = replacements;', + ] + + for (const mutation of mutations) { + const result = instrument(`export class Target { run() { return 1; } } + export class Companion { value = 1; } ${mutation}`) + + assert.equal(result.reason, null, mutation) + assert.deepEqual(result.inventory!.dataClasses, [], mutation) + } +}) + + +test('stores only distinct statement hits across long loops and repeated calls', () => { + const source = `export class Target { + run(amount: number) { + let total = 0; + try { + for (let index = 0; index < amount; index++) { + total += index; + } + } catch { + return -1; + } + return total; + } + }` + const result = instrument(source) + assert.equal(result.reason, null) + const original = load(source, null) + const runtime = load(result.source!, result.counterGlobalName) + const receiver = new runtime.Target() + + assert.equal(receiver['run']!(10_000), new original.Target()['run']!(10_000)) + const firstHits = Array.from(runtime.sandbox[result.counterGlobalName] as Iterable) + assert.equal(firstHits.length, new Set(firstHits).size) + assert.equal(firstHits.length <= result.statements.length, true) + assert.equal(firstHits.every((id) => result.statements.some((statement) => statement.id === id)), true) + const caughtReturn = result.statements.find((statement) => source.slice(statement.startOffset, statement.endOffset) === 'return -1;')! + assert.equal(firstHits.includes(caughtReturn.id), false) + + for (let iteration = 0; iteration < 20; iteration++) receiver['run']!(10_000) + const repeatedHits = Array.from(runtime.sandbox[result.counterGlobalName] as Iterable) + assert.deepEqual(repeatedHits, firstHits) +}) diff --git a/usvm-ts-fast-check/fast-check-adapter/test/local-source-closure.test.ts b/usvm-ts-fast-check/fast-check-adapter/test/local-source-closure.test.ts index 524e77d848..eaf07bad7e 100644 --- a/usvm-ts-fast-check/fast-check-adapter/test/local-source-closure.test.ts +++ b/usvm-ts-fast-check/fast-check-adapter/test/local-source-closure.test.ts @@ -158,7 +158,6 @@ test('binding inspection does not read ignored type imports outside the runtime }) }) - test('single-source captures retain their dedicated validation route', () => { workspace({ 'entry.ts': 'const source = { value: 7 }; const alias = source; ' @@ -167,3 +166,35 @@ test('single-source captures retain their dedicated validation route', () => { assert.deepEqual(inspectLocalSourceClosure(root, path.join(root, 'entry.ts')), { files: ['entry.ts'] }) }) }) + +test('imported class implements an erased interface without a runtime initialization effect', () => { + workspace({ + 'wrapper.ts': "import { HashMap } from './subject'; export function scenario() { return new HashMap().size; }", + 'subject.ts': "import { MapContract } from './type-contract'; " + + 'export class HashMap implements MapContract { size = 0; set(_key: K, _value: V) { this.size++; } }', + 'type-contract.ts': 'export interface MapContract { size: number; set(key: K, value: V): void }', + }, root => { + assert.deepEqual(inspectLocalSourceClosure(root, path.join(root, 'wrapper.ts')), { + files: ['subject.ts', 'wrapper.ts'], + }) + }) +}) + +for (const [name, declaration] of [ + ['effectful extends expression', 'function base() { throw new Error("must not execute"); } export class Subject extends base() {}'], + ['plain runtime extends', 'class Base {} export class Subject extends Base {}'], + ['decorated implements class', 'interface Contract {} function decorate(value: unknown) { return value; } @decorate export class Subject implements Contract {}'], + ['static initializer with implements', 'interface Contract {} export class Subject implements Contract { static value = Math.random(); }'], + ['static block with implements', 'interface Contract {} export class Subject implements Contract { static { throw new Error("must not execute"); } }'], +] as const) { + test(`imported class retains module initialization rejection for ${name}`, () => { + workspace({ + 'wrapper.ts': "import { Subject } from './subject'; export function scenario() { return new Subject(); }", + 'subject.ts': declaration, + }, root => { + const result = inspectLocalSourceClosure(root, path.join(root, 'wrapper.ts')) + assert.equal(result.reasonCode, 'TOP_LEVEL_INITIALIZATION_UNSUPPORTED') + assert.match(result.diagnostic!, /subject\.ts/) + }) + }) +} diff --git a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/TypeScriptSourceInspector.kt b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/TypeScriptSourceInspector.kt index 992fa65184..aa815efc14 100644 --- a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/TypeScriptSourceInspector.kt +++ b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/TypeScriptSourceInspector.kt @@ -10,6 +10,16 @@ import java.util.concurrent.TimeUnit /** Uses the native TypeScript AST when lowered source ranges need validation or replay instrumentation. */ object TypeScriptSourceInspector { + fun instrumentClassCoverage(source: Path, className: String): TypeScriptClassCoverageInstrumentation { + val output = inspect( + source = source, + operation = "instrument-class-coverage", + arguments = listOf(className), + ) + + return PropertyManifestJson.json.decodeFromString(output) + } + fun localSourceClosure(sourceRoot: Path, source: Path): TypeScriptLocalSourceClosure { val output = inspect( source = sourceRoot, @@ -148,3 +158,59 @@ data class TypeScriptLocalSourceClosure( val reasonCode: String? = null, val diagnostic: String? = null, ) + +@Serializable +data class TypeScriptClassStatementDescriptor( + val id: String, + val memberName: String, + val kind: String, + val startOffset: Int, + val endOffset: Int, + val startLine: Int, + val startColumn: Int, + val endLine: Int, + val endColumn: Int, +) + +@Serializable +data class TypeScriptClassCoverageInstrumentation( + val source: String?, + val reason: String?, + val sourceSha256: String, + val instrumentedSha256: String?, + val counterGlobalName: String, + val instrumenterVersion: String, + val statements: List, + val inventory: TypeScriptClassSurfaceInventory?, +) + +@Serializable +data class TypeScriptClassCallableDescriptor( + val name: String, + val kind: String, + val isStatic: Boolean, + val parameterNames: List, + val parameterTypeTexts: List, + val resultTypeText: String?, +) + +@Serializable +data class TypeScriptClassStateFieldDescriptor( + val name: String, + val isEcmaPrivate: Boolean, + val typeText: String?, +) + +@Serializable +data class TypeScriptClassSurfaceInventory( + val constructorParameterNames: List, + val publicMembers: List, + val instanceDataFields: List, + val dataClasses: List = emptyList(), +) + +@Serializable +data class TypeScriptClassDataClassDescriptor( + val exportName: String, + val fields: List, +) From 079b43404b2383fa83c44b838d1f292f6e385541 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Wed, 14 Oct 2026 23:04:23 +0300 Subject: [PATCH 2/2] Collect stopped class inputs through the TypeScript observer Connect class candidate extraction to the merged TS observer factory and collect queued inputs after successful analysis. Remove references to the retired shared callback and option while preserving stopped-versus-terminal candidate accounting. --- .../org/usvm/ts/calls/CallsClassCoverageEngine.kt | 10 +++++++--- .../kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt | 2 +- .../usvm/ts/calls/CallsClassUnknownCallJournalTest.kt | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassCoverageEngine.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassCoverageEngine.kt index 9b7bfd632a..35eb2ee287 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassCoverageEngine.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassCoverageEngine.kt @@ -15,6 +15,7 @@ import org.usvm.machine.TsMachine import org.usvm.machine.TsOptions import org.usvm.machine.TsRuntimeFeatureLimitationEvent import org.usvm.machine.TsSolverQueryEvent +import org.usvm.machine.TsStoppedStatesObserver import org.usvm.machine.call.TsUnknownCallDecision import org.usvm.machine.call.TsUnknownCallEvent import org.usvm.machine.call.TsUnknownCallModelSelection @@ -242,7 +243,8 @@ internal class CurrentTsCallsClassCoverageEngine { ) val limitations = linkedSetOf() val unknownCalls = mutableListOf() - val interpreterObserver = object : TsInterpreterObserver { + val stoppedObserver = TsStoppedStatesObserver(consumeState = candidateObserver::collectStoppedState) + val interpreterObserver = object : TsInterpreterObserver by stoppedObserver { override fun onRuntimeFeatureLimitation(event: TsRuntimeFeatureLimitationEvent) { limitations += "${event.reason.name}: ${event.detail}" } @@ -278,7 +280,6 @@ internal class CurrentTsCallsClassCoverageEngine { stopOnCoverage = CALLS_STOP_ON_COVERAGE, stopOnTargetsReached = false, throwExceptionOnStepFailure = true, - emitRemainingStatesOnStop = true, ), tsOptions = TsOptions( maxArraySize = maxOf(TsOptions().maxArraySize, inputCapacity), @@ -289,13 +290,16 @@ internal class CurrentTsCallsClassCoverageEngine { ), machineObserver = CompositeUMachineObserver(candidateObserver, schedulingObserver), observer = interpreterObserver, + machineObserverFactory = stoppedObserver::createMachineObserver, ).use { machine -> - machine.analyzeWithMetadata( + val result = machine.analyzeWithMetadata( methods = contexts.map(CallsCoverageCandidateContext::method), configureInitialState = { method, state -> byMethod.getValue(method).symbolicInputs.initialize(state) }, ) + stoppedObserver.collectRemainingStates() + result } } candidateObserver.finishTiming() diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt index c18b82d7f9..0b583009c2 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt @@ -405,7 +405,7 @@ internal class CompletedCoverageCandidateObserver( ) } - override fun onStateSuspended(state: TsState) { + fun collectStoppedState(state: TsState) { if (!emitSuspendedCandidates) return val started = TimeSource.Monotonic.markNow() diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassUnknownCallJournalTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassUnknownCallJournalTest.kt index 30e56c2ea1..29c27908d5 100644 --- a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassUnknownCallJournalTest.kt +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassUnknownCallJournalTest.kt @@ -270,7 +270,7 @@ class CallsClassUnknownCallJournalTest { val prefix = state.clone() prefix.methodResult = TsMethodResult.NoCall prefix.unsupportedReason = "Synthetic reached-prefix observer contract" - observer.onStateSuspended(prefix) + observer.collectStoppedState(prefix) } observer.onStateTerminated(state = state, stateReachable = stateReachable) }